Merge pull request #114 from jenkinsci/scale-executors-by-weight

* Scale executors by LaunchSpecification weight

This works nicely with a SpotFleet weighted by vCPU count (easy in the
management console).

* Rebase of #32 allow to scale number of executors by weight

src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloud.java

@@ -101,6 +101,7 @@ public class EC2FleetCloud extends Cloud {
     private final Integer numExecutors;
     private final boolean addNodeOnlyIfRunning;
     private final boolean restrictUsage;
+    private final boolean scaleExecutorsByWeight;
     private final Integer initOnlineTimeoutSec;
     private final Integer initOnlineCheckIntervalSec;
 
@@ -135,7 +136,8 @@ public EC2FleetCloud(final String name,
                          final boolean restrictUsage,
                          final boolean disableTaskResubmit,
                          final Integer initOnlineTimeoutSec,
-                         final Integer initOnlineCheckIntervalSec) {
+                         final Integer initOnlineCheckIntervalSec,
+                         final boolean scaleExecutorsByWeight) {
         super(StringUtils.isBlank(name) ? FLEET_CLOUD_ID : name);
         initCaches();
         this.credentialsId = credentialsId;
@@ -154,6 +156,7 @@ public EC2FleetCloud(final String name,
         this.numExecutors = numExecutors;
         this.addNodeOnlyIfRunning = addNodeOnlyIfRunning;
         this.restrictUsage = restrictUsage;
+        this.scaleExecutorsByWeight = scaleExecutorsByWeight;
         this.disableTaskResubmit = disableTaskResubmit;
         this.initOnlineTimeoutSec = initOnlineTimeoutSec;
         this.initOnlineCheckIntervalSec = initOnlineCheckIntervalSec;
@@ -233,6 +236,10 @@ public Integer getNumExecutors() {
         return numExecutors;
     }
 
+    public boolean isScaleExecutorsByWeight() {
+        return scaleExecutorsByWeight;
+    }
+
     public String getJvmSettings() {
         return "";
     }
@@ -382,7 +389,7 @@ public synchronized FleetStateStats updateStatus() {
                         StringUtils.join(newFleetInstances, ", ") + "]");
             }
             for (final String instanceId : newFleetInstances) {
-                addNewSlave(ec2, instanceId);
+                addNewSlave(ec2, instanceId, stats);
             }
         } catch (final Exception ex) {
             warning(ex, "Unable to set label on node");
@@ -483,7 +490,7 @@ private synchronized void removeNode(String instanceId) {
      * @param ec2        ec2 client
      * @param instanceId instance id
      */
-    private void addNewSlave(final AmazonEC2 ec2, final String instanceId) throws Exception {
+    private void addNewSlave(final AmazonEC2 ec2, final String instanceId, FleetStateStats stats) throws Exception {
         final DescribeInstancesResult result = ec2.describeInstances(
                 new DescribeInstancesRequest().withInstanceIds(instanceId));
         //Can't find this instance, skip it
@@ -500,16 +507,26 @@ private void addNewSlave(final AmazonEC2 ec2, final String instanceId) throws Ex
         if (address == null) return; // Wait some more...
 
         // Generate a random FS root if one isn't specified
-        String effectiveFsRoot = fsRoot;
-        if (StringUtils.isBlank(effectiveFsRoot)) {
+        final String effectiveFsRoot;
+        if (StringUtils.isBlank(fsRoot)) {
             effectiveFsRoot = "/tmp/jenkins-" + UUID.randomUUID().toString().substring(0, 8);
+        } else {
+            effectiveFsRoot = fsRoot;
+        }
+
+        final Double instanceTypeWeight = stats.getInstanceTypeWeights().get(instance.getInstanceType());
+        final int effectiveNumExecutors;
+        if (scaleExecutorsByWeight && instanceTypeWeight != null) {
+            effectiveNumExecutors = (int) Math.max(Math.round(numExecutors * instanceTypeWeight), 1);
+        } else {
+            effectiveNumExecutors = numExecutors;
         }
 
         final EC2FleetAutoResubmitComputerLauncher computerLauncher = new EC2FleetAutoResubmitComputerLauncher(
                 computerConnector.launch(address, TaskListener.NULL), disableTaskResubmit);
         final Node.Mode nodeMode = restrictUsage ? Node.Mode.EXCLUSIVE : Node.Mode.NORMAL;
         final EC2FleetNode node = new EC2FleetNode(instanceId, "Fleet slave for " + instanceId,
-                effectiveFsRoot, numExecutors.toString(), nodeMode, labelString, new ArrayList<NodeProperty<?>>(),
+                effectiveFsRoot, effectiveNumExecutors, nodeMode, labelString, new ArrayList<NodeProperty<?>>(),
                 name, computerLauncher);
 
         // Initialize our retention strategy

src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetNode.java

@@ -18,7 +18,7 @@ public class EC2FleetNode extends Slave implements EphemeralNode {
     private final String cloudName;
 
     @SuppressWarnings("WeakerAccess")
-    public EC2FleetNode(final String name, final String nodeDescription, final String remoteFS, final String numExecutors, final Mode mode, final String label,
+    public EC2FleetNode(final String name, final String nodeDescription, final String remoteFS, final int numExecutors, final Mode mode, final String label,
                         final List<? extends NodeProperty<?>> nodeProperties, final String cloudName, ComputerLauncher launcher) throws IOException, Descriptor.FormException {
         super(name, nodeDescription, remoteFS, numExecutors, mode, label,
                 launcher, RetentionStrategy.NOOP, nodeProperties);

src/main/java/com/amazon/jenkins/ec2fleet/FleetStateStats.java

@@ -6,43 +6,49 @@
 import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesResult;
 import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsRequest;
 import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsResult;
+import com.amazonaws.services.ec2.model.SpotFleetLaunchSpecification;
 import com.amazonaws.services.ec2.model.SpotFleetRequestConfig;
+import com.amazonaws.services.ec2.model.SpotFleetRequestConfigData;
 
 import javax.annotation.Nonnegative;
 import javax.annotation.Nonnull;
+import javax.annotation.concurrent.ThreadSafe;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 /**
  * @see EC2FleetCloud
  */
-@SuppressWarnings("unused")
+@SuppressWarnings({"unused", "WeakerAccess"})
+@ThreadSafe
 public final class FleetStateStats {
 
-    private @Nonnull
-    final String fleetId;
-    private @Nonnegative
-    final int numActive;
-    private @Nonnegative
-    final int numDesired;
-    private @Nonnull
-    final String state;
-    private @Nonnull
-    final Set<String> instances;
-    private @Nonnull
-    final String label;
+    @Nonnull
+    private final String fleetId;
+    @Nonnegative
+    private final int numActive;
+    @Nonnegative
+    private final int numDesired;
+    @Nonnull
+    private final String state;
+    @Nonnull
+    private final Set<String> instances;
+    @Nonnull
+    private final Map<String, Double> instanceTypeWeights;
 
     public FleetStateStats(final @Nonnull String fleetId,
                            final int numDesired, final @Nonnull String state,
                            final @Nonnull Set<String> instances,
-                           final @Nonnull String label) {
+                           final @Nonnull Map<String, Double> instanceTypeWeights) {
         this.fleetId = fleetId;
         this.numActive = instances.size();
         this.numDesired = numDesired;
         this.state = state;
         this.instances = instances;
-        this.label = label;
+        this.instanceTypeWeights = instanceTypeWeights;
     }
 
     @Nonnull
@@ -69,8 +75,8 @@ public Set<String> getInstances() {
     }
 
     @Nonnull
-    public String getLabel() {
-        return label;
+    public Map<String, Double> getInstanceTypeWeights() {
+        return instanceTypeWeights;
     }
 
     public static FleetStateStats readClusterState(final AmazonEC2 ec2, final String fleetId, final String label) {
@@ -95,10 +101,25 @@ public static FleetStateStats readClusterState(final AmazonEC2 ec2, final String
             throw new IllegalStateException("Fleet " + fleetId + " can't be described");
 
         final SpotFleetRequestConfig fleetConfig = fleet.getSpotFleetRequestConfigs().get(0);
+        final SpotFleetRequestConfigData fleetRequestConfig = fleetConfig.getSpotFleetRequestConfig();
+
+        // Index configured instance types by weight:
+        final Map<String, Double> instanceTypeWeights = new HashMap<>();
+        for (SpotFleetLaunchSpecification launchSpecification : fleetRequestConfig.getLaunchSpecifications()) {
+            final String instanceType = launchSpecification.getInstanceType();
+            if (instanceType == null) continue;
+
+            final Double instanceWeight = launchSpecification.getWeightedCapacity();
+            final Double existingWeight = instanceTypeWeights.get(instanceType);
+            if (instanceWeight == null || (existingWeight != null && existingWeight > instanceWeight)) {
+                continue;
+            }
+            instanceTypeWeights.put(instanceType, instanceWeight);
+        }
 
         return new FleetStateStats(fleetId,
-                fleetConfig.getSpotFleetRequestConfig().getTargetCapacity(),
+                fleetRequestConfig.getTargetCapacity(),
                 fleetConfig.getSpotFleetRequestState(), instances,
-                label);
+                instanceTypeWeights);
     }
 }

src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/config.jelly

@@ -70,6 +70,10 @@
       <f:textbox clazz="required positive-number" default="1" />
     </f:entry>
 
+    <f:entry title="${%Scale Executors by Weight}" field="scaleExecutorsByWeight">
+      <f:checkbox />
+    </f:entry>
+
     <f:entry title="${%Max Idle Minutes Before Scaledown}" field="idleMinutes">
       <f:number clazz="required number" min="0" default="0" />
     </f:entry>

src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-scaleExecutorsByWeight.html

@@ -0,0 +1,64 @@
+If unchecked which is default, plugin will not use
+<a href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-instance-weighting">instance
+    weight</a>
+information to scale number of executors per node, and just set number of executors defined in
+configuration field <code>Number of Executors</code>
+
+<p>
+    When it's checked, plugin consumes instance weight information provided by Launch Specification
+    and uses it to scale node number of executors from configuration field <code>Number of Executors</code>
+    <b>Note</b> current implementation doesn't support Launch Template only Launch Specification
+</p>
+
+<p>
+    Example (here instance type from launch specification match with
+    launched instance type):
+
+<table>
+    <thead>
+    <tr>
+        <th>Number of Executors</th>
+        <th>Instance Weight</th>
+        <th>Effective<br>Number of Executors</th>
+    </tr>
+    </thead>
+    <tbody>
+    <tr>
+        <td>1</td>
+        <td>1</td>
+        <td>1</td>
+    </tr>
+    <tr>
+        <td>1</td>
+        <td>0.5</td>
+        <td>1</td>
+    </tr>
+    <tr>
+        <td>1</td>
+        <td>0.1</td>
+        <td>1</td>
+    </tr>
+    <tr>
+        <td>10</td>
+        <td>0.1</td>
+        <td>1</td>
+    </tr>
+    <tr>
+        <td>1</td>
+        <td>1.5</td>
+        <td>2</td>
+    </tr>
+    <tr>
+        <td>1</td>
+        <td>1.44</td>
+        <td>1</td>
+    </tr>
+    </tbody>
+</table>
+</p>
+
+<p>
+    Plugin always set number of executors <b>at least one</b>.
+    If launched instance type doesn't match any weight in launch specification
+    regular number of executors will be used without any scale.
+</p>

src/test/java/com/amazon/jenkins/ec2fleet/AutoResubmitIntegrationTest.java

@@ -79,7 +79,7 @@ public void should_successfully_resubmit_freestyle_task() throws Exception {
         EC2FleetCloud cloud = new EC2FleetCloud(null, "credId", null, "region",
                 null, "fId", "momo", null, new SingleLocalComputerConnector(j), false, false,
                 0, 0, 10, 1, false, false,
-                false, 0, 0);
+                false, 0, 0, false);
         j.jenkins.clouds.add(cloud);
 
         List<QueueTaskFuture> rs = getQueueTaskFutures(1);
@@ -115,7 +115,7 @@ public void should_successfully_resubmit_parametrized_task() throws Exception {
         EC2FleetCloud cloud = new EC2FleetCloud(null, "credId", null, "region",
                 null, "fId", "momo", null, new SingleLocalComputerConnector(j), false, false,
                 0, 0, 10, 1, false, false,
-                false, 0, 0);
+                false, 0, 0, false);
         j.jenkins.clouds.add(cloud);
 
         List<QueueTaskFuture> rs = new ArrayList<>();
@@ -171,7 +171,7 @@ public void should_not_resubmit_if_disabled() throws Exception {
         EC2FleetCloud cloud = new EC2FleetCloud(null, "credId", null, "region",
                 null, "fId", "momo", null, new SingleLocalComputerConnector(j), false, false,
                 0, 0, 10, 1, false, false,
-                true, 0, 0);
+                true, 0, 0, false);
         j.jenkins.clouds.add(cloud);
 
         List<QueueTaskFuture> rs = getQueueTaskFutures(1);