forked from awslabs/ec2-spot-jenkins-plugin
-
Notifications
You must be signed in to change notification settings - Fork 81
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
#76 Don't provision additional capacity until node will be online
- Loading branch information
Showing
14 changed files
with
799 additions
and
427 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
92 changes: 92 additions & 0 deletions
92
src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetOnlineChecker.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package com.amazon.jenkins.ec2fleet; | ||
|
|
||
| import com.google.common.util.concurrent.SettableFuture; | ||
| import hudson.model.Computer; | ||
| import hudson.model.Node; | ||
| import hudson.util.DaemonThreadFactory; | ||
|
|
||
| import javax.annotation.concurrent.ThreadSafe; | ||
| import java.util.concurrent.Executors; | ||
| import java.util.concurrent.ScheduledExecutorService; | ||
| import java.util.concurrent.TimeUnit; | ||
| import java.util.logging.Level; | ||
| import java.util.logging.Logger; | ||
|
|
||
| /** | ||
| * Keep {@link hudson.slaves.NodeProvisioner.PlannedNode#future} not resolved until node will not be online | ||
| * or timeout reached. | ||
| * <p> | ||
| * Default Jenkins node capacity planner {@link hudson.slaves.NodeProvisioner.Strategy} count planned nodes | ||
| * as available capacity, but exclude offline computers {@link Computer#isOnline()} from available capacity. | ||
| * Because EC2 instance requires some time when it was added into fleet to start up, next situation happens: | ||
| * plugin add described capacity as node into Jenkins pool, but Jenkins keeps it as offline as no way to connect, | ||
| * during time when node is offline, Jenkins will try to request more nodes from plugin as offline nodes | ||
| * excluded from capacity. | ||
| * <p> | ||
| * This class fix this situation and keep planned node until instance is really online, so Jenkins planner | ||
| * count planned node as available capacity and doesn't request more. | ||
| * <p> | ||
| * Based on https://github.com/jenkinsci/ec2-plugin/blob/master/src/main/java/hudson/plugins/ec2/EC2Cloud.java#L640 | ||
| * | ||
| * @see EC2FleetCloud | ||
| * @see EC2FleetNode | ||
| */ | ||
| @SuppressWarnings("WeakerAccess") | ||
| @ThreadSafe | ||
| class EC2FleetOnlineChecker implements Runnable { | ||
|
|
||
| private static final Logger LOGGER = Logger.getLogger(EC2FleetOnlineChecker.class.getName()); | ||
| // use daemon thread, so no problem when stop jenkins | ||
| private static final ScheduledExecutorService EXECUTOR = Executors.newSingleThreadScheduledExecutor(new DaemonThreadFactory()); | ||
|
|
||
| public static void start(final Node node, final SettableFuture<Node> future, final long timeout, final long interval) { | ||
| EXECUTOR.execute(new EC2FleetOnlineChecker(node, future, timeout, interval)); | ||
| } | ||
|
|
||
| private final long start; | ||
| private final Node node; | ||
| private final SettableFuture<Node> future; | ||
| private final long timeout; | ||
| private final long interval; | ||
|
|
||
| private EC2FleetOnlineChecker( | ||
| final Node node, final SettableFuture<Node> future, final long timeout, final long interval) { | ||
| this.start = System.currentTimeMillis(); | ||
| this.node = node; | ||
| this.future = future; | ||
| this.timeout = timeout; | ||
| this.interval = interval; | ||
| } | ||
|
|
||
| @Override | ||
| public void run() { | ||
| if (future.isCancelled()) { | ||
| return; | ||
| } | ||
|
|
||
| if (timeout < 1) { | ||
| future.set(node); | ||
| LOGGER.log(Level.INFO, String.format("%s connection check disabled, resolve planned node", node.getNodeName())); | ||
| return; | ||
| } | ||
|
|
||
| if (System.currentTimeMillis() - start > timeout) { | ||
| future.setException(new IllegalStateException( | ||
| "Fail to provision node, cannot connect to " + node.getNodeName() + " in " + timeout + " msec")); | ||
| return; | ||
| } | ||
|
|
||
| final Computer computer = node.toComputer(); | ||
| if (computer != null) { | ||
| if (computer.isOnline()) { | ||
| future.set(node); | ||
| LOGGER.log(Level.INFO, String.format("%s connected, resolve planned node", node.getNodeName())); | ||
| return; | ||
| } | ||
| } | ||
|
|
||
| LOGGER.log(Level.INFO, String.format("%s no connection, wait before retry", node.getNodeName())); | ||
| EXECUTOR.schedule(this, interval, TimeUnit.MILLISECONDS); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-initOnlineTimeoutSec.html
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| Specify maximum time which Jenkins will wait for EC2 instance startup before | ||
| trying to request new instance from fleet. | ||
|
|
||
| <p> | ||
| By default Jenkins expect that node will be available immediately, however EC2 instance | ||
| requires some time to be provision, start and be ready for actual builds. In result | ||
| Jenkins might request more capacity then actually required (over-provision). This setting | ||
| avoids over-provision by force Jenkins to wait until EC2 instance will be online and | ||
| don't request more capacity. | ||
| </p> | ||
|
|
||
| <p> | ||
| In case of positive value, will force Jenkins to wait when EC2 instance will be online, | ||
| no more then specified time in seconds. Cannot be negative. | ||
| In case of <code>0</code>, timeout will be disabled and Jenkins will | ||
| not wait any time to start up EC2 instance, which could lead to over-provision. | ||
| </p> | ||
| <p> | ||
| <b>Note</b> Be careful with very big values (more 5 min) for this settings. In case of any problem | ||
| with node (no java, wrong version of java, Jenkins agent cannot start), so it cannot be online. | ||
| Jenkins will wait, without requesting additional capacity. | ||
| </p> | ||
| <p> | ||
| Behavior of previous versions of plugin could be represented as this version | ||
| with timeout set to <code>0</code>. | ||
| </p> |
Oops, something went wrong.