diff --git a/build.gradle b/build.gradle index bb08eea8..02b1a5a3 100644 --- a/build.gradle +++ b/build.gradle @@ -204,4 +204,3 @@ task releaseBuild { 'cli:build' ) } - diff --git a/cli/src/integration-test/java/com/containersol/minimesos/main/CommandTest.java b/cli/src/integration-test/java/com/containersol/minimesos/main/CommandTest.java index 0f1b2f11..74f7d25b 100644 --- a/cli/src/integration-test/java/com/containersol/minimesos/main/CommandTest.java +++ b/cli/src/integration-test/java/com/containersol/minimesos/main/CommandTest.java @@ -3,10 +3,10 @@ import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; +import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.apache.commons.io.FileUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.json.JSONObject; -import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -33,11 +33,6 @@ public void initTest() { ps = new PrintStream(outputStream, true); } - @After - public void destroyCluster() { - new CommandDestroy().execute(); - } - @Test public void testUpAndDestroy() { CommandUp commandUp = new CommandUp(); @@ -52,8 +47,7 @@ public void testUpAndDestroy() { assertEquals(6, cluster.getMemberProcesses().size()); - CommandDestroy commandDestroy = new CommandDestroy(); - commandDestroy.execute(); + cluster.destroy(new MesosClusterContainersFactory()); assertFalse("Minimesos file at " + minimesosFile + " should be removed", minimesosFile.exists()); } @@ -70,8 +64,7 @@ public void testUp_invalidMinimesosFile() throws IOException { String fileContent = FileUtils.readFileToString(repository.getMinimesosFile(), "UTF-8"); assertEquals("Invalid state file has not been overwritten", cluster.getClusterId(), fileContent); - CommandDestroy commandDestroy = new CommandDestroy(); - commandDestroy.execute(); + cluster.destroy(new MesosClusterContainersFactory()); File minimesosFile = repository.getMinimesosFile(); @@ -91,8 +84,8 @@ public void testUp_alreadyRunning() { assertEquals(firstCluster, secondCluster); - CommandDestroy commandDestroy = new CommandDestroy(); - commandDestroy.execute(); + firstCluster.destroy(new MesosClusterContainersFactory()); + secondCluster.destroy(new MesosClusterContainersFactory()); File minimesosFile = repository.getMinimesosFile(); @@ -112,6 +105,8 @@ public void testInfo_runningCluster() throws IOException { assertTrue(result.contains("Minimesos cluster is running")); assertTrue(result.contains("Mesos version")); + + commandUp.getCluster().destroy(new MesosClusterContainersFactory()); } @Test @@ -136,6 +131,8 @@ public void testState() throws IOException { JSONObject state = new JSONObject(outputStream.toString("UTF-8")); assertEquals("master@" + cluster.getMaster().getIpAddress() + ":5050", state.getString("leader")); + + cluster.destroy(new MesosClusterContainersFactory()); } @Test @@ -154,6 +151,8 @@ public void testInstall() { install.setMarathonFile("src/integration-test/resources/app.json"); install.execute(); + + commandUp.getCluster().destroy(new MesosClusterContainersFactory()); } @Test(expected = MinimesosException.class) @@ -166,6 +165,8 @@ public void testInstall_alreadyRunning() { install.execute(); install.execute(); + + commandUp.getCluster().destroy(new MesosClusterContainersFactory()); } @Test @@ -190,6 +191,7 @@ public void testCompleteInitFile() throws UnsupportedEncodingException { assertTrue(result.contains("MINIMESOS_MARATHON")); + commandUp.getCluster().destroy(new MesosClusterContainersFactory()); } } diff --git a/cli/src/main/java/com/containersol/minimesos/main/CommandDestroy.java b/cli/src/main/java/com/containersol/minimesos/main/CommandDestroy.java index f854f0c2..c3120fa3 100644 --- a/cli/src/main/java/com/containersol/minimesos/main/CommandDestroy.java +++ b/cli/src/main/java/com/containersol/minimesos/main/CommandDestroy.java @@ -27,7 +27,6 @@ public void execute() { MesosCluster cluster = repository.loadCluster(clusterFactory); if (cluster != null) { cluster.destroy(clusterFactory); - repository.deleteClusterFile(); LOGGER.info("Destroyed minimesos cluster with ID " + cluster.getClusterId()); } else { LOGGER.info("Minimesos cluster is not running"); diff --git a/cli/src/main/java/com/containersol/minimesos/main/CommandInit.java b/cli/src/main/java/com/containersol/minimesos/main/CommandInit.java index e15b0253..ddfdccee 100644 --- a/cli/src/main/java/com/containersol/minimesos/main/CommandInit.java +++ b/cli/src/main/java/com/containersol/minimesos/main/CommandInit.java @@ -51,7 +51,7 @@ public String getName() { @Override public void execute() { - File minimesosFile = new File(MesosCluster.getHostDir(), ClusterConfig.DEFAULT_CONFIG_FILE); + File minimesosFile = new File(MesosCluster.getClusterHostDir(), ClusterConfig.DEFAULT_CONFIG_FILE); if (minimesosFile.exists()) { throw new MinimesosException("A minimesosFile already exists in this directory"); diff --git a/minimesos/build.gradle b/minimesos/build.gradle index 98a9022e..41c07fa2 100644 --- a/minimesos/build.gradle +++ b/minimesos/build.gradle @@ -25,7 +25,7 @@ sourceSets { dependencies { compile 'org.codehaus.groovy:groovy-all:2.4.5' - compile 'com.github.docker-java:docker-java:3.0.6' + compile 'com.github.docker-java:docker-java:3.0.7' compile 'junit:junit:4.11' compile 'com.jayway.awaitility:awaitility:1.6.3' compile 'com.mashape.unirest:unirest-java:1.4.8' diff --git a/minimesos/src/main/java/com/containersol/minimesos/cluster/ClusterRepository.java b/minimesos/src/main/java/com/containersol/minimesos/cluster/ClusterRepository.java index 33b93903..1b7483b0 100644 --- a/minimesos/src/main/java/com/containersol/minimesos/cluster/ClusterRepository.java +++ b/minimesos/src/main/java/com/containersol/minimesos/cluster/ClusterRepository.java @@ -84,7 +84,7 @@ public File getMinimesosFile() { * @return directory, where minimesos stores ID file */ public File getMinimesosDir() { - File hostDir = MesosCluster.getHostDir(); + File hostDir = MesosCluster.getClusterHostDir(); File minimesosDir = new File(hostDir, ".minimesos"); if (!minimesosDir.exists()) { if (!minimesosDir.mkdirs()) { diff --git a/minimesos/src/main/java/com/containersol/minimesos/cluster/MesosCluster.java b/minimesos/src/main/java/com/containersol/minimesos/cluster/MesosCluster.java index 8afbc9fa..e38311e8 100644 --- a/minimesos/src/main/java/com/containersol/minimesos/cluster/MesosCluster.java +++ b/minimesos/src/main/java/com/containersol/minimesos/cluster/MesosCluster.java @@ -53,6 +53,8 @@ public class MesosCluster { private List memberProcesses = Collections.synchronizedList(new ArrayList<>()); + private ClusterRepository repository = new ClusterRepository(); + private boolean running = false; /** @@ -193,7 +195,7 @@ public void destroy(MesosClusterFactory factory) { if (clusterId != null) { factory.destroyRunningCluster(clusterId); - File sandboxLocation = new File(getHostDir(), ".minimesos/sandbox-" + clusterId); + File sandboxLocation = new File(MesosCluster.getClusterHostDir(), ".minimesos/sandbox-" + clusterId); if (sandboxLocation.exists()) { try { FileUtils.forceDelete(sandboxLocation); @@ -207,8 +209,9 @@ public void destroy(MesosClusterFactory factory) { LOGGER.info("Minimesos cluster is not running"); } - this.running = false; + repository.deleteClusterFile(); + this.running = false; } /** @@ -352,11 +355,11 @@ public void waitForState(final Predicate predicate) { } /** - * Returns current user directory, which is mapped to host + * Returns the directory on the host from which the cluster was created. * - * @return container directory, which is mapped to current directory on host + * @return directory */ - public static File getHostDir() { + public static File getClusterHostDir() { String sp = System.getProperty(MINIMESOS_HOST_DIR_PROPERTY); if (sp == null) { sp = System.getProperty("user.dir"); @@ -397,7 +400,7 @@ public static InputStream getInputStream(String location) { // location is not an absolute URI, therefore treat it as relative or absolute path File file = new File(location); if (!file.exists()) { - file = new File(getHostDir(), location); + file = new File(getClusterHostDir(), location); } if (file.exists()) { diff --git a/minimesos/src/main/java/com/containersol/minimesos/docker/DockerContainersUtil.java b/minimesos/src/main/java/com/containersol/minimesos/docker/DockerContainersUtil.java index 89ea88b9..61f57a47 100644 --- a/minimesos/src/main/java/com/containersol/minimesos/docker/DockerContainersUtil.java +++ b/minimesos/src/main/java/com/containersol/minimesos/docker/DockerContainersUtil.java @@ -195,12 +195,7 @@ public void onNext(Frame item) { public static void pullImage(String imageName, String imageVersion, long timeoutSecs) { try { final CompletableFuture result = new CompletableFuture<>(); - DockerClientFactory.build().pullImageCmd(imageName).withTag(imageVersion).exec(new FastFailPullImageResultCallback(result)); - result.get(timeoutSecs, TimeUnit.SECONDS); - } catch (TimeoutException e) { - throw new MinimesosException(String.format("# Timeout while pulling image from registry. Try executing the command below manually%ndocker pull %s:%s", imageName, imageVersion), e); - } catch (ExecutionException e) { - throw new MinimesosException(String.format("# Error pulling image from registry. Try executing the command below manually%ndocker pull %s:%s", imageName, imageVersion), e); + DockerClientFactory.build().pullImageCmd(imageName).withTag(imageVersion).exec(new PullImageResultCallback()).awaitCompletion(); } catch (InterruptedException | RuntimeException e) { throw new MinimesosException("Error pulling image or image not found in registry: " + imageName + ":" + imageVersion, e); } @@ -229,32 +224,4 @@ public static Container getContainer(String containerId) { } return container; } - - private static class FastFailPullImageResultCallback extends PullImageResultCallback { - - private final CompletableFuture result; - - public FastFailPullImageResultCallback(CompletableFuture result) { - this.result = result; - } - - @Override - public void onNext(PullResponseItem item) { - String status = item.getStatus(); - if (status == null) { - result.completeExceptionally(new MinimesosException("docker failed to pull image")); - } - } - - @Override - public void onComplete() { - super.onComplete(); - result.complete(null); - } - - @Override - public void onError(Throwable throwable) { - throw new MinimesosException("Pulling of image. However the image is not found: " + throwable.getMessage()); - } - } } diff --git a/minimesos/src/main/java/com/containersol/minimesos/integrationtest/container/AbstractContainer.java b/minimesos/src/main/java/com/containersol/minimesos/integrationtest/container/AbstractContainer.java index c32a0b4c..ccb9e875 100644 --- a/minimesos/src/main/java/com/containersol/minimesos/integrationtest/container/AbstractContainer.java +++ b/minimesos/src/main/java/com/containersol/minimesos/integrationtest/container/AbstractContainer.java @@ -30,7 +30,7 @@ public abstract class AbstractContainer implements ClusterProcess { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractContainer.class); - private static final int IMAGE_PULL_TIMEOUT_SECS = 8 * 60; + private static final int IMAGE_PULL_TIMEOUT_SECS = 30; private MesosCluster cluster; private final ContainerConfig config; diff --git a/minimesos/src/main/java/com/containersol/minimesos/marathon/MarathonContainer.java b/minimesos/src/main/java/com/containersol/minimesos/marathon/MarathonContainer.java index 82fec041..e15369a0 100644 --- a/minimesos/src/main/java/com/containersol/minimesos/marathon/MarathonContainer.java +++ b/minimesos/src/main/java/com/containersol/minimesos/marathon/MarathonContainer.java @@ -224,7 +224,7 @@ public String replaceTokens(String source) { // replace independent from roles tokens String masterContainer = cluster.getMaster().getContainerId(); updatedJson = replaceToken(updatedJson, MesosCluster.TOKEN_NETWORK_GATEWAY, DockerContainersUtil.getGatewayIpAddress(masterContainer)); - updatedJson = replaceToken(updatedJson, TOKEN_HOST_DIR, MesosCluster.getHostDir().getAbsolutePath()); + updatedJson = replaceToken(updatedJson, TOKEN_HOST_DIR, MesosCluster.getClusterHostDir().getAbsolutePath()); return updatedJson; } diff --git a/minimesos/src/main/java/com/containersol/minimesos/mesos/MesosAgentContainer.java b/minimesos/src/main/java/com/containersol/minimesos/mesos/MesosAgentContainer.java index 9ba03244..14dc30ac 100644 --- a/minimesos/src/main/java/com/containersol/minimesos/mesos/MesosAgentContainer.java +++ b/minimesos/src/main/java/com/containersol/minimesos/mesos/MesosAgentContainer.java @@ -9,6 +9,7 @@ import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.Link; +import com.github.dockerjava.api.model.Volume; import java.util.ArrayList; import java.util.List; @@ -24,7 +25,15 @@ public class MesosAgentContainer extends MesosContainerImpl implements MesosAgen private MesosAgentConfig config; - private final static String MESOS_AGENT_SANDBOX_DIR = "/tmp/mesos"; + private final static String MESOS_AGENT_WORK_DIR = "/var/lib/mesos/"; + + private String hostName; + + public MesosAgentContainer(MesosAgentConfig agentConfig) { + super(agentConfig); + this.config = agentConfig; + this.hostName = getRole() + "-" + getUuid(); + } public MesosAgentContainer(MesosCluster cluster, String uuid, String containerId) { this(cluster, uuid, containerId, new MesosAgentConfig(cluster.getConfiguredMesosVersion())); @@ -32,12 +41,7 @@ public MesosAgentContainer(MesosCluster cluster, String uuid, String containerId private MesosAgentContainer(MesosCluster cluster, String uuid, String containerId, MesosAgentConfig config) { super(cluster, uuid, containerId, config); - this.config = config; - } - - public MesosAgentContainer(MesosAgentConfig agentConfig) { - super(agentConfig); - this.config = agentConfig; + this.config = config; } @Override @@ -55,24 +59,26 @@ public int getServicePort() { } public CreateContainerCmd getBaseCommand() { - String hostDir = MesosCluster.getHostDir().getAbsolutePath(); + String hostDir = MesosCluster.getClusterHostDir().getAbsolutePath(); List binds = new ArrayList<>(); - binds.add(Bind.parse("/var/run/docker.sock:/var/run/docker.sock")); + binds.add(Bind.parse("/var/run/docker.sock:/var/run/docker.sock:rw")); binds.add(Bind.parse("/sys/fs/cgroup:/sys/fs/cgroup")); binds.add(Bind.parse(hostDir + ":" + hostDir)); if (getCluster().getMapAgentSandboxVolume()) { - binds.add(Bind.parse(String.format("%s:%s:rw", hostDir + "/.minimesos/sandbox-" + getClusterId() + "/agent-" + getUuid(), MESOS_AGENT_SANDBOX_DIR))); + binds.add(Bind.parse(String.format("%s:%s:rw", hostDir + "/.minimesos/sandbox-" + getClusterId() + "/" + hostName, MESOS_AGENT_WORK_DIR + hostName + "/slaves"))); } return DockerClientFactory.build().createContainerCmd(getImageName() + ":" + getImageTag()) - .withName(getName()) - .withPrivileged(true) - .withEnv(newEnvironment() - .withValues(getMesosAgentEnvVars()) - .withValues(getSharedEnvVars()) - .createEnvironment()) - .withPidMode("host") - .withLinks(new Link(getZooKeeper().getContainerId(), "minimesos-zookeeper")) - .withBinds(binds.stream().toArray(Bind[]::new)); + .withName(getName()) + .withHostName(hostName) + .withPrivileged(true) + .withVolumes(new Volume(MESOS_AGENT_WORK_DIR + hostName)) + .withEnv(newEnvironment() + .withValues(getMesosAgentEnvVars()) + .withValues(getSharedEnvVars()) + .createEnvironment()) + .withPidMode("host") + .withLinks(new Link(getZooKeeper().getContainerId(), "minimesos-zookeeper")) + .withBinds(binds.stream().toArray(Bind[]::new)); } @Override @@ -92,19 +98,21 @@ protected CreateContainerCmd dockerCommand() { return getBaseCommand() .withExposedPorts(exposedPorts.toArray(new ExposedPort[exposedPorts.size()])); - } private Map getMesosAgentEnvVars() { Map envs = new TreeMap<>(); + envs.put("GLOG_v", "1"); envs.put("MESOS_RESOURCES", getResources()); - envs.put("MESOS_IMAGE_PROVIDERS", "docker,appc"); + envs.put("MESOS_WORK_DIR", MESOS_AGENT_WORK_DIR + hostName); + envs.put("MESOS_DOCKER_STORE_DIR", MESOS_AGENT_WORK_DIR + hostName + "/store/docker"); envs.put("MESOS_ISOLATION", "filesystem/linux,docker/runtime,cgroups/cpu,cgroups/mem"); + envs.put("MESOS_IMAGE_PROVIDERS", "docker"); + envs.put("MESOS_SYSTEMD_ENABLE_SUPPORT", "false"); envs.put("MESOS_PORT", String.valueOf(getServicePort())); envs.put("MESOS_MASTER", getFormattedZKAddress()); envs.put("MESOS_SWITCH_USER", "false"); envs.put("MESOS_LOGGING_LEVEL", getLoggingLevel()); - envs.put("MESOS_WORK_DIR", MESOS_AGENT_SANDBOX_DIR); envs.put("SERVICE_IGNORE", "1"); return envs; } diff --git a/minimesos/src/main/java/com/containersol/minimesos/mesos/MesosContainerImpl.java b/minimesos/src/main/java/com/containersol/minimesos/mesos/MesosContainerImpl.java index ed811575..4a6c727c 100644 --- a/minimesos/src/main/java/com/containersol/minimesos/mesos/MesosContainerImpl.java +++ b/minimesos/src/main/java/com/containersol/minimesos/mesos/MesosContainerImpl.java @@ -56,7 +56,6 @@ protected final Map getSharedEnvVars() { envs.put("MESOS_ISOLATOR", "cgroups/cpu,cgroups/mem"); envs.put("MESOS_LOG_DIR", "/var/log"); envs.put("MESOS_LOGGING_LEVEL", getLoggingLevel()); - envs.put("MESOS_WORK_DIR", "/tmp/mesos"); return envs; }