Skip to content
This repository has been archived by the owner on Jun 22, 2018. It is now read-only.

Commit

Permalink
Mesos containerizer can now launch Docker images.
Browse files Browse the repository at this point in the history 
When the Docker store is part of the container the layers cannot be untarred. The following error occurs:

`tar: etc/rc0.d/.wh.K01sendsigs: Cannot open: Operation not permitted`

Therefore a volume /var/lib/mesos/agent-$UUID is created under which the Docker store lives. When the agent is destroyed the volume is removed. The existing sandbox host volume maps to a subdirectory of the agent volume and will be empty when the cluster is destroyed.
  • Loading branch information
@frankscholten
frankscholten committed Jan 31, 2017
1 parent 445cc84 commit 053cc55
Show file tree
Hide file tree
Showing 12 changed files with 59 additions and 82 deletions.
1 change: 0 additions & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,4 +204,3 @@ task releaseBuild {
'cli:build'
)
}

26 changes: 14 additions & 12 deletions cli/src/integration-test/java/com/containersol/minimesos/main/CommandTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
import com.containersol.minimesos.MinimesosException;
import com.containersol.minimesos.cluster.ClusterRepository;
import com.containersol.minimesos.cluster.MesosCluster;
import com.containersol.minimesos.mesos.MesosClusterContainersFactory;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

Expand All @@ -33,11 +33,6 @@ public void initTest() {
ps = new PrintStream(outputStream, true);
}

@After
public void destroyCluster() {
new CommandDestroy().execute();
}

@Test
public void testUpAndDestroy() {
CommandUp commandUp = new CommandUp();
Expand All @@ -52,8 +47,7 @@ public void testUpAndDestroy() {

assertEquals(6, cluster.getMemberProcesses().size());

CommandDestroy commandDestroy = new CommandDestroy();
commandDestroy.execute();
cluster.destroy(new MesosClusterContainersFactory());

assertFalse("Minimesos file at " + minimesosFile + " should be removed", minimesosFile.exists());
}
Expand All @@ -70,8 +64,7 @@ public void testUp_invalidMinimesosFile() throws IOException {
String fileContent = FileUtils.readFileToString(repository.getMinimesosFile(), "UTF-8");
assertEquals("Invalid state file has not been overwritten", cluster.getClusterId(), fileContent);

CommandDestroy commandDestroy = new CommandDestroy();
commandDestroy.execute();
cluster.destroy(new MesosClusterContainersFactory());

File minimesosFile = repository.getMinimesosFile();

Expand All @@ -91,8 +84,8 @@ public void testUp_alreadyRunning() {

assertEquals(firstCluster, secondCluster);

CommandDestroy commandDestroy = new CommandDestroy();
commandDestroy.execute();
firstCluster.destroy(new MesosClusterContainersFactory());
secondCluster.destroy(new MesosClusterContainersFactory());

File minimesosFile = repository.getMinimesosFile();

Expand All @@ -112,6 +105,8 @@ public void testInfo_runningCluster() throws IOException {

assertTrue(result.contains("Minimesos cluster is running"));
assertTrue(result.contains("Mesos version"));

commandUp.getCluster().destroy(new MesosClusterContainersFactory());
}

@Test
Expand All @@ -136,6 +131,8 @@ public void testState() throws IOException {
JSONObject state = new JSONObject(outputStream.toString("UTF-8"));

assertEquals("master@" + cluster.getMaster().getIpAddress() + ":5050", state.getString("leader"));

cluster.destroy(new MesosClusterContainersFactory());
}

@Test
Expand All @@ -154,6 +151,8 @@ public void testInstall() {
install.setMarathonFile("src/integration-test/resources/app.json");

install.execute();

commandUp.getCluster().destroy(new MesosClusterContainersFactory());
}

@Test(expected = MinimesosException.class)
Expand All @@ -166,6 +165,8 @@ public void testInstall_alreadyRunning() {

install.execute();
install.execute();

commandUp.getCluster().destroy(new MesosClusterContainersFactory());
}

@Test
Expand All @@ -190,6 +191,7 @@ public void testCompleteInitFile() throws UnsupportedEncodingException {

assertTrue(result.contains("MINIMESOS_MARATHON"));

commandUp.getCluster().destroy(new MesosClusterContainersFactory());
}

}
1 change: 0 additions & 1 deletion cli/src/main/java/com/containersol/minimesos/main/CommandDestroy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ public void execute() {
MesosCluster cluster = repository.loadCluster(clusterFactory);
if (cluster != null) {
cluster.destroy(clusterFactory);
repository.deleteClusterFile();
LOGGER.info("Destroyed minimesos cluster with ID " + cluster.getClusterId());
} else {
LOGGER.info("Minimesos cluster is not running");
Expand Down
2 changes: 1 addition & 1 deletion cli/src/main/java/com/containersol/minimesos/main/CommandInit.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public String getName() {

@Override
public void execute() {
File minimesosFile = new File(MesosCluster.getHostDir(), ClusterConfig.DEFAULT_CONFIG_FILE);
File minimesosFile = new File(MesosCluster.getClusterHostDir(), ClusterConfig.DEFAULT_CONFIG_FILE);

if (minimesosFile.exists()) {
throw new MinimesosException("A minimesosFile already exists in this directory");
Expand Down
2 changes: 1 addition & 1 deletion minimesos/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ sourceSets {

dependencies {
compile 'org.codehaus.groovy:groovy-all:2.4.5'
compile 'com.github.docker-java:docker-java:3.0.6'
compile 'com.github.docker-java:docker-java:3.0.7'
compile 'junit:junit:4.11'
compile 'com.jayway.awaitility:awaitility:1.6.3'
compile 'com.mashape.unirest:unirest-java:1.4.8'
Expand Down
2 changes: 1 addition & 1 deletion minimesos/src/main/java/com/containersol/minimesos/cluster/ClusterRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public File getMinimesosFile() {
* @return directory, where minimesos stores ID file
*/
public File getMinimesosDir() {
File hostDir = MesosCluster.getHostDir();
File hostDir = MesosCluster.getClusterHostDir();
File minimesosDir = new File(hostDir, ".minimesos");
if (!minimesosDir.exists()) {
if (!minimesosDir.mkdirs()) {
Expand Down
15 changes: 9 additions & 6 deletions minimesos/src/main/java/com/containersol/minimesos/cluster/MesosCluster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ public class MesosCluster {

private List<ClusterProcess> memberProcesses = Collections.synchronizedList(new ArrayList<>());

private ClusterRepository repository = new ClusterRepository();

private boolean running = false;

/**
Expand Down Expand Up @@ -193,7 +195,7 @@ public void destroy(MesosClusterFactory factory) {
if (clusterId != null) {
factory.destroyRunningCluster(clusterId);

File sandboxLocation = new File(getHostDir(), ".minimesos/sandbox-" + clusterId);
File sandboxLocation = new File(MesosCluster.getClusterHostDir(), ".minimesos/sandbox-" + clusterId);
if (sandboxLocation.exists()) {
try {
FileUtils.forceDelete(sandboxLocation);
Expand All @@ -207,8 +209,9 @@ public void destroy(MesosClusterFactory factory) {
LOGGER.info("Minimesos cluster is not running");
}

this.running = false;
repository.deleteClusterFile();

this.running = false;
}

/**
Expand Down Expand Up @@ -352,11 +355,11 @@ public void waitForState(final Predicate<State> predicate) {
}

/**
* Returns current user directory, which is mapped to host
* Returns the directory on the host from which the cluster was created.
*
* @return container directory, which is mapped to current directory on host
* @return directory
*/
public static File getHostDir() {
public static File getClusterHostDir() {
String sp = System.getProperty(MINIMESOS_HOST_DIR_PROPERTY);
if (sp == null) {
sp = System.getProperty("user.dir");
Expand Down Expand Up @@ -397,7 +400,7 @@ public static InputStream getInputStream(String location) {
// location is not an absolute URI, therefore treat it as relative or absolute path
File file = new File(location);
if (!file.exists()) {
file = new File(getHostDir(), location);
file = new File(getClusterHostDir(), location);
}

if (file.exists()) {
Expand Down
35 changes: 1 addition & 34 deletions minimesos/src/main/java/com/containersol/minimesos/docker/DockerContainersUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,12 +195,7 @@ public void onNext(Frame item) {
public static void pullImage(String imageName, String imageVersion, long timeoutSecs) {
try {
final CompletableFuture<Void> result = new CompletableFuture<>();
DockerClientFactory.build().pullImageCmd(imageName).withTag(imageVersion).exec(new FastFailPullImageResultCallback(result));
result.get(timeoutSecs, TimeUnit.SECONDS);
} catch (TimeoutException e) {
throw new MinimesosException(String.format("# Timeout while pulling image from registry. Try executing the command below manually%ndocker pull %s:%s", imageName, imageVersion), e);
} catch (ExecutionException e) {
throw new MinimesosException(String.format("# Error pulling image from registry. Try executing the command below manually%ndocker pull %s:%s", imageName, imageVersion), e);
DockerClientFactory.build().pullImageCmd(imageName).withTag(imageVersion).exec(new PullImageResultCallback()).awaitCompletion();
} catch (InterruptedException | RuntimeException e) {
throw new MinimesosException("Error pulling image or image not found in registry: " + imageName + ":" + imageVersion, e);
}
Expand Down Expand Up @@ -229,32 +224,4 @@ public static Container getContainer(String containerId) {
}
return container;
}

private static class FastFailPullImageResultCallback extends PullImageResultCallback {

private final CompletableFuture<Void> result;

public FastFailPullImageResultCallback(CompletableFuture<Void> result) {
this.result = result;
}

@Override
public void onNext(PullResponseItem item) {
String status = item.getStatus();
if (status == null) {
result.completeExceptionally(new MinimesosException("docker failed to pull image"));
}
}

@Override
public void onComplete() {
super.onComplete();
result.complete(null);
}

@Override
public void onError(Throwable throwable) {
throw new MinimesosException("Pulling of image. However the image is not found: " + throwable.getMessage());
}
}
}
2 changes: 1 addition & 1 deletion ...src/main/java/com/containersol/minimesos/integrationtest/container/AbstractContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public abstract class AbstractContainer implements ClusterProcess {

private static final Logger LOGGER = LoggerFactory.getLogger(AbstractContainer.class);

private static final int IMAGE_PULL_TIMEOUT_SECS = 8 * 60;
private static final int IMAGE_PULL_TIMEOUT_SECS = 30;

private MesosCluster cluster;
private final ContainerConfig config;
Expand Down
2 changes: 1 addition & 1 deletion minimesos/src/main/java/com/containersol/minimesos/marathon/MarathonContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ public String replaceTokens(String source) {
// replace independent from roles tokens
String masterContainer = cluster.getMaster().getContainerId();
updatedJson = replaceToken(updatedJson, MesosCluster.TOKEN_NETWORK_GATEWAY, DockerContainersUtil.getGatewayIpAddress(masterContainer));
updatedJson = replaceToken(updatedJson, TOKEN_HOST_DIR, MesosCluster.getHostDir().getAbsolutePath());
updatedJson = replaceToken(updatedJson, TOKEN_HOST_DIR, MesosCluster.getClusterHostDir().getAbsolutePath());

return updatedJson;
}
Expand Down
52 changes: 30 additions & 22 deletions minimesos/src/main/java/com/containersol/minimesos/mesos/MesosAgentContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.ExposedPort;
import com.github.dockerjava.api.model.Link;
import com.github.dockerjava.api.model.Volume;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -24,20 +25,23 @@ public class MesosAgentContainer extends MesosContainerImpl implements MesosAgen

private MesosAgentConfig config;

private final static String MESOS_AGENT_SANDBOX_DIR = "/tmp/mesos";
private final static String MESOS_AGENT_WORK_DIR = "/var/lib/mesos/";

private String hostName;

public MesosAgentContainer(MesosAgentConfig agentConfig) {
super(agentConfig);
this.config = agentConfig;
this.hostName = getRole() + "-" + getUuid();
}

public MesosAgentContainer(MesosCluster cluster, String uuid, String containerId) {
this(cluster, uuid, containerId, new MesosAgentConfig(cluster.getConfiguredMesosVersion()));
}

private MesosAgentContainer(MesosCluster cluster, String uuid, String containerId, MesosAgentConfig config) {
super(cluster, uuid, containerId, config);
this.config = config;
}

public MesosAgentContainer(MesosAgentConfig agentConfig) {
super(agentConfig);
this.config = agentConfig;
this.config = config;
}

@Override
Expand All @@ -55,24 +59,26 @@ public int getServicePort() {
}

public CreateContainerCmd getBaseCommand() {
String hostDir = MesosCluster.getHostDir().getAbsolutePath();
String hostDir = MesosCluster.getClusterHostDir().getAbsolutePath();
List<Bind> binds = new ArrayList<>();
binds.add(Bind.parse("/var/run/docker.sock:/var/run/docker.sock"));
binds.add(Bind.parse("/var/run/docker.sock:/var/run/docker.sock:rw"));
binds.add(Bind.parse("/sys/fs/cgroup:/sys/fs/cgroup"));
binds.add(Bind.parse(hostDir + ":" + hostDir));
if (getCluster().getMapAgentSandboxVolume()) {
binds.add(Bind.parse(String.format("%s:%s:rw", hostDir + "/.minimesos/sandbox-" + getClusterId() + "/agent-" + getUuid(), MESOS_AGENT_SANDBOX_DIR)));
binds.add(Bind.parse(String.format("%s:%s:rw", hostDir + "/.minimesos/sandbox-" + getClusterId() + "/" + hostName, MESOS_AGENT_WORK_DIR + hostName + "/slaves")));
}
return DockerClientFactory.build().createContainerCmd(getImageName() + ":" + getImageTag())
.withName(getName())
.withPrivileged(true)
.withEnv(newEnvironment()
.withValues(getMesosAgentEnvVars())
.withValues(getSharedEnvVars())
.createEnvironment())
.withPidMode("host")
.withLinks(new Link(getZooKeeper().getContainerId(), "minimesos-zookeeper"))
.withBinds(binds.stream().toArray(Bind[]::new));
.withName(getName())
.withHostName(hostName)
.withPrivileged(true)
.withVolumes(new Volume(MESOS_AGENT_WORK_DIR + hostName))
.withEnv(newEnvironment()
.withValues(getMesosAgentEnvVars())
.withValues(getSharedEnvVars())
.createEnvironment())
.withPidMode("host")
.withLinks(new Link(getZooKeeper().getContainerId(), "minimesos-zookeeper"))
.withBinds(binds.stream().toArray(Bind[]::new));
}

@Override
Expand All @@ -92,19 +98,21 @@ protected CreateContainerCmd dockerCommand() {

return getBaseCommand()
.withExposedPorts(exposedPorts.toArray(new ExposedPort[exposedPorts.size()]));

}

private Map<String, String> getMesosAgentEnvVars() {
Map<String, String> envs = new TreeMap<>();
envs.put("GLOG_v", "1");
envs.put("MESOS_RESOURCES", getResources());
envs.put("MESOS_IMAGE_PROVIDERS", "docker,appc");
envs.put("MESOS_WORK_DIR", MESOS_AGENT_WORK_DIR + hostName);
envs.put("MESOS_DOCKER_STORE_DIR", MESOS_AGENT_WORK_DIR + hostName + "/store/docker");
envs.put("MESOS_ISOLATION", "filesystem/linux,docker/runtime,cgroups/cpu,cgroups/mem");
envs.put("MESOS_IMAGE_PROVIDERS", "docker");
envs.put("MESOS_SYSTEMD_ENABLE_SUPPORT", "false");
envs.put("MESOS_PORT", String.valueOf(getServicePort()));
envs.put("MESOS_MASTER", getFormattedZKAddress());
envs.put("MESOS_SWITCH_USER", "false");
envs.put("MESOS_LOGGING_LEVEL", getLoggingLevel());
envs.put("MESOS_WORK_DIR", MESOS_AGENT_SANDBOX_DIR);
envs.put("SERVICE_IGNORE", "1");
return envs;
}
Expand Down
1 change: 0 additions & 1 deletion minimesos/src/main/java/com/containersol/minimesos/mesos/MesosContainerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ protected final Map<String, String> getSharedEnvVars() {
envs.put("MESOS_ISOLATOR", "cgroups/cpu,cgroups/mem");
envs.put("MESOS_LOG_DIR", "/var/log");
envs.put("MESOS_LOGGING_LEVEL", getLoggingLevel());
envs.put("MESOS_WORK_DIR", "/tmp/mesos");
return envs;
}

Expand Down

0 comments on commit 053cc55

Please sign in to comment.