Merge pull request #48 from ContainerSolutions/bug/47-inner-docker-proxy

Setup inner docker proxy on Mac

src/main/java/org/apache/mesos/mini/MesosCluster.java

@@ -1,5 +1,6 @@
 package org.apache.mesos.mini;
 
+import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.DockerException;
 import com.github.dockerjava.api.InternalServerErrorException;
 import com.mashape.unirest.http.Unirest;
@@ -188,4 +189,8 @@ private void writeLog(String containerName, String containerId) {
     public List<AbstractContainer> getContainers() {
         return Collections.unmodifiableList(containers);
     }
+
+    public DockerClient getInnerDockerClient() {
+        return this.mesosContainer.getInnerDockerClient();
+    }
 }

src/main/java/org/apache/mesos/mini/mesos/InnerDockerProxy.java

@@ -0,0 +1,41 @@
+package org.apache.mesos.mini.mesos;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateContainerCmd;
+import com.github.dockerjava.api.model.ExposedPort;
+import com.github.dockerjava.api.model.Link;
+import com.github.dockerjava.api.model.PortBinding;
+import org.apache.mesos.mini.container.AbstractContainer;
+
+public class InnerDockerProxy extends AbstractContainer {
+
+  private static final String DOCKER_IMAGE = "mwldk/go-tcp-proxy";
+
+  private static final int PROXY_PORT = 2377;
+
+  private MesosContainer mesosContainer;
+
+  public InnerDockerProxy(DockerClient dockerClient, MesosContainer mesosContainer) {
+    super(dockerClient);
+    this.mesosContainer = mesosContainer;
+  }
+
+  @Override
+  protected void pullImage() {
+    pullImage(DOCKER_IMAGE, "latest");
+  }
+
+  @Override
+  protected CreateContainerCmd dockerCommand() {
+    return dockerClient
+        .createContainerCmd(DOCKER_IMAGE)
+        .withLinks(Link.parse(mesosContainer.getContainerId() + ":docker"))
+        .withExposedPorts(ExposedPort.tcp(mesosContainer.getDockerPort()))
+        .withPortBindings(PortBinding.parse("0.0.0.0:" + getProxyPort() + ":" + mesosContainer.getDockerPort()))
+        .withCmd("-l=:" + mesosContainer.getDockerPort(), "-r=docker:" + mesosContainer.getDockerPort());
+  }
+
+  public int getProxyPort() {
+    return PROXY_PORT;
+  }
+}

src/main/java/org/apache/mesos/mini/mesos/MesosClusterConfig.java

@@ -89,7 +89,7 @@ public Builder defaultDockerClient() {
             if (config.getUri().getScheme().startsWith("http") && !System.getProperty("os.name").equals("Linux")) {
                 HttpHost proxy = new HttpHost(config.getUri().getHost(), this.proxyPort);
                 Unirest.setProxy(proxy);
-                Unirest.setTimeouts(5000L, 5000L);
+                Unirest.setTimeouts(20_000L, 20_000L);
             }
             this.dockerClient = DockerClientBuilder.getInstance(config).build();
             return this;

src/main/java/org/apache/mesos/mini/mesos/MesosContainer.java

@@ -2,38 +2,63 @@
 
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.CreateContainerCmd;
-import com.github.dockerjava.api.model.Bind;
-import com.github.dockerjava.api.model.Container;
-import com.github.dockerjava.api.model.ExposedPort;
-import com.github.dockerjava.api.model.Link;
-import com.github.dockerjava.api.model.Volume;
+import com.github.dockerjava.api.model.*;
 import com.github.dockerjava.core.DockerClientBuilder;
 import com.github.dockerjava.core.DockerClientConfig;
-import com.mashape.unirest.http.Unirest;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpHost;
 import org.apache.log4j.Logger;
 import org.apache.mesos.mini.container.AbstractContainer;
 
+import java.io.IOException;
+import java.net.Socket;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import static com.jayway.awaitility.Awaitility.await;
 
 public class MesosContainer extends AbstractContainer {
 
+    private static final int DOCKER_PORT = 2376;
+
     private static Logger LOGGER = Logger.getLogger(MesosContainer.class);
 
     private static final String MESOS_LOCAL_IMAGE = "containersol/mesos-local";
-    public static final String REGISTRY_TAG = "0.2.1";
+    public static final String REGISTRY_TAG = "14";
+
     private final MesosClusterConfig clusterConfig;
     private final String registryContainerId;
+    private DockerClient innerDockerClient;
+    private InnerDockerProxy innerDockerProxy;
 
     public MesosContainer(DockerClient dockerClient, MesosClusterConfig clusterConfig, String registryContainerId) {
         super(dockerClient);
         this.clusterConfig = clusterConfig;
         this.registryContainerId = registryContainerId;
     }
 
+    @Override
+    public void start() {
+        super.start();
+
+        await().atMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS).until(new DockerSocketIsAvailable<Boolean>(this));
+
+        String os = System.getProperty("os.name");
+        DockerClientConfig.DockerClientConfigBuilder innerDockerConfigBuilder;
+        if (!os.equals("Linux")) {
+            LOGGER.info("Mini-Mesos runs on '" + os + "'. Starting inner Docker Proxy");
+            innerDockerProxy = new InnerDockerProxy(clusterConfig.dockerClient, this);
+            innerDockerProxy.start();
+            innerDockerConfigBuilder = DockerClientConfig.createDefaultConfigBuilder();
+            innerDockerConfigBuilder.withUri("http://" + innerDockerConfigBuilder.build().getUri().getHost() + ":" + innerDockerProxy.getProxyPort());
+        } else {
+            innerDockerConfigBuilder = DockerClientConfig.createDefaultConfigBuilder();
+            innerDockerConfigBuilder.withUri("http://" + getIpAddress() + ":" + getDockerPort());
+        }
+        this.innerDockerClient = DockerClientBuilder.getInstance(innerDockerConfigBuilder.build()).build();
+    }
+
     String[] createMesosLocalEnvironment() {
         ArrayList<String> envs = new ArrayList<String>();
         envs.add("NUMBER_OF_SLAVES=" + clusterConfig.numberOfSlaves);
@@ -71,25 +96,52 @@ protected CreateContainerCmd dockerCommand() {
                 .withPrivileged(true)
                         // the registry container will be known as 'private-registry' to mesos-local
                 .withLinks(Link.parse(registryContainerId + ":private-registry"))
-                .withExposedPorts(new ExposedPort(2376))
+                .withExposedPorts(new ExposedPort(getDockerPort()))
                 .withEnv(createMesosLocalEnvironment())
                 .withVolumes(new Volume("/sys/fs/cgroup"))
                 .withBinds(Bind.parse("/sys/fs/cgroup:/sys/fs/cgroup:rw"));
     }
 
+    public int getDockerPort() {
+        return DOCKER_PORT;
+    }
+
     @Override
     public void remove() {
-        DockerClientConfig.DockerClientConfigBuilder innerDockerConfigBuilder = DockerClientConfig.createDefaultConfigBuilder();
-        innerDockerConfigBuilder.withUri("http://" + getIpAddress() + ":2376");
-        DockerClient innerDockerClient = DockerClientBuilder.getInstance(innerDockerConfigBuilder.build()).build();
-
         List<Container> innerContainers = innerDockerClient.listContainersCmd().exec();
         for (Container innerContainer : innerContainers) {
             LOGGER.info("Removing Mesos-Local inner container including volumes: " + innerContainer.getNames()[0]);
-            innerDockerClient.removeContainerCmd(innerContainer.getId());
+            innerDockerClient.removeContainerCmd(innerContainer.getId()).withForce().withRemoveVolumes(true).exec();
+        }
+
+        if (innerDockerProxy != null) {
+            innerDockerProxy.remove();
         }
 
         LOGGER.info("Removing Mesos-Local container");
         super.remove();
     }
-}
+
+    public DockerClient getInnerDockerClient() {
+        return innerDockerClient;
+    }
+
+    class DockerSocketIsAvailable<T> implements Callable<Boolean> {
+
+        private MesosContainer container;
+
+        public DockerSocketIsAvailable(MesosContainer container) {
+            this.container = container;
+        }
+
+        @Override
+        public Boolean call() throws Exception {
+            try (Socket ignored = new Socket(container.getIpAddress(), container.getDockerPort())) {
+                return true;
+            } catch (IOException ignored) {
+                return false;
+            }
+        }
+    }
+
+}

src/test/java/org/apache/mesos/mini/InnerDockerTest.java

@@ -0,0 +1,64 @@
+package org.apache.mesos.mini;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateContainerCmd;
+import com.github.dockerjava.api.model.Container;
+import org.apache.mesos.mini.container.AbstractContainer;
+import org.apache.mesos.mini.docker.PrivateDockerRegistry;
+import org.apache.mesos.mini.mesos.MesosClusterConfig;
+import org.apache.mesos.mini.mesos.MesosContainer;
+import org.junit.Test;
+
+import java.security.SecureRandom;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test inner docker client.
+ * Should instantiate container INSIDE the daemon of the MesosContainer (i.e. not directly visible)
+ * The only way to do this on a mac is via a proxy.
+ */
+public class InnerDockerTest {
+    @Test
+    public void shouldStart() throws InterruptedException {
+        MesosClusterConfig config = MesosClusterConfig.builder().defaultDockerClient()
+                .slaveResources(new String[]{"ports(*):[9200-9200,9300-9300]", "ports(*):[9201-9201,9301-9301]", "ports(*):[9202-9202,9302-9302]"})
+                .build();
+        PrivateDockerRegistry registry = new PrivateDockerRegistry(config.dockerClient, config);
+        registry.start();
+        MesosContainer mesosContainer = new MesosContainer(config.dockerClient, config, registry.getContainerId());
+        mesosContainer.start();
+
+        HelloWorldNoRemoveContainer helloWorldContainer = new HelloWorldNoRemoveContainer(mesosContainer.getInnerDockerClient());
+        helloWorldContainer.start();
+
+        List<Container> containers = mesosContainer.getInnerDockerClient().listContainersCmd().exec();
+        assertEquals(1, containers.size());
+    }
+
+    class HelloWorldNoRemoveContainer extends AbstractContainer {
+
+        public static final String HELLO_WORLD_IMAGE = "tutum/hello-world";
+
+        protected HelloWorldNoRemoveContainer(DockerClient dockerClient) {
+            super(dockerClient);
+        }
+
+        // Override remove method to do nothing. We are force removing in the mesosContainer.
+        @Override
+        public void remove() {
+        }
+
+        @Override
+        protected void pullImage() {
+            pullImage(HELLO_WORLD_IMAGE, "latest");
+        }
+
+        @Override
+        protected CreateContainerCmd dockerCommand() {
+            return dockerClient.createContainerCmd(HELLO_WORLD_IMAGE).withName("hello-world_" + new SecureRandom().nextInt());
+        }
+
+    }
+}