full sample cluster dockerfile

Author: xvrl

Diff for: Dockerfile

@@ -36,30 +36,51 @@ RUN mkdir -p /var/lib/druid
 RUN chown druid:druid /var/lib/druid
 
 # Druid (release tarball)
-RUN wget -q -O - http://static.druid.io/artifacts/releases/druid-services-0.6.121-bin.tar.gz | tar -xzf - -C /usr/local
-RUN ln -s /usr/local/druid-services-0.6.121 /usr/local/druid
+#ENV DRUID_VERSION 0.7.0
+#RUN wget -q -O - http://static.druid.io/artifacts/releases/druid-services-$DRUID_VERSION-bin.tar.gz | tar -xzf - -C /usr/local
+#RUN ln -s /usr/local/druid-services-0.6.121 /usr/local/druid
 
 # Druid (from source)
-#ENV DRUID_VERSION druid-0.6.121
-#RUN git clone -q --branch $DRUID_VERSION --depth 1 https://github.com/metamx/druid.git /tmp/druid
-#WORKDIR /tmp/druid
-#RUN hash=$(git rev-parse --short HEAD); mkdir -p /usr/local/druid-$hash/lib && ln -s /usr/local/druid-$hash /usr/local/druid
-#RUN mvn package -DskipTests=true
-#RUN cp services/target/druid-services-*-selfcontained.jar /usr/local/druid/lib
+ENV DRUID_VERSION druid-0.7.x
+# trigger rebuild only if branch changed
+ADD https://api.github.com/repos/metamx/druid/git/refs/heads/$DRUID_VERSION druid-version.json
+RUN git clone -q --branch $DRUID_VERSION --depth 1 https://github.com/metamx/druid.git /tmp/druid
+RUN git config --global user.email [email protected]
+WORKDIR /tmp/druid
+RUN hash=$(git rev-parse --short HEAD); mkdir -p /usr/local/druid-$hash/lib && ln -s /usr/local/druid-$hash /usr/local/druid
+# package and install Druid locally
+RUN mvn -B release:prepare -DpushChanges=false -DpreparationGoals=clean -DreleaseVersion=$DRUID_VERSION -DdevelopmentVersion=$DRUID_VERSION-SNAPSHOT release:perform -DskipTests=true -DlocalCheckout=true -Dgoals=install
+RUN cp -f target/checkout/services/target/druid-services-$DRUID_VERSION-selfcontained.jar /usr/local/druid/lib
+# pull dependencies for Druid extensions
+RUN mkdir -p /usr/local/druid/repository
+RUN java -Ddruid.extensions.coordinates=[\"io.druid.extensions:druid-s3-extensions:$DRUID_VERSION\"] -Ddruid.extensions.localRepository=/usr/local/druid/repository -Ddruid.extensions.remoteRepositories=[\"file:///root/.m2/repository/\",\"http://repo1.maven.org/maven2/\",\"https://metamx.artifactoryonline.com/metamx/pub-libs-releases-local\"] -cp /usr/local/druid/lib/* io.druid.cli.Main tools pull-deps
+
+WORKDIR /
 
 # Setup metadata store
 RUN /etc/init.d/mysql start && echo "GRANT ALL ON druid.* TO 'druid'@'localhost' IDENTIFIED BY 'diurd'; CREATE database druid;" | mysql -u root && /etc/init.d/mysql stop
 
+# Add sample data
+RUN /etc/init.d/mysql start && java -cp /usr/local/druid/lib/druid-services-*-selfcontained.jar io.druid.cli.Main tools metadata-init --connectURI="jdbc:mysql://localhost:3306/druid" --user=druid --password=diurd && /etc/init.d/mysql stop
+ADD sample-data.sql sample-data.sql
+RUN /etc/init.d/mysql start && echo "Adding sample data" && (cat sample-data.sql | mysql -u root druid) && echo "Stopping" && /etc/init.d/mysql stop
+
+# Setup supervisord
 ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+RUN perl -pi -e "s/\\\$DRUID_VERSION/$DRUID_VERSION/" /etc/supervisor/conf.d/supervisord.conf
 
 # Clean up
 RUN apt-get clean && rm -rf /tmp/* /var/tmp/*
 
 # Expose ports:
-# - 8080: HTTP
+# - 8081: HTTP (coordinator)
+# - 8082: HTTP (broker)
+# - 8083: HTTP (historical)
 # - 3306: MySQL
 # - 2181 2888 3888: ZooKeeper
-EXPOSE 8080
+EXPOSE 8081
+EXPOSE 8082
+EXPOSE 8083
 EXPOSE 3306
 EXPOSE 2181 2888 3888
 

Diff for: Tutorial.md

@@ -42,13 +42,17 @@ docker info
 Build Druid image
 
 ```sh
-docker build -t druid/coordinator .
+docker build -t druid/cluster .
 ```
 
 ## Run Druid
 
 ```sh
-docker run -i --rm -p 3000:8080 -t druid/coordinator
+docker run --rm -i -p 3000:8082 -p 3001:8081 druid/cluster
 ```
 
-Assuming `boot2docker ip` returns `192.168.59.103`, you should be able to access the coordinator console at: http://192.168.59.103:3000/
+Wait a minute or so for the Druid to download the sample data an start up.
+
+Assuming `boot2docker ip` returns `192.168.59.103`, you should be able to
+   - access the coordinator console at: http://192.168.59.103:3001/
+   - list data-sources on the broker http://192.168.59.103:3000/druid/v2/datasources

Diff for: sample-data.sql

@@ -0,0 +1,2 @@
+INSERT INTO druid_segments (id, dataSource, created_date, start, end, partitioned, version, used, payload) VALUES ('wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', 'wikipedia', '2013-08-08T21:26:23.799Z', '2013-08-01T00:00:00.000Z', '2013-08-02T00:00:00.000Z', '0', '2013-08-08T21:22:48.989Z', '1', '{\"dataSource\":\"wikipedia\",\"interval\":\"2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z\",\"version\":\"2013-08-08T21:22:48.989Z\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip\"},\"dimensions\":\"dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace,anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup\",\"metrics\":\"count,delta,variation,added,deleted\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":24664730,\"identifier\":\"wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z\"}');
+

Diff for: supervisord.conf

@@ -16,9 +16,6 @@ command=java
   -Xmx1g
   -Duser.timezone=UTC
   -Dfile.encoding=UTF-8
-  -Ddruid.host=localhost
-  -Ddruid.port=8080
-  -Ddruid.service=druid/coordinator
   -Ddruid.db.connector.connectURI=jdbc:mysql://localhost:3306/druid
   -Ddruid.db.connector.user=druid
   -Ddruid.db.connector.password=diurd
@@ -27,6 +24,48 @@ command=java
   -Ddruid.discovery.curator.path=/discovery
   -Ddruid.curator.compress=true
   -Ddruid.announcer.type=batch
+  -Ddruid.coordinator.startDelay=PT5S
   -cp /usr/local/druid/lib/*
   io.druid.cli.Main server coordinator
 redirect_stderr=true
+
+[program:druid-historical]
+user=druid
+command=java
+  -server
+  -Xmx1g
+  -Duser.timezone=UTC
+  -Dfile.encoding=UTF-8
+  -Ddruid.zk.service.host=localhost
+  -Ddruid.zk.paths.base=/druid
+  -Ddruid.discovery.curator.path=/discovery
+  -Ddruid.curator.compress=true
+  -Ddruid.announcer.type=batch
+  -Ddruid.extensions.coordinates=[\"io.druid.extensions:druid-s3-extensions:$DRUID_VERSION\"]
+  -Ddruid.extensions.localRepository=/usr/local/druid/repository
+  -Ddruid.s3.accessKey=AKIAIMKECRUYKDQGR6YQ
+  -Ddruid.s3.secretKey=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
+  -Ddruid.computation.buffer.size=67108864
+  -Ddruid.segmentCache.locations="[{\"path\":\"/var/tmp/druid/indexCache\",\"maxSize\":5000000000}]"
+  -Ddruid.server.maxSize=5000000000
+  -cp /usr/local/druid/lib/*
+  io.druid.cli.Main server historical
+redirect_stderr=true
+
+[program:druid-broker]
+user=druid
+command=java
+  -server
+  -Xmx1g
+  -Duser.timezone=UTC
+  -Dfile.encoding=UTF-8
+  -Ddruid.zk.service.host=localhost
+  -Ddruid.zk.paths.base=/druid
+  -Ddruid.discovery.curator.path=/discovery
+  -Ddruid.curator.compress=true
+  -Ddruid.announcer.type=batch
+  -Ddruid.computation.buffer.size=67108864
+  -Ddruid.broker.cache.sizeInBytes=33554432
+  -cp /usr/local/druid/lib/*
+  io.druid.cli.Main server broker
+redirect_stderr=true