Build python package in docker

.gitignore

@@ -1,11 +1,10 @@
-# Compiled python modules.
 *.pyc
-
-# Setuptools distribution folder.
 /build/
 /dist/
-
-# Python egg metadata, regenerated from source files by setuptools.
+/.conda/
+/.jupyter/
+/.local/
+/node_modules/
 /*.egg-info
 /*.egg
 /*.eggs

Dockerfile

@@ -0,0 +1,20 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+LABEL maintainer="RStudio Connect <[email protected]>"
+
+ARG NB_UID
+ARG NB_GID
+ARG PY_VERSION
+RUN apt-get update -qq \
+    && apt-get install -y make
+RUN getent group ${NB_GID} || groupadd -g ${NB_GID} builder
+RUN useradd --password password \
+    --create-home \
+    --home-dir /home/builder \
+    --uid ${NB_UID} \
+    --gid ${NB_GID} \
+    builder
+
+USER ${NB_UID}:${NB_GID}
+RUN cd /home/builder \
+    && conda create --yes --name py${PY_VERSION} jupyter

Jenkinsfile

@@ -0,0 +1,120 @@
+#!groovy
+
+def gitClean() {
+  // inspired by: https://issues.jenkins-ci.org/browse/JENKINS-31924
+  // https://issues.jenkins-ci.org/browse/JENKINS-32540
+  // The sequence of reset --hard and clean -fdx first
+  // in the root and then using submodule foreach
+  // is based on how the Jenkins Git SCM clean before checkout
+  // feature works.
+  sh 'git reset --hard'
+  sh 'git clean -ffdx'
+}
+
+// Build the name:tag for a docker image where the tag is the checksum
+// computed from a specified file.
+def imageName(name, filenames) {
+  // If this is extended to support multiple files, be wary of:
+  // https://issues.jenkins-ci.org/browse/JENKINS-26481
+  // closures don't really work.
+
+  // Suck in the contents of the file and then hash the result.
+  def contents = "";
+  for (int i=0; i<filenames.size(); i++) {
+    print "reading ${filenames[i]}"
+    def content = readFile(filenames[i])
+    print "read ${filenames[i]}"
+    contents = contents + content
+  }
+
+  print "hashing ${name}"
+  def tag = java.security.MessageDigest.getInstance("MD5").digest(contents.bytes).encodeHex().toString()
+  print "hashed ${name}"
+  def result = "${name}:${tag}"
+  print "computed image name ${result}"
+  return result
+}
+
+isUserBranch = true
+if (env.BRANCH_NAME == 'master') {
+  isUserBranch = false
+} else if (env.BRANCH_NAME ==~ /^\d+\.\d+\.\d+$/) {
+  isUserBranch = false
+}
+
+messagePrefix = "<${env.JOB_URL}|rsconnect-jupyter pipeline> build <${env.BUILD_URL}|${env.BUILD_DISPLAY_NAME}>"
+
+slackChannelPass = "#rsconnect-bots"
+slackChannelFail = "#rsconnect"
+if (isUserBranch) {
+  slackChannelFail = "#rsconnect-bots"
+}
+
+nodename = 'docker'
+if (isUserBranch) {
+  // poor man's throttling for user branches.
+  nodename = 'connect-branches'
+}
+
+def build_args() {
+  def uid = sh (script: 'id -u jenkins', returnStdout: true).trim()
+  def gid = sh (script: 'id -g jenkins', returnStdout: true).trim()
+  def image = 'continuumio/miniconda3:4.4.10'
+  return " --build-arg PY_VERSION=3 --build-arg BASE_IMAGE=${image} --build-arg NB_UID=${uid} --build-arg NB_GID=${gid} "
+}
+
+try {
+  node(nodename) {
+    timestamps {
+      checkout scm
+      gitClean()
+
+      // If we want to link to the commit, we need to drop down to shell. This
+      // means that we need to be inside a `node` and after checking-out code.
+      // https://issues.jenkins-ci.org/browse/JENKINS-26100 suggests this workaround.
+      gitSHA = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+      shortSHA = gitSHA.take(6)
+
+      // Update our Slack message metadata with commit info once we can obtain it.
+      messagePrefix = messagePrefix + " of <https://github.com/rstudio/rsconnect-jupyter/commit/${gitSHA}|${shortSHA}>"
+
+      // Looking up the author also demands being in a `node`.
+      gitAuthor = sh(returnStdout: true, script: 'git --no-pager show -s --format="%aN" HEAD').trim()
+
+      def dockerImage
+      stage('prepare environment') {
+        dockerImage = pullBuildPush(
+          image_name: 'jenkins/rsconnect-jupyter',
+          image_tag: 'python3',
+          build_arg_nb_uid: 'JENKINS_UID',
+          build_arg_nb_gid: 'JENKINS_GID',
+          build_args: build_args(),
+          push: !isUserBranch
+        )
+      }
+
+      dockerImage.inside() {
+        stage('package') {
+          print "building python wheel package"
+          sh 'make dist'
+          archiveArtifacts artifacts: 'dist/*.whl'
+        }
+      }
+    }
+  }
+
+  // Slack message includes username information.
+  message = "${messagePrefix} by ${gitAuthor} passed"
+  slackSend channel: slackChannelPass, color: 'good', message: message
+} catch(err) {
+  // Slack message includes username information. When master/release fails,
+  // CC the whole connect team.
+  slackNameFail = "unknown"
+  if (!isUserBranch) {
+    slackNameFail = "${gitAuthor} (cc @kenny)"
+  }
+
+  message = "${messagePrefix} by ${slackNameFail} failed: ${err}"
+  slackSend channel: slackChannelFail, color: 'bad', message: message
+  throw err
+}

Makefile

@@ -1,22 +1,81 @@
-.PHONY: test dist develop-setup develop
+.PHONY: clean pull images image2 image3 launch notebook2 notebook3 package dist test dist run
 
+NB_UID=$(shell id -u)
+NB_GID=$(shell id -g)
+
+PY2=rstudio/rsconnect-jupyter-py2
+PY3=rstudio/rsconnect-jupyter-py3
+
+clean:
+	rm -rf build/ dist/ rsconnect.egg-info/
+
+pull:
+#	docker pull $(PY3)
+#	docker pull $(PY2)
+#	docker pull python:2.7
+#	docker pull python:3.6
+	docker pull node:6-slim
+
+images: image2 image3
+
+image2:
+	docker build \
+		--tag $(PY2) \
+		--file Dockerfile \
+		--build-arg BASE_IMAGE=continuumio/miniconda:4.4.10 \
+		--build-arg NB_UID=$(NB_UID) \
+		--build-arg NB_GID=$(NB_GID) \
+		--build-arg PY_VERSION=2 \
+		.
+
+image3:
+	docker build \
+		--tag $(PY3) \
+		--file Dockerfile \
+		--build-arg BASE_IMAGE=continuumio/miniconda3:4.4.10 \
+		--build-arg NB_UID=$(NB_UID) \
+		--build-arg NB_GID=$(NB_GID) \
+		--build-arg PY_VERSION=3 \
+		.
+
+launch:
+	docker run --rm -i -t \
+		-v $(CURDIR)/notebooks$(PY_VERSION):/notebooks \
+		-v $(CURDIR):/rsconnect \
+		-e NB_UID=$(NB_UID) \
+		-e NB_GID=$(NB_GID) \
+		-e PY_VERSION=$(PY_VERSION) \
+		-p :9999:9999 \
+		$(DOCKER_IMAGE) \
+		/rsconnect/run.sh $(TARGET)
+
+
+notebook2:
+	make DOCKER_IMAGE=$(PY2) PY_VERSION=2 TARGET=run launch
+
+notebook3:
+	make DOCKER_IMAGE=$(PY3) PY_VERSION=3 TARGET=run launch
 
 test:
+# TODO run in container
 	python setup.py test
 
 dist:
-# build egg
-	python setup.py sdist
-# build wheel
-
 # wheels don't get built if _any_ file it tries to touch has a timestamp < 1980
 # (system files) so use the current timestamp as a point of reference instead
 	SOURCE_DATE_EPOCH="$(shell date +%s)"; python setup.py bdist_wheel
 
-develop-setup:
-	python setup.py develop
+package:
+	make DOCKER_IMAGE=$(PY3) PY_VERSION=3 TARGET=dist launch
 
-develop:
+run:
+# link python package
+	python setup.py develop
+# install rsconnect as a jupyter extension
 	jupyter-nbextension install --symlink --user --py rsconnect
+# enable js extension
 	jupyter-nbextension enable --py rsconnect
+# enable python extension
 	jupyter-serverextension enable --py rsconnect
+# start notebook
+	jupyter-notebook -y --notebook-dir=/notebooks --ip='*' --port=9999 --no-browser

README.md

@@ -1 +1,30 @@
-TODO
+# Development
+
+Need to run this after checkout and when modifying the docker images
+
+    make images
+
+Launch jupyter in a python 2 environment
+
+    make notebook2
+
+Launch jupyter in a python 3 environment
+
+    make notebook3
+
+> Note: notebooks in the `notebooks2` and `notebooks3` directories will be
+> available in respective python environments.
+
+## Seeing code changes
+
+When modifying JavaScript files simply refresh the browser window to see
+changes.
+
+When modifying Python files restart the jupyter process to see changes.
+
+# Packaging
+
+The following will create a universal [wheel](https://pythonwheels.com/) ready
+to be installed in any python 2 or python 3 environment.
+
+    make package

run.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+if [[ $(id -u) == "0" ]];
+then
+    su -c "$0 $@" "$NB_USER"
+    exit $?
+fi
+
+cd /rsconnect
+export PATH=/opt/conda/bin:$PATH
+source activate "py${PY_VERSION}"
+make -f /rsconnect/Makefile $1