diff --git a/.gitignore b/.gitignore index e5471859..40b89258 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,10 @@ -# Compiled python modules. *.pyc - -# Setuptools distribution folder. /build/ /dist/ - -# Python egg metadata, regenerated from source files by setuptools. +/.conda/ +/.jupyter/ +/.local/ +/node_modules/ /*.egg-info /*.egg /*.eggs diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4d82da86 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +LABEL maintainer="RStudio Connect " + +ARG NB_UID +ARG NB_GID +ARG PY_VERSION +RUN apt-get update -qq \ + && apt-get install -y make +RUN getent group ${NB_GID} || groupadd -g ${NB_GID} builder +RUN useradd --password password \ + --create-home \ + --home-dir /home/builder \ + --uid ${NB_UID} \ + --gid ${NB_GID} \ + builder + +USER ${NB_UID}:${NB_GID} +RUN cd /home/builder \ + && conda create --yes --name py${PY_VERSION} jupyter diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..b56bec66 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,120 @@ +#!groovy + +def gitClean() { + // inspired by: https://issues.jenkins-ci.org/browse/JENKINS-31924 + // https://issues.jenkins-ci.org/browse/JENKINS-32540 + // The sequence of reset --hard and clean -fdx first + // in the root and then using submodule foreach + // is based on how the Jenkins Git SCM clean before checkout + // feature works. + sh 'git reset --hard' + sh 'git clean -ffdx' +} + +// Build the name:tag for a docker image where the tag is the checksum +// computed from a specified file. +def imageName(name, filenames) { + // If this is extended to support multiple files, be wary of: + // https://issues.jenkins-ci.org/browse/JENKINS-26481 + // closures don't really work. + + // Suck in the contents of the file and then hash the result. + def contents = ""; + for (int i=0; i" + + // Looking up the author also demands being in a `node`. + gitAuthor = sh(returnStdout: true, script: 'git --no-pager show -s --format="%aN" HEAD').trim() + + def dockerImage + stage('prepare environment') { + dockerImage = pullBuildPush( + image_name: 'jenkins/rsconnect-jupyter', + image_tag: 'python3', + build_arg_nb_uid: 'JENKINS_UID', + build_arg_nb_gid: 'JENKINS_GID', + build_args: build_args(), + push: !isUserBranch + ) + } + + dockerImage.inside() { + stage('package') { + print "building python wheel package" + sh 'make dist' + archiveArtifacts artifacts: 'dist/*.whl' + } + } + } + } + + // Slack message includes username information. + message = "${messagePrefix} by ${gitAuthor} passed" + slackSend channel: slackChannelPass, color: 'good', message: message +} catch(err) { + // Slack message includes username information. When master/release fails, + // CC the whole connect team. + slackNameFail = "unknown" + if (!isUserBranch) { + slackNameFail = "${gitAuthor} (cc @kenny)" + } + + message = "${messagePrefix} by ${slackNameFail} failed: ${err}" + slackSend channel: slackChannelFail, color: 'bad', message: message + throw err +} diff --git a/Makefile b/Makefile index 3c34c06b..bca357b5 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,81 @@ -.PHONY: test dist develop-setup develop +.PHONY: clean pull images image2 image3 launch notebook2 notebook3 package dist test dist run +NB_UID=$(shell id -u) +NB_GID=$(shell id -g) + +PY2=rstudio/rsconnect-jupyter-py2 +PY3=rstudio/rsconnect-jupyter-py3 + +clean: + rm -rf build/ dist/ rsconnect.egg-info/ + +pull: +# docker pull $(PY3) +# docker pull $(PY2) +# docker pull python:2.7 +# docker pull python:3.6 + docker pull node:6-slim + +images: image2 image3 + +image2: + docker build \ + --tag $(PY2) \ + --file Dockerfile \ + --build-arg BASE_IMAGE=continuumio/miniconda:4.4.10 \ + --build-arg NB_UID=$(NB_UID) \ + --build-arg NB_GID=$(NB_GID) \ + --build-arg PY_VERSION=2 \ + . + +image3: + docker build \ + --tag $(PY3) \ + --file Dockerfile \ + --build-arg BASE_IMAGE=continuumio/miniconda3:4.4.10 \ + --build-arg NB_UID=$(NB_UID) \ + --build-arg NB_GID=$(NB_GID) \ + --build-arg PY_VERSION=3 \ + . + +launch: + docker run --rm -i -t \ + -v $(CURDIR)/notebooks$(PY_VERSION):/notebooks \ + -v $(CURDIR):/rsconnect \ + -e NB_UID=$(NB_UID) \ + -e NB_GID=$(NB_GID) \ + -e PY_VERSION=$(PY_VERSION) \ + -p :9999:9999 \ + $(DOCKER_IMAGE) \ + /rsconnect/run.sh $(TARGET) + + +notebook2: + make DOCKER_IMAGE=$(PY2) PY_VERSION=2 TARGET=run launch + +notebook3: + make DOCKER_IMAGE=$(PY3) PY_VERSION=3 TARGET=run launch test: +# TODO run in container python setup.py test dist: -# build egg - python setup.py sdist -# build wheel - # wheels don't get built if _any_ file it tries to touch has a timestamp < 1980 # (system files) so use the current timestamp as a point of reference instead SOURCE_DATE_EPOCH="$(shell date +%s)"; python setup.py bdist_wheel -develop-setup: - python setup.py develop +package: + make DOCKER_IMAGE=$(PY3) PY_VERSION=3 TARGET=dist launch -develop: +run: +# link python package + python setup.py develop +# install rsconnect as a jupyter extension jupyter-nbextension install --symlink --user --py rsconnect +# enable js extension jupyter-nbextension enable --py rsconnect +# enable python extension jupyter-serverextension enable --py rsconnect +# start notebook + jupyter-notebook -y --notebook-dir=/notebooks --ip='*' --port=9999 --no-browser diff --git a/README.md b/README.md index 1333ed77..233c18ab 100644 --- a/README.md +++ b/README.md @@ -1 +1,30 @@ -TODO +# Development + +Need to run this after checkout and when modifying the docker images + + make images + +Launch jupyter in a python 2 environment + + make notebook2 + +Launch jupyter in a python 3 environment + + make notebook3 + +> Note: notebooks in the `notebooks2` and `notebooks3` directories will be +> available in respective python environments. + +## Seeing code changes + +When modifying JavaScript files simply refresh the browser window to see +changes. + +When modifying Python files restart the jupyter process to see changes. + +# Packaging + +The following will create a universal [wheel](https://pythonwheels.com/) ready +to be installed in any python 2 or python 3 environment. + + make package diff --git a/notebooks2/.keep b/notebooks2/.keep new file mode 100644 index 00000000..e69de29b diff --git a/notebooks3/.keep b/notebooks3/.keep new file mode 100644 index 00000000..e69de29b diff --git a/run.sh b/run.sh new file mode 100755 index 00000000..a1833f74 --- /dev/null +++ b/run.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +if [[ $(id -u) == "0" ]]; +then + su -c "$0 $@" "$NB_USER" + exit $? +fi + +cd /rsconnect +export PATH=/opt/conda/bin:$PATH +source activate "py${PY_VERSION}" +make -f /rsconnect/Makefile $1