diff --git a/.github/workflows/jupyter.base.yml b/.github/workflows/jupyter.base.yml new file mode 100644 index 0000000..25130ce --- /dev/null +++ b/.github/workflows/jupyter.base.yml @@ -0,0 +1,82 @@ +name: Build and push + +on: + workflow_dispatch: + inputs: + py_version: + type: string + required: true + description: "Python version (default = 3.11)" + + push: + branches: + - master + paths: + - "base/**" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/jupyter + +jobs: + build-and-push: + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Setup environment + run: | + echo "PY_VERSION=${PY_VERSION:-3.11}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v3 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@d6a3abf1bdea83574e28d40543793018b6035605 + with: + cosign-release: "v1.7.1" + + # Workaround: https://github.com/docker/build-push-action/issues/461 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + file: Dockerfile + context: base + push: ${{ github.event_name != 'pull_request' }} + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:base-py${{ env.PY_VERSION }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + PY_VERSION=${{ env.PY_VERSION }} diff --git a/base/.dockerignore b/base/.dockerignore new file mode 100644 index 0000000..e69de29 diff --git a/base/Dockerfile b/base/Dockerfile new file mode 100644 index 0000000..d4992ee --- /dev/null +++ b/base/Dockerfile @@ -0,0 +1,84 @@ +ARG PY_VERSION=3.11 +FROM mcr.microsoft.com/devcontainers/python:${PY_VERSION} + +LABEL maintainer="Krzysztof Begiedza " + +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Configure environment variables +ENV SHELL=/bin/bash \ + NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV HOME="/home/${NB_USER}" +ENV DEBIAN_FRONTEND noninteract +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 + +# Rename vscode user to ${NB_USER} (jovyan) +RUN usermod -l ${NB_USER} vscode \ + && usermod -d /home/${NB_USER} -m ${NB_USER} \ + && chown -R ${NB_USER}:${NB_GID} /home/${NB_USER} + +RUN apt-get update --yes && \ + apt-get upgrade --yes && \ + apt-get install --yes --no-install-recommends \ + bzip2 \ + ca-certificates \ + locales \ + sudo \ + tini \ + wget && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen + +# Copy a script to fix permissions +COPY fix-permissions.sh /usr/local/bin/fix-permissions.sh +RUN chmod a+rx /usr/local/bin/fix-permissions.sh + +RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ + sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ + sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ + chmod g+w /etc/passwd && \ + fix-permissions.sh "${HOME}" + +# Install jupyter +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install \ + notebook \ + jupyterhub \ + jupyterlab && \ + jupyter notebook --generate-config && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions.sh "/home/${NB_USER}" + +ENV JUPYTER_PORT=8888 +EXPOSE $JUPYTER_PORT + +ENTRYPOINT [ "tini", "-g", "--" ] + +CMD [ "start-notebook-frozen.sh" ] + +COPY start.sh start-notebook-frozen.sh /usr/local/bin/ +COPY jupyter_server_config.py /etc/jupyter/ + +# Legacy for Jupyter Notebook Server, see: [#1205](https://github.com/jupyter/docker-stacks/issues/1205) +RUN sed -re "s/c.ServerApp/c.NotebookApp/g" \ + /etc/jupyter/jupyter_server_config.py > /etc/jupyter/jupyter_notebook_config.py && \ + fix-permissions.sh /etc/jupyter/ + +USER ${NB_UID} + +WORKDIR "${HOME}" \ No newline at end of file diff --git a/base/fix-permissions.sh b/base/fix-permissions.sh new file mode 100644 index 0000000..472ea28 --- /dev/null +++ b/base/fix-permissions.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# set permissions on a directory +# after any installation, if a directory needs to be (human) user-writable, +# run this script on it. +# It will make everything in the directory owned by the group ${NB_GID} +# and writable by that group. +# Deployments that want to set a specific user id can preserve permissions +# by adding the `--group-add users` line to `docker run`. + +# uses find to avoid touching files that already have the right permissions, +# which would cause massive image explosion + +# right permissions are: +# group=${NB_GID} +# AND permissions include group rwX (directory-execute) +# AND directories have setuid,setgid bits set + +set -e + +for d in "$@"; do + find "${d}" \ + ! \( \ + -group "${NB_GID}" \ + -a -perm -g+rwX \ + \) \ + -exec chgrp "${NB_GID}" -- {} \+ \ + -exec chmod g+rwX -- {} \+ + # setuid, setgid *on directories only* + find "${d}" \ + \( \ + -type d \ + -a ! -perm -6000 \ + \) \ + -exec chmod +6000 -- {} \+ +done \ No newline at end of file diff --git a/base/jupyter_server_config.py b/base/jupyter_server_config.py new file mode 100644 index 0000000..deef9f6 --- /dev/null +++ b/base/jupyter_server_config.py @@ -0,0 +1,60 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +# mypy: ignore-errors + +import os +import stat +import subprocess + +from jupyter_core.paths import jupyter_data_dir + +c = get_config() # noqa: F821 +c.ServerApp.ip = "0.0.0.0" +c.ServerApp.open_browser = False + +# to output both image/svg+xml and application/pdf plot formats in the notebook file +c.InlineBackend.figure_formats = {"png", "jpeg", "svg", "pdf"} + +# https://github.com/jupyter/notebook/issues/3130 +c.FileContentsManager.delete_to_trash = False + +# Generate a self-signed certificate +OPENSSL_CONFIG = """\ +[req] +distinguished_name = req_distinguished_name +[req_distinguished_name] +""" +if "GEN_CERT" in os.environ: + dir_name = jupyter_data_dir() + pem_file = os.path.join(dir_name, "notebook.pem") + os.makedirs(dir_name, exist_ok=True) + + # Generate an openssl.cnf file to set the distinguished name + cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf") + if not os.path.isfile(cnf_file): + with open(cnf_file, "w") as fh: + fh.write(OPENSSL_CONFIG) + + # Generate a certificate if one doesn't exist on disk + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-newkey=rsa:2048", + "-days=365", + "-nodes", + "-x509", + "-subj=/C=XX/ST=XX/L=XX/O=generated/CN=generated", + f"-keyout={pem_file}", + f"-out={pem_file}", + ] + ) + # Restrict access to the file + os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) + c.ServerApp.certfile = pem_file + +# Change default umask for all subprocesses of the notebook server if set in +# the environment +if "NB_UMASK" in os.environ: + os.umask(int(os.environ["NB_UMASK"], 8)) \ No newline at end of file diff --git a/base/run.sh b/base/run.sh new file mode 100644 index 0000000..c8ad074 --- /dev/null +++ b/base/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +docker build -f Dockerfile -t surveily.developer.ai.jupyter:test . +docker run --rm -it --gpus all -p 8888:8888 surveily.developer.ai.jupyter:test +# docker run --rm -it surveily.developer.ai.jupyter:test bash \ No newline at end of file diff --git a/base/start-notebook.sh b/base/start-notebook.sh new file mode 100644 index 0000000..158f3b7 --- /dev/null +++ b/base/start-notebook.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +# The Jupyter command to launch +# JupyterLab by default +DOCKER_STACKS_JUPYTER_CMD="${DOCKER_STACKS_JUPYTER_CMD:=lab}" + +if [[ -n "${JUPYTERHUB_API_TOKEN}" ]]; then + echo "WARNING: using start-singleuser.sh instead of start-notebook.sh to start a server associated with JupyterHub." + exec /usr/local/bin/start-singleuser.sh "$@" +fi + +wrapper="" +if [[ "${RESTARTABLE}" == "yes" ]]; then + wrapper="run-one-constantly" +fi + +# shellcheck disable=SC1091,SC2086 +exec /usr/local/bin/start.sh ${wrapper} python3 -m jupyter ${DOCKER_STACKS_JUPYTER_CMD} ${NOTEBOOK_ARGS} "$@" \ No newline at end of file diff --git a/base/start.sh b/base/start.sh new file mode 100644 index 0000000..fef6b61 --- /dev/null +++ b/base/start.sh @@ -0,0 +1,262 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +# The _log function is used for everything this script wants to log. It will +# always log errors and warnings, but can be silenced for other messages +# by setting JUPYTER_DOCKER_STACKS_QUIET environment variable. +_log () { + if [[ "$*" == "ERROR:"* ]] || [[ "$*" == "WARNING:"* ]] || [[ "${JUPYTER_DOCKER_STACKS_QUIET}" == "" ]]; then + echo "$@" + fi +} +_log "Entered start.sh with args:" "$@" + +# The run-hooks function looks for .sh scripts to source and executable files to +# run within a passed directory. +run-hooks () { + if [[ ! -d "${1}" ]] ; then + return + fi + _log "${0}: running hooks in ${1} as uid / gid: $(id -u) / $(id -g)" + for f in "${1}/"*; do + case "${f}" in + *.sh) + _log "${0}: running script ${f}" + # shellcheck disable=SC1090 + source "${f}" + ;; + *) + if [[ -x "${f}" ]] ; then + _log "${0}: running executable ${f}" + "${f}" + else + _log "${0}: ignoring non-executable ${f}" + fi + ;; + esac + done + _log "${0}: done running hooks in ${1}" +} + +# A helper function to unset env vars listed in the value of the env var +# JUPYTER_ENV_VARS_TO_UNSET. +unset_explicit_env_vars () { + if [ -n "${JUPYTER_ENV_VARS_TO_UNSET}" ]; then + for env_var_to_unset in $(echo "${JUPYTER_ENV_VARS_TO_UNSET}" | tr ',' ' '); do + echo "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" + unset "${env_var_to_unset}" + done + unset JUPYTER_ENV_VARS_TO_UNSET + fi +} + + +# Default to starting bash if no command was specified +if [ $# -eq 0 ]; then + cmd=( "bash" ) +else + cmd=( "$@" ) +fi + +# NOTE: This hook will run as the user the container was started with! +run-hooks /usr/local/bin/start-notebook.d + +# If the container started as the root user, then we have permission to refit +# the jovyan user, and ensure file permissions, grant sudo rights, and such +# things before we run the command passed to start.sh as the desired user +# (NB_USER). +# +if [ "$(id -u)" == 0 ] ; then + # Environment variables: + # - NB_USER: the desired username and associated home folder + # - NB_UID: the desired user id + # - NB_GID: a group id we want our user to belong to + # - NB_GROUP: a group name we want for the group + # - GRANT_SUDO: a boolean ("1" or "yes") to grant the user sudo rights + # - CHOWN_HOME: a boolean ("1" or "yes") to chown the user's home folder + # - CHOWN_EXTRA: a comma separated list of paths to chown + # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown commands + + # Refit the jovyan user to the desired the user (NB_USER) + if id jovyan &> /dev/null ; then + if ! usermod --home "/home/${NB_USER}" --login "${NB_USER}" jovyan 2>&1 | grep "no changes" > /dev/null; then + _log "Updated the jovyan user:" + _log "- username: jovyan -> ${NB_USER}" + _log "- home dir: /home/jovyan -> /home/${NB_USER}" + fi + elif ! id -u "${NB_USER}" &> /dev/null; then + _log "ERROR: Neither the jovyan user or '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." + exit 1 + fi + # Ensure the desired user (NB_USER) gets its desired user id (NB_UID) and is + # a member of the desired group (NB_GROUP, NB_GID) + if [ "${NB_UID}" != "$(id -u "${NB_USER}")" ] || [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then + _log "Update ${NB_USER}'s UID:GID to ${NB_UID}:${NB_GID}" + # Ensure the desired group's existence + if [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then + groupadd --force --gid "${NB_GID}" --non-unique "${NB_GROUP:-${NB_USER}}" + fi + # Recreate the desired user as we want it + userdel "${NB_USER}" + useradd --home "/home/${NB_USER}" --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 --no-log-init "${NB_USER}" + fi + + # Move or symlink the jovyan home directory to the desired users home + # directory if it doesn't already exist, and update the current working + # directory to the new location if needed. + if [[ "${NB_USER}" != "jovyan" ]]; then + if [[ ! -e "/home/${NB_USER}" ]]; then + _log "Attempting to copy /home/jovyan to /home/${NB_USER}..." + mkdir "/home/${NB_USER}" + if cp -a /home/jovyan/. "/home/${NB_USER}/"; then + _log "Success!" + else + _log "Failed to copy data from /home/jovyan to /home/${NB_USER}!" + _log "Attempting to symlink /home/jovyan to /home/${NB_USER}..." + if ln -s /home/jovyan "/home/${NB_USER}"; then + _log "Success creating symlink!" + else + _log "ERROR: Failed copy data from /home/jovyan to /home/${NB_USER} or to create symlink!" + exit 1 + fi + fi + fi + # Ensure the current working directory is updated to the new path + if [[ "${PWD}/" == "/home/jovyan/"* ]]; then + new_wd="/home/${NB_USER}/${PWD:13}" + _log "Changing working directory to ${new_wd}" + cd "${new_wd}" + fi + fi + + # Optionally ensure the desired user get filesystem ownership of it's home + # folder and/or additional folders + if [[ "${CHOWN_HOME}" == "1" || "${CHOWN_HOME}" == "yes" ]]; then + _log "Ensuring /home/${NB_USER} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+(chown options: ${CHOWN_HOME_OPTS})}" + # shellcheck disable=SC2086 + chown ${CHOWN_HOME_OPTS} "${NB_UID}:${NB_GID}" "/home/${NB_USER}" + fi + if [ -n "${CHOWN_EXTRA}" ]; then + for extra_dir in $(echo "${CHOWN_EXTRA}" | tr ',' ' '); do + _log "Ensuring ${extra_dir} is owned by ${NB_UID}:${NB_GID} ${CHOWN_EXTRA_OPTS:+(chown options: ${CHOWN_EXTRA_OPTS})}" + # shellcheck disable=SC2086 + chown ${CHOWN_EXTRA_OPTS} "${NB_UID}:${NB_GID}" "${extra_dir}" + done + fi + + # Update potentially outdated environment variables since image build + export XDG_CACHE_HOME="/home/${NB_USER}/.cache" + + # Prepend ${CONDA_DIR}/bin to sudo secure_path + sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"${CONDA_DIR}/bin:\1\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path + + # Optionally grant passwordless sudo rights for the desired user + if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == "yes" ]]; then + _log "Granting ${NB_USER} passwordless sudo rights!" + echo "${NB_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/added-by-start-script + fi + + # NOTE: This hook is run as the root user! + run-hooks /usr/local/bin/before-notebook.d + + unset_explicit_env_vars + _log "Running as ${NB_USER}:" "${cmd[@]}" + exec sudo --preserve-env --set-home --user "${NB_USER}" \ + PATH="${PATH}" \ + PYTHONPATH="${PYTHONPATH:-}" \ + "${cmd[@]}" + # Notes on how we ensure that the environment that this container is started + # with is preserved (except vars listed in JUPYTER_ENV_VARS_TO_UNSET) when + # we transition from running as root to running as NB_USER. + # + # - We use `sudo` to execute the command as NB_USER. What then + # happens to the environment will be determined by configuration in + # /etc/sudoers and /etc/sudoers.d/* as well as flags we pass to the sudo + # command. The behavior can be inspected with `sudo -V` run as root. + # + # ref: `man sudo` https://linux.die.net/man/8/sudo + # ref: `man sudoers` https://www.sudo.ws/man/1.8.15/sudoers.man.html + # + # - We use the `--preserve-env` flag to pass through most environment + # variables, but understand that exceptions are caused by the sudoers + # configuration: `env_delete` and `env_check`. + # + # - We use the `--set-home` flag to set the HOME variable appropriately. + # + # - To reduce the default list of variables deleted by sudo, we could have + # used `env_delete` from /etc/sudoers. It has higher priority than the + # `--preserve-env` flag and the `env_keep` configuration. + # + # - We preserve PATH and PYTHONPATH explicitly. Note however that sudo + # resolves `${cmd[@]}` using the "secure_path" variable we modified + # above in /etc/sudoers.d/path. Thus PATH is irrelevant to how the above + # sudo command resolves the path of `${cmd[@]}`. The PATH will be relevant + # for resolving paths of any subprocesses spawned by `${cmd[@]}`. + +# The container didn't start as the root user, so we will have to act as the +# user we started as. +else + # Warn about misconfiguration of: granting sudo rights + if [[ "${GRANT_SUDO}" == "1" || "${GRANT_SUDO}" == "yes" ]]; then + _log "WARNING: container must be started as root to grant sudo permissions!" + fi + + JOVYAN_UID="$(id -u jovyan 2>/dev/null)" # The default UID for the jovyan user + JOVYAN_GID="$(id -g jovyan 2>/dev/null)" # The default GID for the jovyan user + + # Attempt to ensure the user uid we currently run as has a named entry in + # the /etc/passwd file, as it avoids software crashing on hard assumptions + # on such entry. Writing to the /etc/passwd was allowed for the root group + # from the Dockerfile during build. + # + # ref: https://github.com/jupyter/docker-stacks/issues/552 + if ! whoami &> /dev/null; then + _log "There is no entry in /etc/passwd for our UID=$(id -u). Attempting to fix..." + if [[ -w /etc/passwd ]]; then + _log "Renaming old jovyan user to nayvoj ($(id -u jovyan):$(id -g jovyan))" + + # We cannot use "sed --in-place" since sed tries to create a temp file in + # /etc/ and we may not have write access. Apply sed on our own temp file: + sed --expression="s/^jovyan:/nayvoj:/" /etc/passwd > /tmp/passwd + echo "${NB_USER}:x:$(id -u):$(id -g):,,,:/home/jovyan:/bin/bash" >> /tmp/passwd + cat /tmp/passwd > /etc/passwd + rm /tmp/passwd + + _log "Added new ${NB_USER} user ($(id -u):$(id -g)). Fixed UID!" + + if [[ "${NB_USER}" != "jovyan" ]]; then + _log "WARNING: user is ${NB_USER} but home is /home/jovyan. You must run as root to rename the home directory!" + fi + else + _log "WARNING: unable to fix missing /etc/passwd entry because we don't have write permission. Try setting gid=0 with \"--user=$(id -u):0\"." + fi + fi + + # Warn about misconfiguration of: desired username, user id, or group id. + # A misconfiguration occurs when the user modifies the default values of + # NB_USER, NB_UID, or NB_GID, but we cannot update those values because we + # are not root. + if [[ "${NB_USER}" != "jovyan" && "${NB_USER}" != "$(id -un)" ]]; then + _log "WARNING: container must be started as root to change the desired user's name with NB_USER=\"${NB_USER}\"!" + fi + if [[ "${NB_UID}" != "${JOVYAN_UID}" && "${NB_UID}" != "$(id -u)" ]]; then + _log "WARNING: container must be started as root to change the desired user's id with NB_UID=\"${NB_UID}\"!" + fi + if [[ "${NB_GID}" != "${JOVYAN_GID}" && "${NB_GID}" != "$(id -g)" ]]; then + _log "WARNING: container must be started as root to change the desired user's group id with NB_GID=\"${NB_GID}\"!" + fi + + # Warn if the user isn't able to write files to ${HOME} + if [[ ! -w /home/jovyan ]]; then + _log "WARNING: no write access to /home/jovyan. Try starting the container with group 'users' (100), e.g. using \"--group-add=users\"." + fi + + # NOTE: This hook is run as the user we started the container as! + run-hooks /usr/local/bin/before-notebook.d + unset_explicit_env_vars + _log "Executing the command:" "${cmd[@]}" + exec "${cmd[@]}" +fi \ No newline at end of file diff --git a/docs/images/header.png b/docs/images/header.png new file mode 100644 index 0000000..4546a8e Binary files /dev/null and b/docs/images/header.png differ