readme and github actions

.env-dist

@@ -7,7 +7,5 @@ USER=gordon
 USER_ID=1000
 USER_GID=1000
 APT_PROXY=
-IMAGE_VERSION=
+IMAGE_VERSION=2308.05
 LISTEN_PORT=8888
-QB_CONF=/home/gordon/Notebooks/etc/broker.toml
-QB_STORAGE=/home/gordon/Notebooks/var/datastore

.github/dependabot.yml

@@ -0,0 +1,21 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "docker" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"

.github/workflows/docker-base-image.yml

@@ -0,0 +1,80 @@
+name: Check docker image
+
+on:
+  schedule:
+    - cron:  '20 4 * * *' # every day at 420 am
+  workflow_dispatch:
+
+env:
+    IMAGE_NAME: gnzsnz/jupyter-quant
+    BASE_IMAGE: python
+    PLATFORMS: linux/amd64,linux/arm64
+
+jobs:
+    check_base:
+        runs-on: ubuntu-latest
+        outputs:
+          needs-updating: ${{ steps.check.outputs.needs-updating }}
+        steps:
+
+          - name: Checkout
+            uses: actions/checkout@v3
+
+          - name: Get enviroment variables
+            run: |
+              grep -v '#' .env-dist | grep '=' > .env
+              while IFS= read -r line; do
+                  echo $line >> $GITHUB_ENV ;
+              done < .env
+
+          - name: Check if update available
+            id: check
+            uses: lucacome/docker-image-update-checker@v1
+            with:
+              base-image: ${{ env.BASE_IMAGE }}:${{ env.PYTHON_VERSION }}-slim
+              image: ${{ env.IMAGE_NAME}}:${{ env.IMAGE_VERSION }}
+              platforms: ${{ env.PLATFORMS }}
+
+    build:
+        runs-on: ubuntu-latest
+        needs: check_base
+        if: needs.check_base.outputs.needs-updating == 'true'
+        steps:
+
+          - name: Checkout
+            uses: actions/checkout@v3
+
+          - name: Create issue
+            env:
+              GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            run: |
+              gh issue create -t "Base images updates found for ${{ env.IMAGE_NAME }}:${{ env.IMAGE_VERSION }}" \
+              -b "A new build&publish might be needed."
+
+          - name: Get enviroment variables
+            run: |
+              grep -v '#' .env-dist | grep '=' > .env
+              while IFS= read -r line; do
+                  echo $line >> $GITHUB_ENV ;
+              done < .env
+
+          - name: Set up QEMU
+            uses: docker/setup-qemu-action@v2
+            with:
+              platforms: ${{ env.PLATFORMS }}
+
+          - name: Set up Docker Buildx
+            uses: docker/setup-buildx-action@v2
+
+          - name: Build Docker image
+            uses: docker/build-push-action@v4
+            with:
+              push: false
+              load: false
+              context: .
+              build-args: |
+                  USER=${{ env.USER }}
+                  USER_ID=${{ env.USER_ID }}
+                  USER_GID=${{ env.USER_GID }}
+                  PYTHON_VERSION=${{ env.PYTHON_VERSION}}
+              tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_VERSION }}

.github/workflows/docker-build-n-test.yml

@@ -0,0 +1,60 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: [ "master", "dev" ]
+  pull_request:
+    branches: [ "master" ]
+env:
+  USER: bastion
+  IMAGE_NAME: gnzsnz/bastion
+  SERVER_PORT: 2022
+
+jobs:
+  buildntest:
+    name: Build and test OpenSSH Bastion
+    strategy:
+      matrix:
+        image_os: [focal, jammy] # bastion docker image OS
+        runner_os: [20.04, 22.04] # CI runner
+        server_os: [focal, jammy] # server&client OS
+
+    runs-on: ubuntu-${{ matrix.runner_os}}
+    continue-on-error: true
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Get enviroment variables
+        run: |
+          grep -v '#' .env-dist | grep '=' > .env
+          while IFS= read -r line; do
+            echo $line >> $GITHUB_ENV ;
+          done < .env
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v4
+        with:
+          push: false
+          load: true
+          context: .
+          build-args: |
+            BASE_VERSION=${{ env.BASE_VERSION }}
+            IMAGE_VERSION=${{ env.IMAGE_VERSION }}
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_VERSION }}-${{ matrix.image_os }}
+
+      - name: Build server and client test image
+        uses: docker/build-push-action@v4
+        with:
+          push: false
+          load: true
+          context: test/
+          file: test/Dockerfile
+          build-args: |
+            BASE_VERSION=${{ env.BASE_VERSION }}
+          tags: openssh:${{ env.IMAGE_VERSION }}-${{ matrix.server_os }}

.github/workflows/docker-publish.yml

@@ -0,0 +1,71 @@
+name: Docker Image Publishing
+
+on:
+  release:
+    types: [published]
+
+env:
+  IMAGE_NAME: gnzsnz/jupyter-quant
+  PLATFORMS: linux/amd64,linux/arm64
+
+jobs:
+  publish:
+    name: Build and test Jupyter Quant
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Get enviroment variables
+        run: |
+          grep -v '#' .env-dist | grep '=' > .env
+          while IFS= read -r line; do
+            echo $line >> $GITHUB_ENV ;
+          done < .env
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: ${{ env.PLATFORMS }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.IMAGE_NAME }}
+            ghcr.io/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Bastion
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          build-args: |
+            USER=${{ env.USER }}
+            USER_ID=${{ env.USER_ID }}
+            USER_GID=${{ env.USER_GID }}
+            PYTHON_VERSION=${{ env.PYTHON_VERSION}}
+          platforms: ${{ env.PLATFORMS }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -63,7 +63,7 @@ ENV JUPYTERLAB_WORKSPACES_DIR="${JUPYTER_CONFIG_DIR}/lab/workspaces"
 ENV JUPYTER_SERVER_ROOT="/home/${USER}/Notebooks"
 # matplotlib
 ENV MPLCONFIGDIR="${BASE_CONFIG}/matplotlib"
-#shell
+# shell
 ENV SHELL="/bin/bash"
 
 COPY --from=builder /usr/share/fonts/truetype /usr/share/fonts/truetype

README.md

@@ -1,8 +1,103 @@
-# Jupyter Quant Research Enviroment
+# Jupyter Quant
 
-- https://github.com/jupyter-lsp/jupyterlab-lsp
+A dockerized Jupyter quant research enviroment.
 
+## Highlights
+
+- Designed for [ephemeral](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#create-ephemeral-containers) containers. Relevant data for your enviroment will survive your container.
+- Optimized for size, it's a 2GB image vs 4GB for jupyter/scipy-notebook
+- Includes all major python packages for data and timeseries analysis, see [requirements](https://github.com/gnzsnz/jupyter-quant/blob/master/requirements.txt)
+- Includes jedi language server
+- It does NOT include conda/mamba. All packages are installed wit pip under ~/.local/lib/python
+- Includes Cython, Numba, bottleneck and numexpr to speed up things
+- The usual suspects are included, numpy, pandas, sci-py, scikit-learn
+- Includes tools for quant analysis, statsmodels, pymc, arch, py_vollib, zipline-reloaded, PyPortfolioOpt, etc.
+- ib_insync for Interactive Broker connectivity. Works well with [IB Gateway](https://github.com/gnzsnz/ib-gateway-docker)
+- sudo, so you can install new packages if needed
+- bash and stow, so you can BYODF (bring your own dot files)
+- Support for [apt cache](https://github.com/gnzsnz/apt-cacher-ng). If you have other linux boxes using you can leverage your cache. apt cache support major linux distributions not only debian/ubuntu.
+
+
+## Volumes
+
+The image is designed to work with 3 volumes:
+
+1.  `quant_data`` - volume for `~/.local`` folder. It contains caches and all python packages. This enables to add additional packages through pip.
+1.  `quant_conf` - volume for `~/.config``, all config goes here. This includes jupyter, ipython, matplotlib, etc
+1.  Bind mount (but you could use a named volume) - volume for all notebooks, under `~/Notebooks``.
+
+This allows to have ephemeral containers and to keep your notebooks (3), your config (2) and your additional packages (1). Eventually you would need to update the image, in this case your notebooks (3) can move without issues, your config (2) should still work but no warranty, and your packages could still be used. Eventually you would need to refresh (1) and less frecuently (2)
+
+## Sample docker compose
+
+```yml
+version: "3.6"
+services:
+  jupyter-quant:
+    image: gnzsnz/jupyter-quant:${IMAGE_VERSION}
+    environment:
+      APT_PROXY: ${APT_PROXY:-}
+    restart: unless-stopped
+    ports:
+      - ${LISTEN_PORT}:8888
+    volumes:
+      - quant_conf:/home/gordon/.config
+      - quant_data:/home/gordon/.local
+      - ${PWD}/Notebooks:/home/gordon/Notebooks
+
+volumes:
+  quant_conf:
+  quant_data:
+```
+
+## Common tasks
+
+- get running server URL
+  
+```bash
+docker exec -it jupyterquant jupyter-server list
+Currently running servers:
+http://40798f7a604a:8888/?token=ebf9e870d2aa0ed877590eb83b4d3bbbdfbd55467422a167 :: /home/gordon/Notebooks
+```
+
+you will need to change hostname, 40798f7a604a in this case by your docker host ip.
+
+- show jupyter config
+
+```bash
+docker exec -it jupyter-quat jupyter-server --show-config
+```
+
+- set password
+
+```bash
+docker exec -it jupyter-quat jupyter-server password
+```
+
+- get help
+
+```bash
+docker exec -it jupyter-quat jupyter-server --help
+docker exec -it jupyter-quat jupyter-lab --help
+```
+
+- get installed packeges
+
+```bash
+docker exec -it jupyter-quat pip list
+# outdated packages
+docker exec -it jupyter-quat pip list -o
+```
+
+- the image's entrypoint supports jupyter-lab parameters, for example
+
+```bash
+docker run -it --rm gnzsnz/jupyter-quant --core-mode
+docker run -it --rm gnzsnz/jupyter-quant --show-config-json
+```
+
+- or run a command in the container
 
 ```bash
-pip install --upgrade --no-cache-dir git+http://192.168.1.2:3000/gnzsnz/[email protected]
+docker run -it --rm gnzsnz/jupyter-quant bash
 ```

docker-compose.yml

@@ -1,6 +1,6 @@
 version: "3.6"
 services:
-  jupyter-lab:
+  jupyter-quant:
     build:
       context: .
       platforms:
@@ -14,7 +14,7 @@ services:
         USER_GID: ${USER_GID}
         PYTHON_VERSION: ${PYTHON_VERSION}
         APT_PROXY: ${APT_PROXY}
-    image: gnzsnz/jupyter-lab:${IMAGE_VERSION}
+    image: gnzsnz/jupyter-quant:${IMAGE_VERSION}
     environment:
       APT_PROXY: ${APT_PROXY}
       QB_CONF: ${QB_CONF}
@@ -23,10 +23,11 @@ services:
     ports:
       - ${LISTEN_PORT}:8888
     volumes:
-      - lab_conf:/home/gordon/.config
-      - lab_data:/home/gordon/.local
+      - quant_conf:/home/gordon/.config
+      - quant_data:/home/gordon/.local
       - ${PWD}/Notebooks:/home/gordon/Notebooks
 
 volumes:
-  lab_data:
-  lab_conf:
+  quant_conf:
+  quant_data:
+