Merge pull request #1 from gnzsnz/dev

Dev

.env-dist

@@ -9,3 +9,5 @@ USER_GID=1000
 APT_PROXY=
 IMAGE_VERSION=2308.05
 LISTEN_PORT=8888
+#BYODF=/home/gordon/Notebooks/etc/dotfiles
+BYODF=

.github/workflows/docker-build-n-test.yml

@@ -50,8 +50,9 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           push: false
-          load: true
+          load: false
           context: .
+          platforms: ${{ env.PLATFORMS }}
           build-args: |
             USER=${{ env.USER }}
             USER_ID=${{ env.USER_ID }}

.gitignore

@@ -1,2 +1,3 @@
 .env
-./Notebooks/
+Notebooks/
+docker-compose.yml

Dockerfile

@@ -82,7 +82,7 @@ RUN if [ -n "$APT_PROXY" ]; then \
   ;fi && \
   groupadd --gid ${USER_GID} ${USER} && \
   useradd -ms /bin/bash --uid ${USER_ID} --gid ${USER_GID} ${USER} && \
-  echo "${USER} ALL=(ALL) NOPASSWD:ALL" | tee -a /etc/sudoers 
+  echo "${USER} ALL=(ALL) NOPASSWD:ALL" | tee -a /etc/sudoers
 
 USER $USER_ID:$USER_GID
 

README.md

@@ -8,27 +8,19 @@ A dockerized Jupyter quant research enviroment.
 - Optimized for size, it's a 2GB image vs 4GB for jupyter/scipy-notebook
 - Includes all major python packages for data and timeseries analysis, see [requirements](https://github.com/gnzsnz/jupyter-quant/blob/master/requirements.txt)
 - Includes jedi language server
-- It does NOT include conda/mamba. All packages are installed wit pip under ~/.local/lib/python
+- It does NOT include conda/mamba. All packages are installed with pip under ~/.local/lib/python. Which should be mounted in a dedicated volume to preserver your enviroment.
 - Includes Cython, Numba, bottleneck and numexpr to speed up things
 - The usual suspects are included, numpy, pandas, sci-py, scikit-learn
 - Includes tools for quant analysis, statsmodels, pymc, arch, py_vollib, zipline-reloaded, PyPortfolioOpt, etc.
 - ib_insync for Interactive Broker connectivity. Works well with [IB Gateway](https://github.com/gnzsnz/ib-gateway-docker)
-- sudo, so you can install new packages if needed
-- bash and stow, so you can BYODF (bring your own dot files)
+- sudo, so you can install new packages if needed.
+- bash and stow, so you can BYODF (bring your own dot files). Plus common command line utilities like git, less, nano (tiny), jq, ssh, curl, bash completion and others.
 - Support for [apt cache](https://github.com/gnzsnz/apt-cacher-ng). If you have other linux boxes using you can leverage your cache. apt cache support major linux distributions not only debian/ubuntu.
+- It does not include a build environment. If you need to install a package that does not provide wheels you can build your wheels, as explained in [common tasks](#common-tasks)
 
+## Quick Start
 
-## Volumes
-
-The image is designed to work with 3 volumes:
-
-1.  `quant_data`` - volume for `~/.local`` folder. It contains caches and all python packages. This enables to add additional packages through pip.
-1.  `quant_conf` - volume for `~/.config``, all config goes here. This includes jupyter, ipython, matplotlib, etc
-1.  Bind mount (but you could use a named volume) - volume for all notebooks, under `~/Notebooks``.
-
-This allows to have ephemeral containers and to keep your notebooks (3), your config (2) and your additional packages (1). Eventually you would need to update the image, in this case your notebooks (3) can move without issues, your config (2) should still work but no warranty, and your packages could still be used. Eventually you would need to refresh (1) and less frecuently (2)
-
-## Sample docker compose
+Create a `docker-compose.yml` file with this content
 
 ```yml
 version: "3.6"
@@ -37,6 +29,7 @@ services:
     image: gnzsnz/jupyter-quant:${IMAGE_VERSION}
     environment:
       APT_PROXY: ${APT_PROXY:-}
+      BYODF: ${BYODF:-}
     restart: unless-stopped
     ports:
       - ${LISTEN_PORT}:8888
@@ -50,6 +43,25 @@ volumes:
   quant_data:
 ```
 
+You can use `.env-dist` as your starting point.
+
+```bash
+cp .env-dist .env
+# verify everything looks good
+docker compose config
+docker compose up
+```
+
+## Volumes
+
+The image is designed to work with 3 volumes:
+
+1.  `quant_data` - volume for ~/.local folder. It contains caches and all python packages. This enables to add additional packages through pip.
+1.  `quant_conf` - volume for ~/.config, all config goes here. This includes jupyter, ipython, matplotlib, etc
+1.  Bind mount (but you could use a named volume) - volume for all notebooks, under `~/Notebooks`.
+
+This allows to have ephemeral containers and to keep your notebooks (3), your config (2) and your additional packages (1). Eventually you would need to update the image, in this case your notebooks (3) can move without issues, your config (2) should still work but no warranty, and your packages could still be used. Eventually you would need to refresh (1) and less frecuently (2)
+
 ## Common tasks
 
 - get running server URL
@@ -60,33 +72,39 @@ Currently running servers:
 http://40798f7a604a:8888/?token=ebf9e870d2aa0ed877590eb83b4d3bbbdfbd55467422a167 :: /home/gordon/Notebooks
 ```
 
-you will need to change hostname, 40798f7a604a in this case by your docker host ip.
+or
+
+```bash
+docker logs -t jupyter-quant 2>&1 | grep '127.0.0.1:8888/lab?token='
+```
+
+You will need to change hostname (40798f7a604a in this case) or 127.0.0.1 by your docker host ip.
 
 - show jupyter config
 
 ```bash
-docker exec -it jupyter-quat jupyter-server --show-config
+docker exec -it jupyter-quant jupyter-server --show-config
 ```
 
 - set password
 
 ```bash
-docker exec -it jupyter-quat jupyter-server password
+docker exec -it jupyter-quant jupyter-server password
 ```
 
 - get help
 
 ```bash
-docker exec -it jupyter-quat jupyter-server --help
-docker exec -it jupyter-quat jupyter-lab --help
+docker exec -it jupyter-quant jupyter-server --help
+docker exec -it jupyter-quant jupyter-lab --help
 ```
 
-- get installed packeges
+- get installed packages
 
 ```bash
-docker exec -it jupyter-quat pip list
+docker exec -it jupyter-quant pip list
 # outdated packages
-docker exec -it jupyter-quat pip list -o
+docker exec -it jupyter-quant pip list -o
 ```
 
 - the image's entrypoint supports jupyter-lab parameters, for example
@@ -101,3 +119,17 @@ docker run -it --rm gnzsnz/jupyter-quant --show-config-json
 ```bash
 docker run -it --rm gnzsnz/jupyter-quant bash
 ```
+
+- build wheels outside the container and import wheels to container
+
+```bash
+# make sure python version match .env-dist
+docker run -it --rm -v $PWD/wheels:/wheels python:3.11 bash
+pip wheel --no-cache-dir --wheel-dir /wheels numpy
+```
+
+This will build wheels for numpy (ot any other package that you need) and save the file in $PWD/wheels. Then you can copy the wheels in your notebooks mount (3 above) and install it within the container. You can even drag and drop into jupyter.
+
+- Install your dotfiles.
+
+`git clone` your dotfiles to `Notebook/etc/dotfiles`, set enviroment variable `BYODF=/home/gordon/Notebook/etc/dotfiles` in your docker compose. When the container starts up stow will create links like `/home/gordon/.bashrc`

docker-compose.yml → docker-compose.yml-dist

@@ -6,8 +6,6 @@ services:
       platforms:
         - "linux/amd64"
         #- "linux/arm64"
-        #- linux/arm32v7"
-        #- linux/arm64v8"
       args:
         USER: ${USER}
         USER_ID: ${USER_ID}
@@ -17,8 +15,7 @@ services:
     image: gnzsnz/jupyter-quant:${IMAGE_VERSION}
     environment:
       APT_PROXY: ${APT_PROXY}
-      QB_CONF: ${QB_CONF}
-      QB_STORAGE: ${QB_STORAGE}
+      BYODF: ${BYODF:-}
     restart: unless-stopped
     ports:
       - ${LISTEN_PORT}:8888

entrypoint.sh

@@ -12,10 +12,6 @@
 set -e
 
 DAEMON=jupyter-lab
-#JUPYTER_OPT=''
-
-# jupyterlab-lsp
-JUPYTER_OPT='--ContentsManager.allow_hidden=True'
 
 # APT Proxy Cache
 if [ -n "${APT_PROXY}" ]; then
@@ -30,6 +26,9 @@ if [ -d "$BYODF" ]; then
     git -C "$BYODF" reset --hard
 fi;
 
+# jupyterlab-lsp
+JUPYTER_OPT='--ContentsManager.allow_hidden=True'
+
 # language server symlink
 if [ ! -L "${JUPYTER_SERVER_ROOT}"/.lsp_symlink ]; then
     ln -s / .lsp_symlink

requirements.txt

@@ -25,4 +25,4 @@ scipy
 seaborn
 statsmodels
 yfinance
-zipline-reloaded 
+zipline-reloaded