Merge pull request #4645 from shaneknapp/data100-jl4

[DH-51] initial commit of data100-jl4 hub
berkeley-dsep-infra · Jun 14, 2023 · bcb6846 · bcb6846
2 parents f182873 + 9293fe0
commit bcb6846
Show file tree

Hide file tree

Showing 14 changed files with 486 additions and 0 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -276,6 +276,12 @@ jobs:
             hubploy deploy --timeout 30m data100 hub ${CIRCLE_BRANCH}
           no_output_timeout: 30m
 
+      - run:
+          name: Deploy data100-jl4
+          command: |
+            hubploy deploy --timeout 30m data100-jl4 hub ${CIRCLE_BRANCH}
+          no_output_timeout: 30m
+
       - run:
           name: Deploy shiny
           command: |
@@ -473,6 +479,15 @@ workflows:
                 ignore:
                 - staging
                 - prod
+      - hubploy/build-image:
+          deployment: data100-jl4
+          name: data100-jl4 image build
+          # Filters can only be per-job? wtf
+          filters:
+              branches:
+                ignore:
+                - staging
+                - prod
       - hubploy/build-image:
           deployment: stat159
           name: stat159 image build
@@ -659,6 +674,15 @@ workflows:
               branches:
                 only:
                 - staging
+      - hubploy/build-image:
+          deployment: data100-jl4
+          name: data100-jl4 image build
+          push: true
+          # Filters can only be per-job? wtf
+          filters:
+              branches:
+                only:
+                - staging
       - hubploy/build-image:
           deployment: stat159
           name: stat159 image build
@@ -760,6 +784,7 @@ workflows:
               - datahub image build
               - data8 image build
               - data100 image build
+              - data100-jl4 image build
 #              - astro image build
               - stat159 image build
               - stat20 image build

diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -64,6 +64,8 @@ node-placeholder-scaler:
   - "deployments/cee/**"
 "hub: data100":
   - "deployments/data100/**"
+"hub: data100-jl4":
+  - "deployments/data100-jl4/**"
 "hub: data101":
   - "deployments/data101/**"
 "hub: data102":

diff --git a/deployments/data100-jl4/config/common.yaml b/deployments/data100-jl4/config/common.yaml
@@ -0,0 +1,54 @@
+nfsPVC:
+  enabled: true
+  nfs:
+    serverIP: 10.203.15.114
+
+jupyterhub:
+  hub:
+    config:
+      Authenticator:
+        admin_users:
+          # infrastructure
+          - balajialwar
+          - felder
+          - gmerritt
+          - rylo
+          - sknapp
+
+#  prePuller:
+#    extraImages:
+#      postgres:
+#        name: gcr.io/ucb-datahub-2018/jupyterhub-postgres
+#        tag: 0.0.1-n3657.h4f7f88c
+  singleuser:
+    defaultUrl: "/lab"
+    extraEnv:
+      GH_SCOPED_CREDS_CLIENT_ID: Iv1.f79b4903c7ea2847
+      GH_SCOPED_CREDS_APP_URL: https://github.com/apps/data100-berkeley-datahub-access
+    nodeSelector:
+      hub.jupyter.org/pool-name: data100-pool
+    storage:
+      type: static
+      static:
+        pvcName: home-nfs-v3
+        subPath: "{username}"
+      extraVolumeMounts:
+        - name: home
+          mountPath: /home/jovyan/shared
+          subPath: _shared
+          readOnly: true
+    memory:
+      guarantee: 2G
+      limit: 2G
+    image: {}
+
+  custom:
+    admin:
+      mem_limit: 4G
+      mem_guarantee: 2G
+      extraVolumeMounts:
+        - name: home
+          mountPath: /srv/homes
+        - name: home
+          mountPath: /home/jovyan/shared-readwrite
+          subPath: _shared
diff --git a/deployments/data100-jl4/config/prod.yaml b/deployments/data100-jl4/config/prod.yaml
@@ -0,0 +1,35 @@
+nfsPVC:
+  nfs:
+    shareName: shares/data100-jl4/prod
+
+jupyterhub:
+  ingress:
+    enabled: true
+    hosts:
+      - data100-jl4.datahub.berkeley.edu
+    tls:
+      - secretName: tls-cert
+        hosts:
+          - data100-jl4.datahub.berkeley.edu
+  proxy:
+    chp:
+      resources:
+        requests:
+          # Give data100 a lot of guaranteed CPU, prevent 503s?
+          # https://github.com/berkeley-dsep-infra/datahub/issues/2677
+          cpu: 1
+
+  hub:
+    resources:
+      requests:
+        # Give data100 a lot of guaranteed CPU, prevent 503s?
+        # https://github.com/berkeley-dsep-infra/datahub/issues/2677
+        cpu: 1
+    db:
+      pvc:
+        # This also holds logs
+        storage: 40Gi
+
+  scheduling:
+    userPlaceholder:
+      enabled: false
diff --git a/deployments/data100-jl4/config/staging.yaml b/deployments/data100-jl4/config/staging.yaml
@@ -0,0 +1,16 @@
+nfsPVC:
+  nfs:
+    shareName: shares/data100-jl4/staging
+
+jupyterhub:
+  prePuller:
+    continuous:
+      enabled: false
+  ingress:
+    enabled: true
+    hosts:
+      - data100-jl4-staging.datahub.berkeley.edu
+    tls:
+      - secretName: tls-cert
+        hosts:
+          - data100-jl4-staging.datahub.berkeley.edu
diff --git a/deployments/data100-jl4/hubploy.yaml b/deployments/data100-jl4/hubploy.yaml
@@ -0,0 +1,19 @@
+images:
+  images:
+    - name: us-central1-docker.pkg.dev/ucb-datahub-2018/user-images/data100-jl4-user-image
+      path: image/
+      repo2docker:
+        base_image: docker.io/library/buildpack-deps:jammy
+  registry:
+    provider: gcloud
+    gcloud:
+      project: ucb-datahub-2018
+      service_key: gcr-key.json
+
+cluster:
+  provider: gcloud
+  gcloud:
+    project: ucb-datahub-2018
+    service_key: gke-key.json
+    cluster: fall-2019
+    zone: us-central1
diff --git a/deployments/data100-jl4/image/apt.txt b/deployments/data100-jl4/image/apt.txt
@@ -0,0 +1,93 @@
+# Some linux packages for basic terminal work, particularly
+# oriented at users new to Unix/cmd line environments.
+
+# Basic unix tools
+man
+man-db
+manpages-posix
+manpages-dev
+manpages-posix-dev
+
+# Download tools
+curl
+wget
+
+# Core text editors on a *nix box: vim and emacs
+vim
+emacs-nox
+emacs-goodies-el
+
+# A couple of CLI editors that are easier than vim
+# micro  # currently not working on 18.04
+nano
+jed
+jed-extra
+
+# powerful terminal-based file manager, better than the one in JLab
+mc
+
+# for easily managing multiple repositories with one command (perl-doc
+# is needed for its help pages to work)
+mr
+perl-doc
+
+# Regular build tools for compiling common stuff
+build-essential
+gfortran
+
+# Dependencies for nbconvert
+texlive-xetex
+texlive-fonts-recommended
+texlive-plain-generic
+# https://github.com/berkeley-dsep-infra/datahub/issues/3719
+texlive-lang-chinese
+lmodern
+
+# Other useful document-related tools
+pandoc
+latexdiff
+
+# Some useful git utilities use basic Ruby
+ruby
+
+# Other niceties for command-line work and life
+ack   # powerful grep-like tool
+pydf  # colorized disk usage
+tmux
+screen
+htop
+nnn   # cmd line file manager
+zsh
+rsync
+tig  # console UI for git
+multitail
+
+# For later, these are not available in 18.04
+#browsh # text-based web browser, occasionally handy
+#dasel  # json/yml/csv/etc data wrangling at the terminal
+#fzf   # fuzzy file finder
+
+## This section adds tools for desktop environment usage
+dbus-x11
+xorg
+xubuntu-icon-theme
+xfce4
+xfce4-goodies
+xclip
+xsel
+firefox
+chromium-browser
+
+# GUI text editors
+emacs
+vim-gtk3
+gedit
+
+# Git clients and tools
+git-gui
+gitg
+qgit
+meld
+
+# For jupyter-tree-download. Ref: https://github.com/berkeley-dsep-infra/datahub/issues/3979			
+zip
diff --git a/deployments/data100-jl4/image/environment.yml b/deployments/data100-jl4/image/environment.yml
@@ -0,0 +1,101 @@
+name: data100-s23
+
+channels:
+- conda-forge
+
+dependencies:
+- python==3.9
+- syncthing==1.20.4
+- git
+- altair==5.0.1
+- beautifulsoup4==4.11.1
+- black==22.6.0
+- bokeh==2.4.3
+- bqplot==0.12.34
+- cartopy==0.21.0
+- coverage==6.4.4
+- cython==0.29.32
+- dask==2023.6.0
+- dask-labextension==6.1.0
+- distributed==2023.6.0
+- fortran-magic==0.7
+- h5netcdf==1.0.2
+- h5py==3.7.0
+- hdf4==4.2.15
+- hdf5==1.12.2
+- intake==0.6.5
+- intake-esm==2021.8.17
+- intake-xarray==0.6.0
+- ipycanvas==0.12.1
+- ipydatagrid==1.1.12
+- ipympl==0.9.2
+- ipyparallel==8.4.1
+- jsonschema==4.17.3
+- jupyter-book==0.15.1
+- jupyter-repo2docker==2022.2.0
+  # - jupyter-resource-usage==0.6.1
+- jupyter_bokeh
+- jupyterlab==4.0.2
+- jupyterlab-favorites==3.0.0
+- jupyterlab-geojson==3.2.0
+  # - jupyterlab-git==0.39.0
+  # - jupyterlab-link-share==0.3.0
+- jupyterlab-variableinspector==3.0.9
+- jupyterlab_pygments==0.2.2
+- jupyterlab_server==2.23.0
+- jupyterlab_widgets==3.0.7
+- jupytext==1.14.0
+- jupyter_server==2.6.0
+- matplotlib==3.5.3
+- matplotlib-inline==0.1.6
+- mock==4.0.3
+- nbdime==3.1.1
+- networkx==2.8.6
+- numba
+- numpy==1.23.2
+- pandas==1.4.3
+- pandoc==2.19.2
+- pandocfilters==1.5.0
+- pep8==1.7.1
+- pillow==9.2.0
+- plotly==5.10.0
+- pooch==1.6.0
+- prettytable==3.4.1
+- pyarrow==9.0.0
+- pypdf2==2.10.4
+- pytables==3.7.0
+- pytest==7.1.2
+- pytest-cov==3.0.0
+- pytest-notebook==0.6.1
+- python-pdfkit==1.0.0 
+- requests==2.28.1
+- scikit-image==0.19.3
+- scikit-learn==1.1.2
+- scipy==1.9.0
+- seaborn==0.11.2
+- sphinx-jupyterbook-latex==0.5.2
+- sqlparse==0.4.3
+- statsmodels==0.13.2
+- sympy==1.10.1
+- tornado==6.2.0
+- tqdm==4.64.0
+- xarray==2022.6.0
+- xlrd==2.0.1
+- micro==2.0.8
+- websockify
+- folium==0.14.0
+- sqlalchemy==1.4.29
+- pip
+- pip:
+  - -r infra-requirements.txt
+  - jupyter-desktop-server
+    # For push authentication to GitHub
+  - gh-scoped-creds==4.1
+  # Upgrade separate from what everyone else uses for now
+  # https://github.com/berkeley-dsep-infra/datahub/issues/3693
+  - otter-grader==4.0.1
+  - ipython-sql==0.4.1
+  - geopandas==0.12.1
+  - iwut==0.0.4
+  - tensorflow-cpu==2.12.0
+  - jupyter_collaboration