From 7406a2a89ec1569e857f6d6f6f3141b67c6c9502 Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Thu, 14 Sep 2023 08:58:59 -0400 Subject: [PATCH] refactor: update dev container and pod (#477) * fix: remove syntax directive from conda Dockerfile * fix: update devcontainer for compatibility with dp - Update devcontainer name to "pyrovelocity-dev". - Update dockerfile path to "../dockerfiles/Dockerfile.conda". - Remove the "docker-in-docker" feature. * fix: add `.envrc` to `.gitignore` * feat: add dockerignore * fix: update vscode settings and extensions * fix: add make targets to install direnv and just * feat: add example envrc * feat: add justfile for devcontainer --------- Co-authored-by: pyrovelocity[bot] <144943013+pyrovelocity-bot@users.noreply.github.com> --- .devcontainer/devcontainer.json | 7 +- .dockerignore | 121 ++++++++++++++++++++++++++++++++ .envrc.example | 12 ++++ .gitignore | 2 +- .vscode/extensions.json | 22 ++++++ .vscode/settings.json | 6 ++ Makefile | 45 ++++++++++-- dockerfiles/Dockerfile.conda | 1 - justfile | 119 +++++++++++++++++++++++++++++++ 9 files changed, 322 insertions(+), 13 deletions(-) create mode 100644 .dockerignore create mode 100644 .envrc.example create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json create mode 100644 justfile diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e72524a35..a514271af 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,12 +1,9 @@ { - "name": "pyrovelocity", + "name": "pyrovelocity-dev", "build": { - "dockerfile": "../dockerfiles/Dockerfile.conda.cpu", + "dockerfile": "../dockerfiles/Dockerfile.conda", "context": ".." }, - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {} - }, "customizations": { "vscode": { "extensions": [ diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..895211531 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,121 @@ +# +archive/ + +# +.DS_Store + +# +monkeytype.sqlite3 + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env +.envrc + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# IDE settings +*.code-workspace +.VSCodeCounter +.idea/ +*.pdf +*.tif +*.h5ad +.nox/ diff --git a/.envrc.example b/.envrc.example new file mode 100644 index 000000000..f5aa5364d --- /dev/null +++ b/.envrc.example @@ -0,0 +1,12 @@ +export CONTAINER_BUILDER=podman # path to container management executable +export GITHUB_USERNAME=username # github username associated to uploading startup scripts as github gists +export GITHUB_ORG_NAME=pinellolab # name of the github org or user containing the github repository with code for development +export GITHUB_REPO_NAME=pyrovelocity # name of a github repository with a conda environment yaml file +export GITHUB_BRANCH_NAME=master # name of github repository branch to checkout +export POD_DISK_SIZE=400Gi # size of the PVC to mount to the devpod +export POD_ACCELERATOR_TYPE=nvidia-tesla-t4 # cluster-specific node selector for accelerator type | nvidia-l4 | nvidia-tesla-a100 +export POD_MIN_CPU=16 # cpu resource request for kubernetes pod +export POD_MIN_MEM=64Gi # ram resource request for kubernetes pod +export POD_MAX_CPU=30 # cpu resource limit for kubernetes pod +export POD_MAX_MEM=96Gi # ram resource limit for kubernetes pod +export POD_MAX_ACCEL=1 # accelerator resource limit for kubernetes pod diff --git a/.gitignore b/.gitignore index 5b3f12e97..895211531 100644 --- a/.gitignore +++ b/.gitignore @@ -91,6 +91,7 @@ celerybeat-schedule # dotenv .env +.envrc # virtualenv .venv @@ -112,7 +113,6 @@ ENV/ # IDE settings *.code-workspace -.vscode/ .VSCodeCounter .idea/ *.pdf diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..5264dc10c --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,22 @@ +{ + "recommendations": [ + "donjayamanne.python-extension-pack", + "redhat.vscode-yaml", + "sclu1034.justfile", + "ms-vscode.makefile-tools", + "hashicorp.terraform", + "ms-python.black-formatter", + "googlecloudtools.cloudcode", + "ms-kubernetes-tools.vscode-kubernetes-tools", + "ms-vsliveshare.vsli", + "eamodio.gitlens", + "GitHub.vscode-pull-request-github", + "github.vscode-github-actions", + "ms-azuretools.vscode-docker", + "ms-toolsai.jupyter", + "iterative.dvc", + "njzy.stats-bar", + "vscode-icons-team.vscode-icons", + "poimandres.theme-poimandres" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..751bf1b3a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true + } +} diff --git a/Makefile b/Makefile index 5f1414c56..9de5b5456 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,45 @@ help: ## Display this help. (Default) help_sort: ## Display alphabetized version of help. @grep -hE '^[A-Za-z0-9_ \-]*?:.*##.*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +#------------- +# system / dev +#------------- + +install_direnv: ## Install direnv to `/usr/local/bin`. Check script before execution: https://direnv.net/ . + @which direnv > /dev/null || \ + (curl -sfL https://direnv.net/install.sh | bash && \ + sudo install -c -m 0755 direnv /usr/local/bin && \ + rm -f ./direnv) + @echo "see https://direnv.net/docs/hook.html" + +install_just: ## Install just. Check script before execution: https://just.systems/ . + @which cargo > /dev/null || (curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh) + @cargo install just + +precommit: ## Run pre-commit hooks using nox. + nox -x -rs pre-commit + +env_print: ## Print a subset of environment variables defined in ".envrc" file. + env | grep "TF_VAR\|GITHUB\|GH_\|GCP_\|MLFLOW" | sort + +approve_prs: ## Approve github pull requests from bots: PR_ENTRIES="2-5 10 12-18" + for entry in $(PR_ENTRIES); do \ + if [[ "$$entry" == *-* ]]; then \ + start=$${entry%-*}; \ + end=$${entry#*-}; \ + for pr in $$(seq $$start $$end); do \ + @gh pr review $$pr --approve; \ + done; \ + else \ + @gh pr review $$entry --approve; \ + fi; \ + done + + +#---------------- +# web application +#---------------- + st: ## Run streamlit app in local environment. streamlit run app/app.py \ --server.port=8080 \ @@ -96,9 +135,3 @@ ifdef GCP_RUN_SERVICE_NAME else @echo 'Run "make help" and define necessary variables' endif - -precommit: ## Run pre-commit hooks using nox. - nox -x -rs pre-commit - -env_print: ## Print a subset of environment variables defined in ".env" file. - env | grep "TF_VAR\|GITHUB\|GH_\|GCP_\|MLFLOW" | sort diff --git a/dockerfiles/Dockerfile.conda b/dockerfiles/Dockerfile.conda index 390fedd48..fde371107 100644 --- a/dockerfiles/Dockerfile.conda +++ b/dockerfiles/Dockerfile.conda @@ -1,4 +1,3 @@ -# syntax=docker/dockerfile:1 FROM condaforge/mambaforge:23.1.0-1 LABEL maintainer="pyrovelocity team" diff --git a/justfile b/justfile new file mode 100644 index 000000000..eac03bf9e --- /dev/null +++ b/justfile @@ -0,0 +1,119 @@ +# Default command when 'just' is run without arguments +# Run 'just ' to execute a command. +default: list + +# Display help +help: + @printf "\nSee Makefile targets for just and direnv installation." + @printf "\nRun 'just -n ' to print what would be executed...\n\n" + @just --list --unsorted + @echo "\n...by running 'just '.\n" + @echo "This message is printed by 'just help'." + @echo "Just 'just' will just list the available recipes.\n" + +# List just recipes +list: + @just --list --unsorted + +# List evaluated just variables +vars: + @just --evaluate + +builder := env_var_or_default('BUILDER', 'podman') +container_user := "runner" +container_home := "/home" / container_user +container_work := container_home / "work" +git_username := env_var_or_default('GITHUB_USERNAME', 'pinellolab') +git_org_name := env_var_or_default('GITHUB_ORG_NAME', 'pinellolab') +git_repo_name := env_var_or_default('GITHUB_REPO_NAME', 'pyrovelocity') +git_branch_name := env_var_or_default('GITHUB_BRANCH_NAME', 'master') +container_registry := "ghcr.io/" + git_org_name + "/" +pod_accelerator_type := env_var_or_default('POD_ACCELERATOR_TYPE', 'nvidia-tesla-t4') +accelerator_node_selector := "gpu-type=" + pod_accelerator_type + +container_type := "dev" # or "app" +container_image := if container_type == "dev" { + "pyrovelocitygpu" + } else if container_type == "app" { + "pyrovelocityapp" + } else { + error("container_type must be either 'dev' or 'app'") + } +container_tag := "latest" + +pod_source_type := env_var_or_default('POD_SOURCE_TYPE', 'git') +pod_git_provider := env_var_or_default('POD_GIT_PROVIDER', 'github') +pod_disk_size := env_var_or_default('POD_DISK_SIZE', '400Gi') +pod_min_cpu := env_var_or_default('POD_MIN_CPU', '16') +pod_min_mem := env_var_or_default('POD_MIN_MEM', '64Gi') +pod_max_cpu := env_var_or_default('POD_MAX_CPU', '32') +pod_max_mem := env_var_or_default('POD_MAX_MEM', '96Gi') +pod_max_accel := env_var_or_default('POD_MAX_ACCEL', '1') +pod_resources := "requests.cpu=" + pod_min_cpu + ",requests.memory=" + pod_min_mem + ",limits.cpu=" + pod_max_cpu + ",limits.memory=" + pod_max_mem + ",limits.nvidia.com/gpu=" + pod_max_accel + +architecture := if arch() == "x86_64" { + "amd64" + } else if arch() == "aarch64" { + "arm64" + } else { + error("unsupported architecture must be amd64 or arm64") + } + +opsys := if os() == "macos" { + "darwin" + } else if os() == "linux" { + "linux" + } else { + error("unsupported operating system must be darwin or linux") + } + +devpod_release := "latest" # or "v0.3.7" or "v0.4.0-alpha.4" + +devpod_binary_url := if devpod_release == "latest" { + "https://github.com/loft-sh/devpod/releases/latest/download/devpod-" + opsys + "-" + architecture +} else { + "https://github.com/loft-sh/devpod/releases/download/" + devpod_release + "/devpod-" + opsys + "-" + architecture +} + +# Install devpod (check/set: devpod_release) +[unix] +install-devpod: + curl -L -o devpod {{devpod_binary_url}} && \ + sudo install -c -m 0755 devpod /usr/local/bin && \ + rm -f devpod + which devpod + devpod version + +# Print devpod info +devpod: + devpod version && echo + devpod context list + devpod provider list + devpod list + +# Install and use devpod kubernetes provider +provider: + devpod provider add kubernetes --silent || true \ + && devpod provider use kubernetes + +# Run latest container_image in current kube context +pod: + devpod up \ + --debug \ + --devcontainer-image {{container_registry}}{{container_image}}:{{container_tag}} \ + --provider kubernetes \ + --ide vscode \ + --open-ide \ + --source {{pod_source_type}}:https://{{pod_git_provider}}.com/{{git_username}}/{{git_repo_name}} \ + --provider-option DISK_SIZE={{pod_disk_size}} \ + --provider-option NODE_SELECTOR={{accelerator_node_selector}} \ + --provider-option RESOURCES={{pod_resources}} \ + {{container_image}} + +# Stop devpod (check container_image hasn't changed with -n) +stop: + devpod stop {{container_image}} + +# Delete devpod (check container_image hasn't changed with -n) +delete: + devpod delete {{container_image}}