Devcontainer for development environment

.devcontainer/Dockerfile

@@ -0,0 +1,66 @@
+# Copyright (C) 2024 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+FROM cvat/server:dev
+
+ARG HOST_USER_UID
+ARG HOST_USER_GID
+ARG NUCLIO_VERSION
+ARG OPA_VERSION
+
+ENV DJANGO_CONFIGURATION="development"
+ENV DJANGO_SETTINGS_MODULE cvat.settings.${DJANGO_CONFIGURATION}
+
+USER root
+
+# Save commit hash of datumaro git repository for using it while updating virtal environment in
+# postcreate.sh script
+RUN mkdir -p /cvat/metadata \
+    && sed --quiet 's|^datumaro @ git+https://github.com/cvat-ai/datumaro.git@\([0-9a-f]\{40\}\)$|\1|p' \
+    /home/django/cvat/requirements/base.txt > /cvat/metadata/datumaro_commit.txt
+
+# It is needed that the host user uid matches with the container user uid, else \
+# permission error occur on the bind volume mounts
+# So original django user is replaced by devcontainer with host uid value
+RUN userdel -r django 2>/dev/null \
+    && groupadd --gid ${HOST_USER_GID} devcontainer \
+    && useradd --uid ${HOST_USER_UID} --gid ${HOST_USER_GID} \
+    --create-home --shell /bin/bash --comment "" "devcontainer"
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -qy \
+        parallel=20210822+ds-2 \
+        wget=1.21.2-2ubuntu1
+
+RUN curl --silent "https://api.github.com/repos/nuclio/nuclio/releases/tags/${NUCLIO_VERSION}" \
+        | sed --silent 's/"browser_download_url": "\(.*nuctl.*linux.*amd64\)"/\1/Ip' \
+        | wget --tries 3 --output-document /usr/local/bin/nuctl --quiet --input-file - \
+    && chmod +x /usr/local/bin/nuctl
+
+RUN curl --silent "https://api.github.com/repos/open-policy-agent/opa/releases/tags/v${OPA_VERSION}" \
+        | sed --silent 's/"browser_download_url": "\(.*opa.*linux.*amd64.*static\)"/\1/Ip' \
+        | wget --tries 3 --output-document /usr/local/bin/opa --quiet --input-file - \
+    && chmod +x /usr/local/bin/opa
+
+RUN /usr/bin/python3 -m venv /opt/venv-test
+ENV PATH="/opt/venv-test/bin:${PATH}"
+
+# This command shall the slow in linux docker host running old kernel as it shall do
+# regular copy instead of metadata only copy for chown operation.
+# Tracking this issue at https://github.com/docker/for-linux/issues/388#issuecomment-1108689413
+RUN chown -R devcontainer /opt/venv /opt/venv-test
+
+# If the volume mounted directories does not exist then they shall be created
+# automatically during and they shall be owned by the root user and shall be
+# inaccessible by the container user. Therefore, create these directories as the
+# container user during build
+USER devcontainer
+WORKDIR /home/devcontainer/workspaces
+
+RUN mkdir -p /home/devcontainer/workspaces/data /home/devcontainer/workspaces/keys \
+    /home/devcontainer/workspaces/logs /home/devcontainer/workspaces/static \
+    /home/devcontainer/workspaces/node_modules /home/devcontainer/.cache/pip \
+    /home/devcontainer/.vscode-server-insiders /home/devcontainer/.vscode-server \
+    /home/devcontainer/.ssh

.devcontainer/cvat_server_init.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -eu
+
+echo_bold() {
+    bold=$(tput bold)
+    normal=$(tput sgr0)
+    text="${1}"
+    echo "${bold}${text}${normal}"
+}
+
+error_handler() {
+    local error_code=${?}
+    local error_command="${BASH_COMMAND}"
+    local error_line="${BASH_LINENO}"
+
+    echo_bold "ERROR: ${0}: Error occurred on line ${error_line}: ${error_command} (exit code: ${error_code})" >&2
+    exit 1
+}
+
+db_operations() {
+    # Wait for postgres and redis containers to be running and available
+    ./wait-for-it.sh "${CVAT_POSTGRES_HOST}:${CVAT_POSTGRES_PORT:-5432}" -t 0
+    ./wait-for-it.sh "${CVAT_REDIS_INMEM_HOST}:${CVAT_REDIS_INMEM_PORT}" -t 0
+    ./wait-for-it.sh "${CVAT_REDIS_ONDISK_HOST}:${CVAT_REDIS_ONDISK_PORT}" -t 0
+
+    python manage.py migrate
+    while ! python manage.py migrate --check; do
+        echo_bold "Waiting for migrations to finish"
+        sleep 10
+    done
+    python manage.py createsuperuser --no-input >/dev/null 2>&1
+    echo_bold "INFO: Done database migrate and create superuser"
+}
+
+collect_static() {
+    python manage.py collectstatic --noinput
+    echo_bold "INFO: Done collect static files"
+}
+
+trap error_handler ERR
+workspacefolder="$(dirname "$(dirname "$(realpath "$0")")")"
+cd "${workspacefolder}"
+source /opt/venv/bin/activate
+echo_bold "INFO: Start operations"
+export -f echo_bold && export -f db_operations && export -f collect_static
+parallel --jobs 2 --ungroup --halt now,fail=1 ::: db_operations collect_static
+echo_bold "INFO: Done operations"
+echo_bold "INFO: Run django server to serve the OPA client"
+python manage.py runserver 0.0.0.0:"${CVAT_PORT:-8080}"
+exit 0

.devcontainer/devcontainer-lock.json

@@ -0,0 +1,19 @@
+{
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "version": "2.4.3",
+      "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:e9e1d402031416ed5fc500f242c27ffa1043a27b5ba612e6596ea62503c8ae70",
+      "integrity": "sha256:e9e1d402031416ed5fc500f242c27ffa1043a27b5ba612e6596ea62503c8ae70"
+    },
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "2.10.2",
+      "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:23ae11a86089da5f0b98a6edd603f91831802b7f2d5ef1e104e1b94a3beb546c",
+      "integrity": "sha256:23ae11a86089da5f0b98a6edd603f91831802b7f2d5ef1e104e1b94a3beb546c"
+    },
+    "ghcr.io/devcontainers/features/node:1": {
+      "version": "1.4.1",
+      "resolved": "ghcr.io/devcontainers/features/node@sha256:6c4f444d8e3eb68594d60ec128bce76a80647edc2b9a94960ca377895855145f",
+      "integrity": "sha256:6c4f444d8e3eb68594d60ec128bce76a80647edc2b9a94960ca377895855145f"
+    }
+  }
+}

.devcontainer/devcontainer.json

@@ -0,0 +1,70 @@
+{
+    "name": "cvat-local",
+    "initializeCommand": "${localWorkspaceFolder}/.devcontainer/init.sh",
+    "postCreateCommand": "${containerWorkspaceFolder}/.devcontainer/postcreate.sh",
+    "dockerComposeFile": [
+        "${localWorkspaceFolder}/.devcontainer/docker-compose.rendered.yml"
+    ],
+    "service": "cvat_server",
+    "overrideCommand": false,
+    "runServices": [
+        "cvat_db",
+        "cvat_opa",
+        "cvat_redis_inmem",
+        "cvat_redis_ondisk"
+    ],
+    "workspaceFolder": "/home/devcontainer/workspaces",
+    "remoteUser": "devcontainer",
+    "containerUser": "devcontainer",
+    "remoteEnv": {"HOME": "/home/devcontainer", "USER": "devcontainer"},
+    "containerEnv": {"HOME": "/home/devcontainer", "USER": "devcontainer"},
+    "shutdownAction": "stopCompose",
+    "forwardPorts": [
+        8070
+    ],
+    "features": {
+        "ghcr.io/devcontainers/features/common-utils:2": {
+            "installZsh": true,
+            "configureZshAsDefaultShell": true,
+            "installOhMyZsh": true,
+            "installOhMyZshConfig": true,
+            "upgradePackages": true,
+            "username": "devcontainer",
+            "userUid": "automatic",
+            "userGid": "automatic"
+        },
+        "ghcr.io/devcontainers/features/node:1": {
+            "nodeGypDependencies": true,
+            "version": "16.20.2",
+            "nvmVersion": "latest"
+        },
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+    },
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "ms-vscode.js-debug",
+                "ms-python.python",
+                "ms-python.debugpy",
+                "dbaeumer.vscode-eslint",
+                "stylelint.vscode-stylelint",
+                "esbenp.prettier-vscode",
+                "ymotongpoo.licenser",
+                "shardulm94.trailing-spaces",
+                "streetsidesoftware.code-spell-checker",
+                "mtxr.sqltools",
+                "mtxr.sqltools-driver-pg"
+            ],
+            "settings": {
+                "python.defaultInterpreterPath": "/opt/venv/bin/python",
+                // Ideally venvPath should be an array, tracking this issue here
+                // https://github.com/microsoft/vscode-python/issues/1479
+                "python.venvPath": "/opt/venv-test/bin",
+                "terminal.integrated.automationProfile.linux": {
+                    "path": "/bin/bash"
+                },
+                "terminal.integrated.defaultProfile.linux": "zsh (2)"
+            }
+        }
+    }
+}

.devcontainer/dist.env

@@ -0,0 +1 @@
+GIT_BRANCH_ISOLATION=true

.devcontainer/docker-compose.nuclio.yml

@@ -0,0 +1,20 @@
+services:
+  nuclio:
+    container_name: nuclio
+    image: quay.io/nuclio/dashboard:1.11.24-amd64
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      http_proxy:
+      https_proxy:
+      no_proxy: ${no_proxy:-}
+      NUCLIO_CHECK_FUNCTION_CONTAINERS_HEALTHINESS: 'true'
+      NUCLIO_DASHBOARD_DEFAULT_FUNCTION_MOUNT_MODE: 'volume'
+    ports:
+      - '8070:8070'
+    logging:
+      driver: "json-file"
+      options:
+        max-size: 100m
+        max-file: "3"

.devcontainer/docker-compose.yml

@@ -0,0 +1,86 @@
+services:
+  cvat_server:
+    image: cvat/server:devcontainer
+    build:
+      dockerfile: .devcontainer/Dockerfile
+      context: .
+      args:
+        - pull # To fetch upstream base image (cvat/server:dev) changes on container rebuild
+        - HOST_USER_UID=${HOST_USER_UID:-1001}
+        - HOST_USER_GID=${HOST_USER_GID:-1001}
+        - NUCLIO_VERSION=1.13.0
+        - OPA_VERSION=0.63.0
+    restart: no
+    entrypoint: ["/bin/bash", "-c"]
+    command: ["sleep infinity"]
+    environment:
+      DEVCONTAINER: 1
+      CVAT_SERVERLESS: 1
+      IAM_OPA_HOST: http://opa:8181
+      DJANGO_LOG_SERVER_HOST: ${DJANGO_LOG_SERVER_HOST:-}
+      DJANGO_SUPERUSER_USERNAME: ${DJANGO_SUPERUSER_USERNAME:-admin}
+      DJANGO_SUPERUSER_PASSWORD: ${DJANGO_SUPERUSER_PASSWORD:-admin}
+      DJANGO_SUPERUSER_EMAIL: ${DJANGO_SUPERUSER_EMAIL:[email protected]}
+    volumes: !override
+      - .:/home/devcontainer/workspaces
+      - ${HOME}/.ssh:/home/devcontainer/.ssh
+      - cvat_data:/home/devcontainer/workspaces/data
+      - cvat_keys:/home/devcontainer/workspaces/keys
+      - cvat_logs:/home/devcontainer/workspaces/logs
+      # For improving write performance on non-linux machines, static files should be
+      # mounted to named volumes
+      - cvat_node_modules:/home/devcontainer/workspaces/node_modules
+      - cvat_server_static:/home/devcontainer/workspaces/static
+      - cvat_pip_cache:/home/devcontainer/.cache/pip
+      # Preserve important user data between rebuilds
+      - cvat_user_data:/home/devcontainer/
+      # Exclude .vscode-server directories to enable reinstall of extensions upon rebuild
+      - /home/devcontainer/.vscode-server-insiders
+      - /home/devcontainer/.vscode-server
+    labels:
+      - com.devcontainer.git_branch_${GIT_BRANCH:-}
+
+  cvat_db:
+    restart: no
+    labels:
+      - com.devcontainer.git_branch=${GIT_BRANCH:-}
+
+  cvat_redis_inmem:
+    restart: no
+    labels:
+      - com.devcontainer.git_branch=${GIT_BRANCH:-}
+
+  cvat_redis_ondisk:
+    restart: no
+    labels:
+      - com.devcontainer.git_branch=${GIT_BRANCH:-}
+
+  cvat_opa:
+    restart: no
+    labels:
+      - com.devcontainer.git_branch=${GIT_BRANCH:-}
+
+# VS code shuts down containers upon closing the window. If in any case if it fails to shut
+# down the containers they should not restart upon system restart. This shall save
+# unncessary memory usage of the system when not developing
+
+volumes:
+  cvat_node_modules:
+  cvat_server_static:
+  cvat_pip_cache:
+  cvat_user_data:
+  cvat_db:
+    name: cvat_db_${GIT_BRANCH:-}
+  cvat_data:
+    name: cvat_data_${GIT_BRANCH:-}
+  cvat_keys:
+    name: cvat_keys_${GIT_BRANCH:-}
+  cvat_logs:
+    name: cvat_logs_${GIT_BRANCH:-}
+  cvat_inmem_db:
+    name: cvat_inmem_db_${GIT_BRANCH:-}
+  cvat_cache_db:
+    name: cvat_cache_db_${GIT_BRANCH:-}
+
+# Volumes are parameterized with the current git branch values to isolate development environment based on git branches
+