diff --git a/.github/workflows/build_agent_container.yaml b/.github/workflows/build_agent_container.yaml new file mode 100644 index 00000000..c13bf13b --- /dev/null +++ b/.github/workflows/build_agent_container.yaml @@ -0,0 +1,74 @@ +name: Build and push container image + +# Configures this workflow to run every time a tag is created +on: + push: + branches: + - main + - docker_ci + paths: + - agent/** + - .github/workflows/build_agent_container.yaml + +# NOTE: we may want to switch to matrix build for multi-platform support if this is taking too long +# https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners + + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Setup for multi-platform + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build the agent container image + id: build + run: | + apt-get update && apt-get install -y make git jq + cd agent + export TAGS="-t ${REGISTRY@L}/${{env.IMAGE_NAME}}/agent:${{ github.sha }}" + export REGISTRY=${REGISTRY@L} + # Get the last tag and use it as the env var AGENT_VERSION if it doesn't exist use 0.0.0+{github.sha} + export AGENT_VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "0.0.0+${{ github.sha }}") + make docker-build-only agent_version=${AGENT_VERSION} + cat metadata.json + echo "digest=$(cat metadata.json | jq -r .\"containerimage.digest\")" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + env: + AGENT_IMAGE: ${{env.IMAGE_NAME}}/agent + + # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds). + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v2 + with: + subject-name: ${{ env.REGISTRY }}/${{env.IMAGE_NAME}}/agent + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + diff --git a/agent/Makefile b/agent/Makefile index 0d48629f..4890c1ca 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -27,7 +27,7 @@ help: ## Display this help. venv: ## Sets up a python venv at `./venv` python3 -m venv venv $(VENV)pip install hatch coverage - $(VENV)hatch config set dirs.project "[\"${PWD}\"]" + $(VENV)hatch config set dirs.project "[\"$(shell pwd)\"]" ##@ Test .PHONY: test @@ -46,8 +46,8 @@ format: ##@ Build .PHONY: build -build: ## Builds using hatch to `skyhook-agent/dist` - $(VENV)hatch -p skyhook-agent version ${1:unkown} +build: ## Builds using hatch to `dist` + $(VENV)hatch -p skyhook-agent version $(build_version) $(VENV)hatch -p skyhook-agent build -c ##@ Publish @@ -55,6 +55,7 @@ build: ## Builds using hatch to `skyhook-agent/dist` publish: ## Publishes using hatch $(VENV)hatch -p skyhook-agent publish +DOCKER_CMD ?= docker BUILD_ARGS ?= ifndef GITLAB_CI BUILD_ARGS = --push @@ -62,27 +63,32 @@ COMMIT_SHORT_SHA := $(shell git rev-parse --short HEAD) endif docker-setup: - test ! $(docker context ls | grep builder) || docker context create builder; - docker buildx create --platform linux/amd64,linux/arm64 --use builder - docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64 + test ! $($(DOCKER_CMD) context ls | grep builder) || $(DOCKER_CMD) context create builder; + $(DOCKER_CMD) buildx create --platform linux/amd64,linux/arm64 --use builder + $(DOCKER_CMD) run --privileged --rm tonistiigi/binfmt --install amd64,arm64 + +ACTUAL_TAGS=$(shell echo "-t $(REGISTRY)/$(AGENT_IMAGE):$(shell date +%y.%m.%d-%H%M%S)-$(COMMIT_SHORT_SHA) $(TAGS)" | tr A-Z a-z) +.PHONY: docker-build-only +docker-build-only: + @echo "Building skyhook-agent $(DOCKER_CMD) image with tags: $(ACTUAL_TAGS)" + $(DOCKER_CMD) buildx build $(BUILD_ARGS) --build-arg AGENT_VERSION=$(AGENT_VERSION) --platform linux/amd64,linux/arm64 $(ACTUAL_TAGS) --metadata-file=metadata.json -f docker/Dockerfile . ##@ Docker Build .PHONY: docker-build -docker-build docker-setup ## Builds skyhook-agent docker image using docker buildx. - @TAGS="-t $(REGISTRY)/$(AGENT_IMAGE):$(shell date +%y.%m.%d-%H%M%S)-$(COMMIT_SHORT_SHA)" - docker buildx build $(BUILD_ARGS) --platform linux/amd64,linux/arm64 $(TAGS) -f docker/Dockerfile . +docker-build: docker-build-only docker-setup ## Builds skyhook-agent docker image using docker buildx. + @echo "Built skyhook-agent $(DOCKER_CMD) image." ##@ Vendor .PHONY: vendor vendor: ## Uses Unearth to vendor all dependencies locally. - python3 -m venv ./venv_vendor - ./venv_vendor/bin/pip install unearth toml - dependencies=$(shell python -c 'import toml; print(" ".join(toml.loads(open("skyhook-agent/pyproject.toml","r").read())["project"]["dependencies"]))') - rm -rf vendor - mkdir -p vendor - for dep in $(dependencies); do \ - ./venv_vendor/bin/unearth --no-binary -d ./vendor $(dep) >> vendor/lock_file; \ - done + python3 -m venv ./venv_vendor + ./venv_vendor/bin/pip install unearth toml + dependencies=$(shell python -c 'import toml; print(" ".join(toml.loads(open("skyhook-agent/pyproject.toml","r").read())["project"]["dependencies"]))') + rm -rf vendor + mkdir -p vendor + for dep in $(dependencies); do \ + ./venv_vendor/bin/unearth --no-binary -d ./vendor $(dep) >> vendor/lock_file; \ + done ##@ Clean .PHONY: clean diff --git a/agent/docker/Dockerfile b/agent/docker/Dockerfile index 1c30e7d8..1b41b1e6 100644 --- a/agent/docker/Dockerfile +++ b/agent/docker/Dockerfile @@ -1,14 +1,17 @@ -FROM python:3.10-alpine as builder +FROM python:3.12-alpine AS builder -ARG CI_COMMIT_TAG:-0.0.0 +ARG AGENT_VERSION COPY . /code WORKDIR /code -RUN apk add bash -RUN USE_VENV=false /code/cmds.sh setup -RUN USE_VENV=false /code/cmds.sh build ${CI_COMMIT_TAG} +RUN echo "AGENT_VERSION=${AGENT_VERSION}" +RUN apk update && apk add bash make build-base gcc python3-dev musl-dev linux-headers +RUN make test +RUN make clean +RUN make venv +RUN make build build_version=${AGENT_VERSION} -FROM python:3.10-alpine +FROM python:3.12-alpine RUN mkdir -p /skyhook-agent-wheels COPY --from=builder /code/skyhook-agent/dist/* /skyhook-agent-wheels diff --git a/agent/hatch.toml b/agent/hatch.toml index 25ef014e..6aa5f878 100644 --- a/agent/hatch.toml +++ b/agent/hatch.toml @@ -1,3 +1,5 @@ +mode = "local" + [envs.default] dependencies = [ "coverage[toml]", @@ -16,7 +18,7 @@ cov = [ ] [[envs.all.matrix]] -python = ["3.8", "3.9", "3.10"] +python = ["3.12"] [envs.lint] detached = true @@ -40,6 +42,3 @@ all = [ "style", "typing", ] - -[version] -source = "vcs" \ No newline at end of file diff --git a/agent/testing/path b/agent/skyhook-agent/LICENSE.txt similarity index 100% rename from agent/testing/path rename to agent/skyhook-agent/LICENSE.txt diff --git a/agent/skyhook-agent/README.md b/agent/skyhook-agent/README.md new file mode 100644 index 00000000..e69de29b diff --git a/agent/pyproject.toml b/agent/skyhook-agent/pyproject.toml similarity index 90% rename from agent/pyproject.toml rename to agent/skyhook-agent/pyproject.toml index 0b4d3281..3b667583 100644 --- a/agent/pyproject.toml +++ b/agent/skyhook-agent/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling", "hatch-vcs"] +requires = ["hatchling"] build-backend = "hatchling.build" [project] @@ -34,6 +34,7 @@ Source = "https://github.com/nvidia.com/skyhook" [project.scripts] controller = "skyhook_agent.controller:cli" + [tool.hatch.version] path = "src/skyhook_agent/__about__.py" @@ -61,8 +62,8 @@ omit = [ ] [tool.coverage.paths] -skyhook_agent = ["src/skyhook_agent", "*/skyhook-agent/src/skyhook_agent"] -tests = ["tests", "*/skyhook-agent/tests"] +skyhook_agent = ["src/skyhook_agent", "*/src/skyhook_agent"] +tests = ["tests", "*/tests"] [tool.coverage.report] exclude_lines = [ diff --git a/agent/src/skyhook_agent/__about__.py b/agent/skyhook-agent/src/skyhook_agent/__about__.py similarity index 86% rename from agent/src/skyhook_agent/__about__.py rename to agent/skyhook-agent/src/skyhook_agent/__about__.py index e22f573a..2f7883d6 100644 --- a/agent/src/skyhook_agent/__about__.py +++ b/agent/skyhook-agent/src/skyhook_agent/__about__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -__version__ = "0.0.6" +__version__ = "0.0.0" diff --git a/agent/src/skyhook_agent/__init__.py b/agent/skyhook-agent/src/skyhook_agent/__init__.py similarity index 100% rename from agent/src/skyhook_agent/__init__.py rename to agent/skyhook-agent/src/skyhook_agent/__init__.py diff --git a/agent/src/skyhook_agent/config.py b/agent/skyhook-agent/src/skyhook_agent/config.py similarity index 100% rename from agent/src/skyhook_agent/config.py rename to agent/skyhook-agent/src/skyhook_agent/config.py diff --git a/agent/src/skyhook_agent/controller.py b/agent/skyhook-agent/src/skyhook_agent/controller.py similarity index 100% rename from agent/src/skyhook_agent/controller.py rename to agent/skyhook-agent/src/skyhook_agent/controller.py diff --git a/agent/src/skyhook_agent/enums.py b/agent/skyhook-agent/src/skyhook_agent/enums.py similarity index 100% rename from agent/src/skyhook_agent/enums.py rename to agent/skyhook-agent/src/skyhook_agent/enums.py diff --git a/agent/src/skyhook_agent/interrupts.py b/agent/skyhook-agent/src/skyhook_agent/interrupts.py similarity index 100% rename from agent/src/skyhook_agent/interrupts.py rename to agent/skyhook-agent/src/skyhook_agent/interrupts.py diff --git a/agent/src/skyhook_agent/schemas/v1/skyhook-agent-schema.json b/agent/skyhook-agent/src/skyhook_agent/schemas/v1/skyhook-agent-schema.json similarity index 100% rename from agent/src/skyhook_agent/schemas/v1/skyhook-agent-schema.json rename to agent/skyhook-agent/src/skyhook_agent/schemas/v1/skyhook-agent-schema.json diff --git a/agent/src/skyhook_agent/schemas/v1/step-schema.json b/agent/skyhook-agent/src/skyhook_agent/schemas/v1/step-schema.json similarity index 100% rename from agent/src/skyhook_agent/schemas/v1/step-schema.json rename to agent/skyhook-agent/src/skyhook_agent/schemas/v1/step-schema.json diff --git a/agent/src/skyhook_agent/step.py b/agent/skyhook-agent/src/skyhook_agent/step.py similarity index 100% rename from agent/src/skyhook_agent/step.py rename to agent/skyhook-agent/src/skyhook_agent/step.py diff --git a/agent/tests/__init__.py b/agent/skyhook-agent/tests/__init__.py similarity index 100% rename from agent/tests/__init__.py rename to agent/skyhook-agent/tests/__init__.py diff --git a/agent/tests/test_config.py b/agent/skyhook-agent/tests/test_config.py similarity index 100% rename from agent/tests/test_config.py rename to agent/skyhook-agent/tests/test_config.py diff --git a/agent/tests/test_controller.py b/agent/skyhook-agent/tests/test_controller.py similarity index 100% rename from agent/tests/test_controller.py rename to agent/skyhook-agent/tests/test_controller.py diff --git a/agent/tests/test_enums.py b/agent/skyhook-agent/tests/test_enums.py similarity index 100% rename from agent/tests/test_enums.py rename to agent/skyhook-agent/tests/test_enums.py diff --git a/agent/tests/test_interrupts.py b/agent/skyhook-agent/tests/test_interrupts.py similarity index 100% rename from agent/tests/test_interrupts.py rename to agent/skyhook-agent/tests/test_interrupts.py diff --git a/agent/tests/test_steps.py b/agent/skyhook-agent/tests/test_steps.py similarity index 100% rename from agent/tests/test_steps.py rename to agent/skyhook-agent/tests/test_steps.py