diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml new file mode 100644 index 0000000..78327b1 --- /dev/null +++ b/.github/workflows/functional.yaml @@ -0,0 +1,25 @@ +name: Functional tests + +on: + workflow_call: + +jobs: + functional_test: + name: Operator functional tests via tox + timeout-minutes: 10 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: v3.11.3 + + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1.9.0 + + - name: Run test + timeout-minutes: 10 + run: tools/functional_test.sh \ No newline at end of file diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..41a1fec --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,25 @@ +name: on push to main + +on: + push: + branches: + - main + +concurrency: + group: main + cancel-in-progress: true + +jobs: + unit_tests: + uses: ./.github/workflows/tox.yaml + + publish_images: + uses: ./.github/workflows/publish-images.yaml + + publish_charts: + needs: [publish_images] + uses: ./.github/workflows/publish-charts.yaml + + functional_tests: + needs: [publish_images] + uses: ./.github/workflows/functional.yaml \ No newline at end of file diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000..001cb01 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,38 @@ +name: on pull request + +on: + pull_request: + types: + - opened + - synchronize + - ready_for_review + - edited + - reopened + branches: + - main + +concurrency: + group: ${{ github.head_ref }} + cancel-in-progress: true + +jobs: + # Run the unit tests on every PR, even from external repos + unit_tests: + uses: ./.github/workflows/tox.yaml + + # When the PR is from a branch of the main repo, publish images and charts + publish_images: + uses: ./.github/workflows/publish-images.yaml + if: github.repository == 'stackhpc/coral-credits' + + publish_charts: + needs: [publish_images] + uses: ./.github/workflows/publish-charts.yaml + if: github.repository == 'stackhpc/coral-credits' + + # The functional tests require the runner image, so we can only run them + # once the image has been built, and on PRs from the main repo + functional_tests: + needs: [publish_images] + uses: ./.github/workflows/functional.yaml + if: github.repository == 'stackhpc/coral-credits' \ No newline at end of file diff --git a/.github/workflows/publish-charts.yaml b/.github/workflows/publish-charts.yaml new file mode 100644 index 0000000..f4219b8 --- /dev/null +++ b/.github/workflows/publish-charts.yaml @@ -0,0 +1,33 @@ +name: Publish charts + +on: + workflow_call: + outputs: + chart-version: + description: The chart version that was published + value: ${{ jobs.publish_charts.outputs.chart-version }} + +jobs: + publish_charts: + name: Publish Helm charts to GitHub pages + runs-on: ubuntu-latest + outputs: + chart-version: ${{ steps.semver.outputs.version }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + # This is important for the semver action to work correctly + # when determining the number of commits since the last tag + fetch-depth: 0 + + - name: Get SemVer version for current commit + id: semver + uses: stackhpc/github-actions/semver@master + + - name: Publish Helm charts + uses: stackhpc/github-actions/helm-publish@master + with: + token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ steps.semver.outputs.version }} + app-version: ${{ steps.semver.outputs.short-sha }} \ No newline at end of file diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yaml similarity index 100% rename from .github/workflows/publish-docs.yml rename to .github/workflows/publish-docs.yaml diff --git a/.github/workflows/publish-images.yaml b/.github/workflows/publish-images.yaml new file mode 100644 index 0000000..96f4664 --- /dev/null +++ b/.github/workflows/publish-images.yaml @@ -0,0 +1,46 @@ +name: Publish images + +on: + workflow_call: + +jobs: + build_push_coral_credits_image: + name: Build and push coral credits image + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # needed for signing the images with GitHub OIDC Token + packages: write # required for pushing container images + security-events: write # required for pushing SARIF files + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Calculate metadata for image + id: image-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/stackhpc/coral-credits + # Produce the branch name or tag and the SHA as tags + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix= + + - name: Build and push image + uses: stackhpc/github-actions/docker-multiarch-build-push@master + with: + cache-key: coral-credits + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.image-meta.outputs.tags }} + labels: ${{ steps.image-meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/tag.yaml b/.github/workflows/tag.yaml new file mode 100644 index 0000000..05888b4 --- /dev/null +++ b/.github/workflows/tag.yaml @@ -0,0 +1,13 @@ +name: on tag + +on: + push: + tags: ['**'] + +jobs: + publish_images: + uses: ./.github/workflows/publish-images.yaml + + publish_charts: + needs: [publish_images] + uses: ./.github/workflows/publish-charts.yaml \ No newline at end of file diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml new file mode 100644 index 0000000..8544b62 --- /dev/null +++ b/.github/workflows/tox.yaml @@ -0,0 +1,38 @@ +name: Tox unit tests + +on: + workflow_call: + +jobs: + build: + name: Tox unit tests and linting + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox + + - name: Test with tox + run: tox + + # TODO(tylerchristie): add unit tests + # - name: Generate coverage reports + # run: tox -e cover + + - name: Archive code coverage results + uses: actions/upload-artifact@v4 + with: + name: code-coverage-report + path: cover/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 71d458f..c33aafa 100644 --- a/.gitignore +++ b/.gitignore @@ -72,5 +72,4 @@ context # don't check in django bits -coral_credits/settings.py db.sqlite3 diff --git a/.stestr.conf b/.stestr.conf index b6665cb..84ca3cc 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,3 +1,3 @@ [DEFAULT] -test_path=./azimuth_caas_operator/tests +test_path=./coral-credits/tests top_dir=./ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..96cb07b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,66 @@ +FROM ubuntu:jammy as build-image + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install --no-install-recommends python3.10-venv git -y && \ + rm -rf /var/lib/apt/lists/* + +# build into a venv we can copy across +RUN python3 -m venv /venv +ENV PATH="/venv/bin:$PATH" + +COPY ./requirements.txt /coral-credits/requirements.txt +RUN pip install -U pip setuptools +RUN pip install --requirement /coral-credits/requirements.txt + +# Django fails to load templates if this is installed the "regular" way +# If we use an editable mode install then it works +COPY . /coral-credits +RUN pip install --no-deps -e /coral-credits + +# +# Now the image we run with +# +FROM ubuntu:jammy as run-image + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install --no-install-recommends python3 tini ca-certificates -y && \ + rm -rf /var/lib/apt/lists/* + +# Copy across the venv +COPY --from=build-image /venv /venv +ENV PATH="/venv/bin:$PATH" + +# Copy across the app +COPY --from=build-image /coral-credits /coral-credits + +# Create the user that will be used to run the app +ENV APP_UID 1001 +ENV APP_GID 1001 +ENV APP_USER app +ENV APP_GROUP app +RUN groupadd --gid $APP_GID $APP_GROUP && \ + useradd \ + --no-create-home \ + --no-user-group \ + --gid $APP_GID \ + --shell /sbin/nologin \ + --uid $APP_UID \ + $APP_USER + +# Don't buffer stdout and stderr as it breaks realtime logging +ENV PYTHONUNBUFFERED 1 + +# Install application configuration using flexi-settings +ENV DJANGO_SETTINGS_MODULE flexi_settings.settings +ENV DJANGO_FLEXI_SETTINGS_ROOT /etc/coral-credits/settings.py +COPY ./etc/coral-credits /etc/coral-credits +RUN mkdir -p /etc/coral-credits/settings.d + +# By default, serve the app on port 8080 using the app user +EXPOSE 8080 +USER $APP_UID +ENTRYPOINT ["tini", "-g", "--"] +#TODO(tylerchristie): use gunicorn + wsgi like azimuth +CMD ["python", "/coral-credits/manage.py", "runserver", "0.0.0.0:8080"] \ No newline at end of file diff --git a/charts/.helmignore b/charts/.helmignore new file mode 100644 index 0000000..691fa13 --- /dev/null +++ b/charts/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/charts/Chart.yaml b/charts/Chart.yaml new file mode 100644 index 0000000..2a9c058 --- /dev/null +++ b/charts/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: coral-credits +description: Helm chart for deploying the Coral Credits API. +type: application +# The version and appVersion are set by the CI script +version: 0.1.0 +appVersion: "main" \ No newline at end of file diff --git a/charts/files/settings/01-django.yaml b/charts/files/settings/01-django.yaml new file mode 100644 index 0000000..91365cf --- /dev/null +++ b/charts/files/settings/01-django.yaml @@ -0,0 +1,19 @@ +SECRET_KEY: {{ .Values.settings.secretKey | default (randAlphaNum 64) }} +DEBUG: {{ .Values.settings.debug }} +DATABASES: + default: + ENGINE: {{ .Values.settings.database.engine | default "django.db.backends.sqlite3" }} + NAME: {{ .Values.settings.database.name | default "/data/db.sqlite3" }} + {{- if .Values.settings.database.user }} + USER: {{ .Values.settings.database.user }} + {{- end }} + {{- if .Values.settings.database.password }} + PASSWORD: {{ .Values.settings.database.password }} + {{- end }} + {{- if .Values.settings.database.host }} + HOST: {{ .Values.settings.database.host }} + {{- end }} + {{- if .Values.settings.database.port }} + PORT: {{ .Values.settings.database.port }} + {{- end }} + diff --git a/charts/templates/_helpers.tpl b/charts/templates/_helpers.tpl new file mode 100644 index 0000000..7d62d6c --- /dev/null +++ b/charts/templates/_helpers.tpl @@ -0,0 +1,53 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "coral-credits.name" -}} +{{- .Chart.Name | lower | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "coral-credits.fullname" -}} +{{- if contains .Chart.Name .Release.Name }} +{{- .Release.Name | lower | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name .Chart.Name | lower | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "coral-credits.chart" -}} +{{- + printf "%s-%s" .Chart.Name .Chart.Version | + replace "+" "_" | + trunc 63 | + trimSuffix "-" | + trimSuffix "." | + trimSuffix "_" +}} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "coral-credits.selectorLabels" -}} +app.kubernetes.io/name: {{ include "coral-credits.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "coral-credits.labels" -}} +helm.sh/chart: {{ include "coral-credits.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{ include "coral-credits.selectorLabels" . }} +{{- end }} \ No newline at end of file diff --git a/charts/templates/deployment.yaml b/charts/templates/deployment.yaml new file mode 100644 index 0000000..ac833af --- /dev/null +++ b/charts/templates/deployment.yaml @@ -0,0 +1,71 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "coral-credits.fullname" . }} + labels: {{ include "coral-credits.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: {{ include "coral-credits.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: {{ include "coral-credits.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml . | nindent 8 }} + {{- end }} + securityContext: {{ toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: migrate-db + image: {{ printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} + command: + - python + - /coral-credits/manage.py + - migrate + volumeMounts: + - name: data + mountPath: /data + - name: runtime-settings + mountPath: /etc/coral-credits/settings.d + readOnly: true + containers: + - name: api + securityContext: {{ toYaml .Values.securityContext | nindent 12 }} + image: {{ printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + #env: + # TODO(tylerchristie): inject stuff here + # TODO(tylerchristie): need metrics at some point + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: {{ toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + - name: runtime-settings + mountPath: /etc/coral-credits/settings.d + readOnly: true + {{- with .Values.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "coral-credits.fullname" . }} + - name: runtime-settings + secret: + secretName: {{ include "coral-credits.fullname" . }} diff --git a/charts/templates/pvc.yaml b/charts/templates/pvc.yaml new file mode 100644 index 0000000..74ff7d1 --- /dev/null +++ b/charts/templates/pvc.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "coral-credits.fullname" . }} + labels: {{ include "coral-credits.labels" . | nindent 4 }} +spec: + accessModes: {{ toYaml .Values.persistence.accessMode | nindent 4 }} + {{- with .Values.persistence.storageClass }} + {{- if (eq "-" .) }} + storageClassName: "" + {{- else }} + storageClassName: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.persistence.volumeBindingMode }} + volumeBindingMode: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- with .Values.persistence.selector }} + selector: {{ toYaml . | nindent 4 }} + {{- end -}} + {{- with .Values.persistence.volumeName }} + volumeName: {{ . }} + {{- end -}} \ No newline at end of file diff --git a/charts/templates/service.yaml b/charts/templates/service.yaml new file mode 100644 index 0000000..1bdff7c --- /dev/null +++ b/charts/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "coral-credits.fullname" . }} + labels: {{ include "coral-credits.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + # TODO(tylerchristie): expose monitoring + selector: {{ include "coral-credits.selectorLabels" . | nindent 6 }} \ No newline at end of file diff --git a/charts/templates/settings.yaml b/charts/templates/settings.yaml new file mode 100644 index 0000000..5ac5a9f --- /dev/null +++ b/charts/templates/settings.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "coral-credits.fullname" . }} + labels: {{ include "coral-credits.labels" . | nindent 4 }} +type: Opaque +# Use data because of https://github.com/helm/helm/issues/10010 +# Not doing so means that AWX-related keys are not removed on transition to the CRD +data: + 01-django.yaml: | + {{- tpl (.Files.Get "files/settings/01-django.yaml") . | b64enc | nindent 4 }} \ No newline at end of file diff --git a/charts/values.yaml b/charts/values.yaml new file mode 100644 index 0000000..9b3756d --- /dev/null +++ b/charts/values.yaml @@ -0,0 +1,111 @@ +# The operator image to use +image: + repository: ghcr.io/stackhpc/coral-credits + pullPolicy: IfNotPresent + tag: "" # Defaults to appVersion + +imagePullSecrets: [] + +# Liveness probe for the operator +livenessProbe: + httpGet: + path: /_status/ + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + +# Startup probe for the operator +startupProbe: + httpGet: + path: /_status/ + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + +# Readiness probe for the operator +readinessProbe: + httpGet: + path: /_status/ + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + initialDelaySeconds: 10 + +# Pod-level security context +podSecurityContext: + runAsNonRoot: true + +# Persistence +persistence: + # The size of the PVC + size: 2Gi + # The access modes for the PVC + accessMode: + - ReadWriteOnce + # The storage class to use for the PVC + # If not given, the default storage class is used + # If set to "-" then storageClassName is set to "", disabling dynamic provisioning + storageClass: + # The volume binding mode for the created PVC + # If not given, the default volume binding mode for the storage class is used + volumeBindingMode: + # The label selector to use to filter eligible PVs + # Useful if PVs have been provisioned in advance + selector: + # The name of a specific PV to bind + # Useful if you want to bind to a specific pre-provisioned PV + volumeName: + + +# Container-level security context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + +# Django settings +settings: + # The Django secret key + # If not given, a randomly generated key will be used + # However this will be different on each deployment which may cause sessions to be terminated + secretKey: + # Use debug mode (recommended false in production) + debug: false + # Database settings + database: + # Database engine (default: django.db.backends.sqlite3) + engine: + # Database name (default: /data/db.sqlite3) + name: + # Database user (optional) + user: + # Database password (optional) + password: + # Database host (optional) + host: + # Database port (optional) + port: + +# Resource requests and limits for the containers +resources: {} + +# replica count +replicaCount: 1 + +# Service details for the api +service: + type: ClusterIP + port: 8080 + +# Node selector for pods +nodeSelector: {} + +# Affinity rules for pods +affinity: {} + +# Tolerations for pods +tolerations: [] diff --git a/coral_credits/api/admin.py b/coral_credits/api/admin.py index a3d670b..6c0395b 100644 --- a/coral_credits/api/admin.py +++ b/coral_credits/api/admin.py @@ -1,4 +1,3 @@ -from auditlog import models as auditlog_models from django.contrib import admin from coral_credits.api import models diff --git a/coral_credits/api/models.py b/coral_credits/api/models.py index 79790e1..01cd4a5 100644 --- a/coral_credits/api/models.py +++ b/coral_credits/api/models.py @@ -1,5 +1,3 @@ -from auditlog.mixins import LogAccessMixin -from auditlog.registry import auditlog from django.db import models # TODO(tylerchristie): add allocation window in here, to simplify. diff --git a/coral_credits/api/serializers.py b/coral_credits/api/serializers.py index 8564b3e..506b303 100644 --- a/coral_credits/api/serializers.py +++ b/coral_credits/api/serializers.py @@ -1,8 +1,6 @@ -from django.contrib import admin from rest_framework import serializers from coral_credits.api import models -from django.contrib.auth.models import User class ResourceClassSerializer(serializers.HyperlinkedModelSerializer): diff --git a/coral_credits/api/tests.py b/coral_credits/api/tests.py index 7ce503c..53f254d 100644 --- a/coral_credits/api/tests.py +++ b/coral_credits/api/tests.py @@ -1,3 +1,11 @@ from django.test import TestCase # Create your tests here. + + +class DBTestDryRun(TestCase): + def setUp(self): + pass + + def test_animals_can_speak(self): + pass diff --git a/coral_credits/api/views.py b/coral_credits/api/views.py index a557de4..47bbb20 100644 --- a/coral_credits/api/views.py +++ b/coral_credits/api/views.py @@ -3,8 +3,7 @@ from rest_framework import permissions, viewsets from rest_framework.response import Response -from coral_credits.api import models -from coral_credits.api import serializers +from coral_credits.api import models, serializers class ResourceClassViewSet(viewsets.ModelViewSet): @@ -21,9 +20,7 @@ class ResourceProviderViewSet(viewsets.ModelViewSet): class AccountViewSet(viewsets.ViewSet): def list(self, request): - """ - List all Credit Accounts - """ + """List all Credit Accounts""" queryset = models.CreditAccount.objects.all() serializer = serializers.CreditAccountSerializer( queryset, many=True, context={"request": request} @@ -31,9 +28,7 @@ def list(self, request): return Response(serializer.data) def retrieve(self, request, pk=None): - """ - Retreives a Credit Account Summary - """ + """Retreives a Credit Account Summary""" queryset = models.CreditAccount.objects.all() account = get_object_or_404(queryset, pk=pk) serializer = serializers.CreditAccountSerializer( @@ -67,7 +62,7 @@ def retrieve(self, request, pk=None): consume_resource = resource_consumer["resource_class"]["name"] if ( resource_allocation["resource_class"]["name"] - == consume_resource + == consume_resource # noqa: W503 ): resource_allocation["resource_hours_remaining"] -= float( resource_consumer["resource_hours"] @@ -76,8 +71,7 @@ def retrieve(self, request, pk=None): return Response(account_summary) def update(self, request, pk=None): - """ - Add a resource request + """Add a resource request Example request:: { @@ -90,7 +84,8 @@ def update(self, request, pk=None): "resource_class": { "name": "CPU" }, - # TODO(tylerchristie): This should just be total amount of resource requested in reservation. + # TODO(tylerchristie): This should just be total amount of + # resource requested in reservation. "resource_hours": 2.0 } ] diff --git a/coral_credits/urls.py b/coral_credits/urls.py index f521338..60e472d 100644 --- a/coral_credits/urls.py +++ b/coral_credits/urls.py @@ -22,7 +22,6 @@ from coral_credits.api import views - router = routers.DefaultRouter() router.register(r"resource_class", views.ResourceClassViewSet) router.register(r"resource_provider", views.ResourceProviderViewSet) diff --git a/etc/coral-credits/app.py b/etc/coral-credits/app.py new file mode 100644 index 0000000..cfcb8da --- /dev/null +++ b/etc/coral-credits/app.py @@ -0,0 +1,64 @@ +""" +Application-specific settings. +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# Database +# https://docs.djangoproject.com/en/5.0/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": "/data/db.sqlite3", + } +} + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django_extensions", + "drf_spectacular", + "rest_framework", + "auditlog", + "coral_credits.api", +] + +AUDITLOG_INCLUDE_ALL_MODELS = True + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +REST_FRAMEWORK = { + "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", +} + +ROOT_URLCONF = "coral_credits.urls" + +SPECTACULAR_SETTINGS = { + "TITLE": "Coral Credits API", + "DESCRIPTION": 'Coral credits is a resource management system that helps build a \ + "coral reef style" fixed capacity cloud, cooperatively sharing community \ + resources through interfaces such as: Azimuth, OpenStack Blazar and Slurm.', + "VERSION": "0.1.0", + "SERVE_INCLUDE_SCHEMA": False, + # OTHER SETTINGS +} + +WSGI_APPLICATION = "coral_credits.wsgi.application" diff --git a/etc/coral-credits/defaults.py b/etc/coral-credits/defaults.py new file mode 100644 index 0000000..fa29ac9 --- /dev/null +++ b/etc/coral-credits/defaults.py @@ -0,0 +1,128 @@ +""" +Django settings for coral_credits project. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/5.0/ref/settings/ +""" + +import logging +import os + +from django.core.management.utils import get_random_secret_key + +# By default, don't run in DEBUG mode +DEBUG = False + +# In a Docker container, ALLOWED_HOSTS is always '*' - let the proxy worry about hosts +ALLOWED_HOSTS = ["*"] + +# Make sure Django interprets the script name correctly if set +if "SCRIPT_NAME" in os.environ: + FORCE_SCRIPT_NAME = os.environ["SCRIPT_NAME"] + +# Set a default random secret key +# This can be overridden by files included later if desired +SECRET_KEY = get_random_secret_key() + +# All logging should go to stdout/stderr to be collected + +LOG_FORMAT = ( + "[%(levelname)s] [%(asctime)s] [%(name)s:%(lineno)s] [%(threadName)s] %(message)s" +) +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "format": LOG_FORMAT, + }, + }, + "filters": { + # Logging filter that only accepts records with a level < WARNING + # This allows us to log level >= WARNING to stderr and level < WARNING to stdout + "less_than_warning": { + "()": "django.utils.log.CallbackFilter", + "callback": lambda record: record.levelno < logging.WARNING, + }, + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "default", + "filters": ["less_than_warning"], + }, + "stderr": { + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr", + "formatter": "default", + "level": "WARNING", + }, + }, + "loggers": { + "": { + "handlers": ["stdout", "stderr"], + "level": "DEBUG" if DEBUG else "INFO", + "propogate": True, + }, + }, +} + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +# Password validation +# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa: E501 + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/5.0/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/5.0/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/etc/coral-credits/settings.py b/etc/coral-credits/settings.py new file mode 100644 index 0000000..6ffff3a --- /dev/null +++ b/etc/coral-credits/settings.py @@ -0,0 +1,19 @@ +""" +Root settings file + +Includes settings files from a sibling directory called settings.d. +""" + +from pathlib import Path + +from flexi_settings import include, include_dir + + +base_dir = Path(__file__).resolve().parent + +# First, include the defaults +include(base_dir / "defaults.py") +# Then include the application-level settings +include(base_dir / "app.py") +# Then include the runtime settings from a directory +include_dir(base_dir / "settings.d") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..14146ab --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "coral_credits" +version = "0.1.0" +description = "Django API for managing resources in a fixed capacity cloud." +readme = "README.md" +authors = [{ name = "Tyler Christie", email = "tyler@stackhpc.com" }] +requires-python = ">=3.10" + +[project.urls] +repository = "https://github.com/stackhpc/coral-credits.git" + +[tool.isort] +profile = "black" +multi_line_output = 3 + +[tool.setuptools.packages.find] +where = ["."] +include = ["coral_credits*"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 979fe0f..232fd73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,8 @@ -django -djangorestframework -django-auditlog +django-auditlog==3.0.0 +Django==5.0.6 +django-flexi-settings @ git+https://github.com/stackhpc/django-flexi-settings.git@079359cc1e2d380a15ae6149ebffbcdae8094276 +django-settings-object @ git+https://github.com/cedadev/django-settings-object.git@2b66c0fc5eae92972df5210b4bc43f7d95ad9ceb +djangorestframework==3.15.2 +django-extensions==3.2.3 +drf-spectacular==0.27.2 +tzdata==2024.1 \ No newline at end of file diff --git a/tilt-component.yaml b/tilt-component.yaml new file mode 100644 index 0000000..df6c501 --- /dev/null +++ b/tilt-component.yaml @@ -0,0 +1,6 @@ +chart: ./charts + +images: + coral-credits: + context: . + chart_path: image diff --git a/tools/functional_test.sh b/tools/functional_test.sh new file mode 100755 index 0000000..04c9651 --- /dev/null +++ b/tools/functional_test.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# Function to check if port is open +check_port() { + nc -z localhost 8080 + return $? +} + +# Function to check HTTP status +check_http_status() { + local status=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/_status/) + if [ "$status" -eq 204 ]; then + return 0 + else + echo "Error: Expected HTTP status code 204, but got $status" + return 1 + fi +} + +# Set variables +CHART_NAME="coral-credits" +RELEASE_NAME=$CHART_NAME +NAMESPACE=$CHART_NAME + +# Install the CaaS operator from the chart we are about to ship +# Make sure to use the images that we just built +helm upgrade $RELEASE_NAME ./charts \ + --dependency-update \ + --namespace $NAMESPACE \ + --create-namespace \ + --install \ + --wait \ + --timeout 10m \ + --set-string image.tag=${GITHUB_SHA::7} + +# Wait for rollout +kubectl rollout status deployment/$RELEASE_NAME -n $NAMESPACE --timeout=300s -w +# Port forward in the background +kubectl port-forward -n $NAMESPACE svc/$RELEASE_NAME 8080:8080 & + +# Wait for port to be open +echo "Waiting for port 8080 to be available..." +for i in {1..30}; do + if check_port; then + echo "Port 8080 is now open" + break + fi + if [ $i -eq 30 ]; then + echo "Timeout waiting for port 8080" + exit 1 + fi + sleep 1 +done + +# Check HTTP status with retries +echo "Checking HTTP status..." +for i in {1..10}; do + if check_http_status; then + echo "Success: HTTP status code is 204." + exit 0 + fi + echo "Attempt $i failed. Retrying in 3 seconds..." + sleep 3 +done + +echo "Failed to get correct HTTP status after 10 attempts" +# Get pod logs on failure + +# Construct the selector +SELECTOR="app.kubernetes.io/name=$CHART_NAME,app.kubernetes.io/instance=$RELEASE_NAME" + +# Get the pod name +POD_NAME=$(kubectl get pods -n $NAMESPACE -l $SELECTOR -o jsonpath="{.items[0].metadata.name}") + +# Get the logs +kubectl logs -n $NAMESPACE $POD_NAME + +exit 1 + +#TODO(tylerchristie) check more things \ No newline at end of file diff --git a/tox.ini b/tox.ini index 180a792..e10a110 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ minversion = 4.0.0 envlist = py3,black,pep8 skipsdist = True + [testenv] basepython = python3.10 usedevelop = True @@ -13,7 +14,8 @@ setenv = OS_TEST_TIMEOUT=60 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = stestr run {posargs} +; Ignore for now as we have no unit tests +; commands = stestr run {posargs} [testenv:pep8] commands = @@ -28,7 +30,7 @@ allowlist_externals = black [testenv:cover] setenv = VIRTUAL_ENV={envdir} - PYTHON=coverage run --source azimuth_caas_operator --parallel-mode + PYTHON=coverage run --source coral-credits --parallel-mode commands = stestr run {posargs} coverage combine @@ -44,7 +46,8 @@ commands = mkdocs build # E123, E125 skipped as they are invalid PEP-8. show-source = True # TODO add headers and remove H102 -ignore = E123,E125,H102 +# H301 skipped as neither isort nor black format like this. +ignore = E123,E125,H102, H301 builtins = _ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build # match black