WIP

.devcontainer/devcontainer.json

@@ -0,0 +1,50 @@
+{
+  "name": "Development",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bookworm",
+  "forwardPorts": [3000, 8000],
+  "init": true,
+  "remoteEnv": {
+    "HOMEBREW_AUTO_UPDATE_SECS": "604800"
+  },
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "configureZshAsDefaultShell": true
+    },
+    "ghcr.io/devcontainers/features/azure-cli:1": {},
+    "ghcr.io/jlaundry/devcontainer-features/azure-functions-core-tools:1": {},
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+  },
+  "onCreateCommand": "cd /workspaces/scrape-it-now && python3 -m venv .venv",
+  "updateContentCommand": "cd /workspaces/scrape-it-now && . .venv/bin/activate && make upgrade install",
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "python.defaultInterpreterPath": ".venv/bin/python"
+      },
+      "extensions": [
+        "bierner.markdown-mermaid",
+        "bradlc.vscode-tailwindcss",
+        "DavidAnson.vscode-markdownlint",
+        "EditorConfig.EditorConfig",
+        "github.vscode-github-actions",
+        "mechatroner.rainbow-csv",
+        "mikestead.dotenv",
+        "ms-azuretools.vscode-azurefunctions",
+        "ms-azuretools.vscode-bicep",
+        "ms-python.black-formatter",
+        "ms-python.debugpy",
+        "ms-python.isort",
+        "ms-python.pylint",
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "ms-toolsai.jupyter",
+        "ms-vscode.azurecli",
+        "ms-vscode.makefile-tools",
+        "qwtel.sqlite-viewer",
+        "redhat.vscode-yaml",
+        "visualstudioexptteam.vscodeintellicode",
+        "wholroyd.jinja"
+      ]
+    }
+  }
+}

.editorconfig

@@ -0,0 +1,18 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.py]
+indent_size = 4
+
+[Makefile]
+indent_size = 4
+indent_style = tab

.env.example

@@ -0,0 +1,19 @@
+# OpenAI
+AZURE_OPENAI_API_KEY=xxx
+AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=gpt-4o-2024-05-13
+AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=text-embedding-3-large-1
+AZURE_OPENAI_EMBEDDING_DIMENSIONS=3072
+AZURE_OPENAI_ENDPOINT=https://xxx.openai.azure.com
+OPENAI_API_VERSION=2024-06-01
+
+# AI Search
+AZURE_SEARCH_API_KEY=xxx
+AZURE_SEARCH_ENDPOINT=https://xxx.search.windows.net
+AZURE_SEARCH_INDEX=learn
+
+# Blob Storage
+AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=xxx;AccountKey=xxx;EndpointSuffix=core.windows.net
+AZURE_STORAGE_CONTAINER=learn-scraping
+
+# Application Insights
+APPLICATIONINSIGHTS_CONNECTION_STRING=xxx

.github/dependabot.yml

@@ -0,0 +1,17 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: ""
+    target-branch: develop
+    schedule:
+      interval: weekly
+  - package-ecosystem: pip
+    directory: ""
+    target-branch: develop
+    schedule:
+      interval: weekly
+  - package-ecosystem: "devcontainers"
+    directory: ""
+    target-branch: develop
+    schedule:
+      interval: weekly

.github/workflows/codeql.yml

@@ -0,0 +1,42 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - develop
+      - main
+  pull_request:
+    branches:
+      - develop
+      - main
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    permissions:
+      actions: read
+      contents: read
+      packages: read
+      security-events: write
+    runs-on: ubuntu-22.04
+    timeout-minutes: 360
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - build-mode: none
+            language: python
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          build-mode: ${{ matrix.build-mode }}
+          languages: ${{ matrix.language }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/pipeline.yaml

@@ -0,0 +1,199 @@
+name: Pipeline
+
+on:
+  push:
+    branches:
+      - develop
+      - feat/*
+      - hotfix/*
+      - main
+  pull_request:
+    branches:
+      - develop
+      - feat/*
+      - hotfix/*
+      - main
+
+env:
+  API_CONTAINER_NAME: ${{ github.repository }}/api
+  CONTAINER_REGISTRY_GHCR: ghcr.io
+  CONTAINER_PLATFORMS: linux/amd64,linux/arm64/v8
+  # https://github.com/docker/buildx/releases
+  BUILDX_VERSION: 0.11.2
+
+jobs:
+  init:
+    name: Init
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+    outputs:
+      VERSION: ${{ steps.version.outputs.version }}
+      VERSION_FULL: ${{ steps.version.outputs.version_full }}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          # We need all Git history for "version.sh"
+          fetch-depth: 0
+          # Ensure "version.sh" submodule are up-to-date
+          submodules: recursive
+
+      - name: Generate versions
+        id: version
+        run: |
+          echo "version=$(bash cicd/version/version.sh -g . -c)" >> $GITHUB_OUTPUT
+          echo "version_full=$(bash cicd/version/version.sh -g . -c -m)" >> $GITHUB_OUTPUT
+
+  build-image:
+    name: Build & publish image
+    permissions:
+      contents: write
+      packages: write
+    runs-on: ubuntu-22.04
+    needs:
+      - init
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Configure Git
+        run: |
+          git config user.name "${{ github.actor }}"
+          git config user.email "${{ github.actor }}@users.noreply.github.com"
+
+      - name: Setup QEMU
+        id: setup-qemu
+        uses: docker/[email protected]
+        with:
+          platforms: ${{ env.CONTAINER_PLATFORMS }}
+
+      - name: Setup Docker Buildx
+        uses: docker/[email protected]
+        with:
+          version: v${{ env.BUILDX_VERSION }}
+
+      - name: Login to registry - GitHub
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.CONTAINER_REGISTRY_GHCR }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Container meta
+        id: meta
+        uses: docker/[email protected]
+        with:
+          images: ${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.API_CONTAINER_NAME }}
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=ref,event=branch
+            type=ref,event=pr
+            type=schedule
+            type=schedule,pattern={{date 'YYYYMMDD'}}
+            type=semver,pattern={{version}},value=${{ needs.init.outputs.VERSION_FULL }}
+            type=sha
+          labels: |
+            org.opencontainers.image.documentation=https://github.com/${{ env.API_CONTAINER_NAME }}
+            org.opencontainers.image.vendor=${{ github.actor }}
+
+      - name: Build/push container
+        uses: docker/[email protected]
+        with:
+          build-args: |
+            VERSION=${{ needs.init.outputs.VERSION_FULL }}
+          cache-from: type=gha
+          cache-to: type=gha
+          context: .
+          file: cicd/api.Dockerfile
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: ${{ env.CONTAINER_PLATFORMS }}
+          provenance: true
+          push: true
+          sbom: true
+          tags: ${{ steps.meta.outputs.tags }}
+
+  create-release:
+    name: Create release
+    permissions:
+      contents: write
+    runs-on: ubuntu-22.04
+    needs:
+      - build-image
+      - init
+    outputs:
+      RELEASE_ID: ${{ steps.create-release.outputs.result }}
+    # Only publish on non-scheduled main branch, as there is only one Helm repo and we cannot override an existing version
+    if: (github.event_name != 'schedule') && (github.ref == 'refs/heads/main')
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Create release
+        id: create-release
+        uses: actions/[email protected]
+        with:
+          script: |
+            const isMain = context.ref == `refs/heads/main`;
+            const repoName = context.repo.repo;
+
+            console.log(isMain ? 'Creating release for default branch' : 'Creating release for non-default branch');
+
+            const { data } = await github.rest.repos.createRelease({
+              draft: true,
+              generate_release_notes: true,
+              name: `${repoName} v${{ needs.init.outputs.VERSION }}`,
+              owner: context.repo.owner,
+              prerelease: !isMain,
+              repo: repoName,
+              tag_name: 'v${{ needs.init.outputs.VERSION }}',
+              target_commitish: context.ref,
+            });
+            return data.id
+
+  attest-dependencies:
+    name: Attest - Dependencies
+    permissions:
+      contents: write
+    runs-on: ubuntu-22.04
+    needs:
+      - create-release
+      - init
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Login to registry - GitHub
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.CONTAINER_REGISTRY_GHCR }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run attestation
+        uses: advanced-security/[email protected]
+        with:
+          dockerImagesToScan: ${{ env.CONTAINER_REGISTRY_GHCR }}/${{ env.API_CONTAINER_NAME }}:${{ needs.init.outputs.VERSION_FULL }}
+
+  publish-release:
+    name: Publish release
+    permissions:
+      contents: write
+    runs-on: ubuntu-22.04
+    needs:
+      - attest-dependencies
+      - create-release
+    # Only publish on non-scheduled default branch
+    if: (github.event_name != 'schedule') && (github.ref == 'refs/heads/main')
+    steps:
+      - name: publish release
+        id: publish-release
+        uses: actions/[email protected]
+        with:
+          script: |
+            github.rest.repos.updateRelease({
+              draft: false,
+              owner: context.repo.owner,
+              release_id: ${{ needs.create-release.outputs.RELEASE_ID }},
+              repo: context.repo.repo,
+            });