Python automation script

.github/workflows/pr.yaml → .github/workflows/pr-integration-tests.yaml

@@ -1,40 +1,49 @@
-name: Pull Request - Integration Tests
+name: Integration Tests
 on:
   pull_request:
     types: [opened, edited, synchronize, reopened, ready_for_review]
     branches:
       - "**"
+  push:
+    branches:
+      - main
 
 jobs:
   integration-tests:
     name: "Integration Tests"
     runs-on: ubuntu-latest
     steps:
       #----------------------------------------------
-      #       check-out repo and install poetry
+      #       Check out repo
       #----------------------------------------------
       - name: Check out repository
         uses: actions/checkout@v3
       #----------------------------------------------
+      #       Get docker compose
+      #----------------------------------------------
+      - name: Initialize Docker Compose
+        uses: isbang/[email protected]
+      #----------------------------------------------
       #       Get changed files
       #----------------------------------------------
       - name: Get changed files
         id: changed-files
         uses: tj-actions/changed-files@v40
-      - name: Initialize Docker Compose
-        uses: isbang/[email protected]
       #----------------------------------------------
-      #       Run integration Tests
+      #       Get changed plugins
       #----------------------------------------------
-      - name: Run integration tests
-        id: integration-tests
+      - name: Get changed plugins
+        id: changed-plugins
         run: |
 
-          # This will navigate to each subdirectory and perform integration tests
-          # If we have a directory that isn't a plugin then it will need to skip it. Currently there is none. 
+          # Collects all the plugin names that have changes. 
+          # If there is directory that isn't a plugin then it will need to skip it. Currently skipping plugin_globals
           declare -a changed_dirs=()
           for dir in ./*/; do
             current_folder=$(basename "$dir")
+            if [[ $current_folder == "plugin_globals" ]]; then
+              continue
+            fi
             for changed_file in ${{ steps.changed-files.outputs.all_changed_files }}; do
               if [[ $changed_file == *"$current_folder"* ]]; then
                 if ! [[ ${changed_dirs[*]} =~ $current_folder ]]; then
@@ -44,11 +53,18 @@ jobs:
             done
           done
 
+          echo "changed-plugins=${changed_dirs[*]}" >> $GITHUB_OUTPUT
 
-          for dir in ${changed_dirs[*]}; do
+      #----------------------------------------------
+      #       Get changed plugins
+      #----------------------------------------------
+      - name: Run Integration Tests
+        id: integration-tests
+        run: |
+          for dir in ${{ steps.changed-plugins.outputs.changed-plugins }}; do
             echo "Running integration tests for $dir"
             cd $dir/integration
             docker compose build
-            docker compose up --exit-code-from tests
+            docker compose run tests
             cd ../..
           done

.github/workflows/pr-linting-and-unit-tests.yaml

@@ -0,0 +1,97 @@
+name: Linting and Unit Tests
+on:
+  pull_request:
+    types: [opened, edited, synchronize, reopened, ready_for_review]
+    branches:
+      - "**"
+  push:
+    branches:
+      - main
+
+jobs:
+  linting-and-unit-tests:
+    name: "Code quality and unit tests"
+    runs-on: ubuntu-latest
+    outputs:
+      changed-plugins: ${{ steps.changed-plugins.outputs.cache-hit }}
+    steps:
+      #----------------------------------------------
+      #       Check out repo
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v3
+      #----------------------------------------------
+      #       Install python and poetry with cache
+      #----------------------------------------------
+      - name: Install poetry
+        run: pipx install poetry
+        id: setup-poetry
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+          cache: "poetry"
+      #----------------------------------------------
+      #       Get changed files
+      #----------------------------------------------
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v40
+      #----------------------------------------------
+      #       Get changed plugins
+      #----------------------------------------------
+      - name: Get changed plugins
+        id: changed-plugins
+        run: |
+
+          # Collects all the plugin names that have changes. 
+          # If there is directory that isn't a plugin then it will need to skip it. Currently skipping plugin_globals
+          declare -a changed_dirs=()
+          for dir in ./*/; do
+            current_folder=$(basename "$dir")
+            if [[ $current_folder == "plugin_globals" ]]; then
+              continue
+            fi
+            for changed_file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+              if [[ $changed_file == *"$current_folder"* ]]; then
+                if ! [[ ${changed_dirs[*]} =~ $current_folder ]]; then
+                  changed_dirs+=("$current_folder")
+                fi
+              fi
+            done
+          done
+
+          echo "changed-plugins=${changed_dirs[*]}" >> $GITHUB_OUTPUT
+      #----------------------------------------------
+      #       Install dependencies
+      #----------------------------------------------
+      - name: Install dependencies
+        id: install-dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: |
+          for dir in ${{ steps.changed-plugins.outputs.changed-plugins }}; do
+            cd $dir
+            poetry install --no-interaction --no-root
+            cd ..
+          done
+      #----------------------------------------------
+      #       Lint plugins
+      #----------------------------------------------
+      - name: Lint plugins
+        id: lint-plugins
+        run: |
+          for dir in ${{ steps.changed-plugins.outputs.changed-plugins }}; do
+            cd $dir
+            poetry run ruff check .
+            cd ..
+          done
+      #----------------------------------------------
+      #       Unit tests
+      #----------------------------------------------
+      - name: Unit test plugins
+        id: unit-tests
+        run: |
+          for dir in ${{ steps.changed-plugins.outputs.changed-plugins }}; do
+            cd $dir
+            poetry run pytest
+            cd ..
+          done

README.md

@@ -11,6 +11,16 @@ This repository contains approved and tested plugins for Aries Cloudagent Python
 
 IMPORTANT: docker-in-docker can be a little flaky, so if you encounter a messages such as: "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?" you should probably reboot VS Code.
 
+## Repo Management Script
+
+A script was developed to help with maitenance of the repo called `repo_manager.py`. To run it you need a current version of poetry and python available. 
+Run `python repo_manager.py` and you will be met with 3 options. 
+ - (1) Is used for starting or adding a new plugin. It will generate all the common scaffolding for a plugin which has the expected format. 
+ - (2) Is used for updating and changing common poetry dependencies and configurations. It takes the poetry sections in the `pyproject.toml` files from the `plugin_globals` directory and combines them with the local plugin poetry sections. For the dependencies the common will be overridden by the globals. The other config sections will be replaced by the global configs. Then the lock files will be removed and re-installed. 
+ - (3) Will take common development files like the `.devcontainer` directory from the globals and replace and tag the files. Using this you can make chages to every plugins development files from only editing them in one place.
+
+ IMPORTANT: This script processes the `pyproject.toml` sections by empty lines. Please do not have unnessecary empty lines between sections. 
+
 ## Plugin Documentation
 
 The development team should describe what the plugin does, any limitations (ex only in multitenant mode), any known issues interacting with other plugins, etc. Full documentation including a plugin_config sample should be provided.
@@ -69,27 +79,23 @@ CMD ["aca-py"]
 example config file (local single tenant):
 
 ```
-auto-provision: true
 label: plugins-agent
 
+admin-insecure-mode: true
+admin: [0.0.0.0, 9061]
+
 inbound-transport:
    - [http, 0.0.0.0, 9060]
-
 outbound-transport: http
+endpoint: http://host.docker.internal:9060
+
+genesis-url: https://indy.igrant.io/genesis
 
 emit-new-didcomm-prefix: true
 wallet-type: askar
 wallet-storage-type: default
 
-admin-insecure-mode: true
-
-admin: [0.0.0.0, 9061]
-
-endpoint: http://host.docker.internal:9060
-
-genesis-url: https://indy.igrant.io/genesis
-
-# Connections
+auto-provision: true
 debug-connections: true
 auto-accept-invites: true
 auto-accept-requests: true
@@ -98,7 +104,6 @@ auto-respond-messages: true
 
 log-level: info
 
-# plugins
 plugin:
   - basicmessage_storage.v1_0
   - connection_update.v1_0

basicmessage_storage/.devcontainer/devcontainer.json

@@ -1,58 +1,48 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/python
 {
-    "name": "basicmessage_storage",
-    "build": {
-        "dockerfile": "Dockerfile",
-        "context": "..",
-        "args": {
-          "VARIANT": "3.9-bullseye",
-          "POETRY_VERSION": "1.4.2"
-        }
-    },
-    "customizations": {
-        "vscode": {
-            "extensions": [
-              "ms-python.python",
-              "ms-python.vscode-pylance"
-            ],
-            "settings": {
-                "python.testing.pytestArgs": [
-                    "./basicmessage_storage",
-                    "--no-cov"
-                ],
-                "python.testing.unittestEnabled": false,
-                "python.testing.pytestEnabled": true,
-                "python.testing.pytestPath": "pytest",
-                "editor.defaultFormatter": null,
-                "editor.formatOnSave": false, // enable per language
-                "[python]": {
-                    "editor.formatOnSave": true
-                },
-                "python.formatting.provider": "black",
-                "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-                "python.formatting.blackArgs": []
-            }
-        }
-    },
-
-    "features": {
-        "docker-in-docker": "latest"
-    },
-
-    // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-    "remoteUser": "vscode",
-
-    "remoteEnv": {
-        "RUST_LOG":"aries-askar::log::target=error"
-        //"PATH": "${containerEnv:PATH}:${workspaceRoot}/.venv/bin"
-    },
-
-    "mounts": [],
-    // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    "forwardPorts": [
-        3000,
-        3001
-    ],
-    "postCreateCommand": "bash ./.devcontainer/post-install.sh"
-  }
+  "name": "basicmessage_storage",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": "..",
+    "args": {
+      "VARIANT": "3.9-bullseye",
+      "POETRY_VERSION": "1.4.2"
+    }
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": ["ms-python.python", "ms-python.vscode-pylance"],
+      "settings": {
+        "python.testing.pytestArgs": ["./basicmessage_storage", "--no-cov"],
+        "python.testing.unittestEnabled": false,
+        "python.testing.pytestEnabled": true,
+        "python.testing.pytestPath": "pytest",
+        "editor.defaultFormatter": null,
+        "editor.formatOnSave": false, // enable per language
+        "[python]": {
+          "editor.formatOnSave": true
+        },
+        "python.formatting.provider": "black",
+        "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+        "python.formatting.blackArgs": []
+      }
+    }
+  },
+
+  "features": {
+    "docker-in-docker": "latest"
+  },
+
+  // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+  "remoteUser": "vscode",
+
+  "remoteEnv": {
+    "RUST_LOG": "aries-askar::log::target=error"
+  },
+
+  "mounts": [],
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  "forwardPorts": [3000, 3001],
+  "postCreateCommand": "bash ./.devcontainer/post-install.sh"
+}