Merge branch 'master' of github.com:crim-ca/weaver into job_output_tr…

…ansform
crim-ca · Sep 24, 2024 · d6b8c6c · d6b8c6c
2 parents dcde84e + 81e12e4
commit d6b8c6c
Show file tree

Hide file tree

Showing 210 changed files with 22,086 additions and 3,921 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -15,6 +15,7 @@ downloads
 env
 
 ## Node
+package.json
 package-lock.json
 node_modules
 

diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml
@@ -54,10 +54,22 @@ title = "gitleaks config"
 	regex = '''(?i)(api_key|apikey|secret)(.{0,20})?['|"][0-9a-zA-Z]{16,45}['|"]'''
 	tags = ["key", "API", "generic"]
 	[rules.allowlist]
-        description = "ignore old commit secret (v0.1.0)"
-        commits = ["11cdaf9bb4ffa9eb060ae58dd81268012fd60c28"]
-        paths = ['''magpie/security.py''']
-        regexes = ['''randomsecretstring''']
+		description = "ignore process ID containing secret as plain word"
+		commits = [
+			'6927799b84fac00ec582dbd946031d6547c5a898',
+			'ca1e0b0ac5ee9e0a676f7f29a14648688fcce9de',
+			'b49611bca182a952b7a91c0f56f73433ce444a24',
+			'6927799b84fac00ec582dbd946031d6547c5a898',
+			'ca1e0b0ac5ee9e0a676f7f29a14648688fcce9de',
+			'b49611bca182a952b7a91c0f56f73433ce444a24',
+		]
+		paths = [
+			'''tests/functional/test_workflow.py''',
+		]
+		regexes = [
+			'''EchoSecrets''',
+			'''WorkflowEchoSecrets''',
+		]
 [[rules]]
 	description = "Google API key"
 	regex = '''AIza[0-9A-Za-z\\-_]{35}'''
@@ -107,12 +119,13 @@ title = "gitleaks config"
 	regex = '''(?i)twilio(.{0,20})?['\"][0-9a-f]{32}['\"]'''
 	tags = ["key", "twilio"]
 [allowlist]
-    description = "Allowlisted files"
-    files = [
-    # original contents
-    '''^\.?gitleaks.toml$''',
-    '''(.*?)(jpg|gif|doc|pdf|bin)$''',
-    '''(go.mod|go.sum)$''',
-    # extra ignores
-    '''weaver/wps_restapi/examples/vault_file_uploaded.json''',
-    '''.+(.js.map)$''']
+	description = "Allowlisted files"
+	files = [
+	# original contents
+	'''^\.?gitleaks.toml$''',
+	'''(.*?)(jpg|gif|doc|pdf|bin)$''',
+	'''(go.mod|go.sum)$''',
+	# extra ignores
+	'''weaver/wps_restapi/examples/vault_file_uploaded.json''',
+	'''.+(.js.map)$'''
+	]
diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml
@@ -2,6 +2,12 @@ name: Greetings
 
 on: [pull_request, issues]
 
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   greeting:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml
@@ -1,17 +1,29 @@
 # reference: https://github.com/github/issue-labeler
 
-# FIXME: find a way to avoid the label to remove tags...
-#name: "Issue Labeler"
-#on:
-#  issues:
-#    types: [opened, edited]
-#
-#jobs:
-#  triage:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - uses: github/[email protected]
-#      with:
-#        repo-token: "${{ secrets.GITHUB_TOKEN }}"
-#        configuration-path: .github/labeler-words.yml
-#        enable-versioned-regex: 0
+name: "Issue Labeler"
+on:
+  issues:
+    types: [opened, edited]
+
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: github/[email protected]
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"
+        configuration-path: .github/labeler-words.yml
+        enable-versioned-regex: 0
+        include-title: 1
+        include-body: 1
+        sync-labels: 0
diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -12,6 +12,12 @@ on:
   schedule:
     - cron: '0 */12 * * *'
 
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   original-labeler:
     # reference: https://github.com/actions/labeler

diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml
@@ -14,6 +14,12 @@ on:
   - pull_request
   - push
 
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   #trufflehog:
   #  runs-on: ubuntu-latest

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -7,32 +7,37 @@ on:
   - release
   - workflow_dispatch
 
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   # see: https://github.com/fkirc/skip-duplicate-actions
   skip_duplicate:
     continue-on-error: true
     runs-on: ubuntu-latest
     outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip && ! contains(github.ref, 'refs/tags') }}
+      should_skip: ${{ steps.skip_duplicate.outputs.should_skip && ! contains(github.ref, 'refs/tags') && ! contains(github.ref, 'origin/master') }}
     steps:
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@master
         with:
-          concurrent_skipping: "same_content"
+          concurrent_skipping: "same_content_newer"
           skip_after_successful_duplicate: "true"
-          do_not_skip: '["pull_request", "workflow_dispatch", "schedule", "release"]'
+          do_not_skip: '["workflow_dispatch", "schedule", "release"]'
 
   # see: https://github.com/actions/setup-python
   tests:
-    # FIXME: https://github.com/fkirc/skip-duplicate-actions/issues/90
-    #   disable for now because the tests never run... somehow similar config works in Magpie...
-    # needs: skip_duplicate
-    # if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}
+    needs: skip_duplicate
+    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.allow-failure }}
     env:
       # override make command to install directly in active python
       CONDA_CMD: ""
+      DOCKER_TEST_EXEC_ARGS: "-T"
     services:
       # Label used to access the service container
       mongodb:
@@ -42,26 +47,17 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         allow-failure: [false]
         test-case: [test-unit-only, test-func-only]
         include:
-          # end-of-life python
-          - os: ubuntu-20.04
-            python-version: "3.6"
-            allow-failure: true
-            test-case: test-unit-only
-          - os: ubuntu-20.04
-            python-version: "3.6"
-            allow-failure: true
-            test-case: test-func-only
           # experimental python
           - os: ubuntu-latest
-            python-version: "3.11"
+            python-version: "3.12"
             allow-failure: true
             test-case: test-unit-only
           - os: ubuntu-latest
-            python-version: "3.11"
+            python-version: "3.12"
             allow-failure: true
             test-case: test-func-only
           # linter tests
@@ -96,9 +92,10 @@ jobs:
       - name: Setup Python
         # skip python setup if running with docker
         if: ${{ matrix.test-case != 'test-docker' }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: "${{ matrix.python-version }}"
+          cache: 'pip'
       - name: Parse Python Version
         id: python-semver
         run: |
@@ -116,11 +113,14 @@ jobs:
         if: ${{ matrix.test-case != 'test-docker' }}
         # install package and dependencies directly,
         # skip sys/conda setup to use active python
-        run: make install-sys install-pkg install-pip install-raw install-dev version
+        run: make install-sys install-pkg install-pip install-raw install-dev install-dev-npm
       - name: Display Packages
         # skip python setup if running with docker
         if: ${{ matrix.test-case != 'test-docker' }}
-        run: pip freeze
+        run: |
+          pip freeze
+          npm list
+          make version
       #- name: Setup Environment Variables
       #  uses: c-py/action-dotenv-to-setenv@v2
       #  with:

diff --git a/.gitignore b/.gitignore
@@ -10,7 +10,6 @@ venv
 environment.yml
 
 ## Node
-package.json
 package-lock.json
 node_modules
 

diff --git a/.pylintrc b/.pylintrc
@@ -32,7 +32,8 @@ limit-inference-results=100
 # usually to register additional checkers.
 # https://pylint.pycqa.org/en/latest/technical_reference/extensions.html
 load-plugins=pylint.extensions.docparams,
-             pylint.extensions.mccabe
+             pylint.extensions.mccabe,
+             pylint_per_file_ignores
 
 # https://pylint.pycqa.org/en/latest/technical_reference/extensions.html#design-checker-options
 max-complexity = 24
@@ -94,6 +95,7 @@ disable=C0111,missing-docstring,
         R0401,cyclic-import,
         R0801,duplicate-code,
         R0901,too-many-ancestors,
+        R0902,too-many-instance-attributes,
         R0904,too-many-public-methods,
         R0912,too-many-branches,
         R0914,too-many-locals,
@@ -125,6 +127,9 @@ disable=C0111,missing-docstring,
 
 # note: (C0412, ungrouped-imports) is managed via isort tool, ignore false positives
 
+per-file-ignores =
+  tests/*:R1729
+
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
@@ -376,9 +381,11 @@ class-attribute-naming-style=any
 # Naming style matching correct class names.
 class-naming-style=PascalCase
 
-# Regular expression matching correct class names. Overrides class-naming-
-# style.
-#class-rgx=
+# Regular expression matching correct class names. Overrides class-naming-style.
+# Allow typing definitions that are matched as 'classes' to have slightly more versatile names.
+class-rgx=((_{0,2}[A-Z][a-zA-Z0-9]+)|((CWL|PKG|WPS|OAS|JSON|IO|ANY)_[a-zA-Z0-9_]+Types?))$
+
+typealias-rgx=_{0,2}(?!T[A-Z]|Type)((CWL|PKG|WPS|OAS|JSON|IO|ANY)_)?[A-Z]+[a-z0-9]+(?:[A-Z][a-z0-9]+)*$
 
 # Naming style matching correct constant names.
 const-naming-style=UPPER_CASE
@@ -402,7 +409,7 @@ function-naming-style=snake_case
 good-names=i,j,k,v,kv,ex,x,y,z,f,h,db,kw,dt,q,ns,id,s3,to,_
 
 # Include a hint for the correct naming format with invalid-name.
-include-naming-hint=no
+include-naming-hint=yes
 
 # Naming style matching correct inline iteration names.
 inlinevar-naming-style=any

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -1,13 +1,15 @@
 # configuration to setup readthedocs
 version: 2
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
 sphinx:
   configuration: docs/source/conf.py
 #formats: all
 formats:
   - htmlzip
 python:
-  version: "3.7"
   install:
     - requirements: requirements-sys.txt
     - requirements: requirements-doc.txt
-  system_packages: false
diff --git a/.remarkrc b/.remarkrc
diff --git a/.stylelintrc.json b/.stylelintrc.json
diff --git a/AUTHORS.rst b/AUTHORS.rst
@@ -9,3 +9,4 @@ Contributors
 * David Byrns <[email protected]> @dbyrns
 * David Caron <[email protected]> @davidcaron
 * Mathieu Provencher <[email protected]> @MatProv
+* Misha Schwartz @mishaschwartz