feat: enhance release management (enable GitHub release and publish to PyPI)

.github/workflows/publish.yml

@@ -0,0 +1,38 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      release-tag:
+        type: string
+        required: true
+        description: Git tag of the istribution to be published
+jobs:
+  publish-to-pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/compliance-to-policy
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download the distributions from release
+        run: gh release download ${{ github.event.inputs.release-tag }} -D dist -p '*.tar.gz' -p '*.whl'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.REPOSITORY }}
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/release.yml

@@ -0,0 +1,143 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: release
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build with semantic versioning
+    runs-on: ubuntu-latest
+    outputs:
+      release-tag: ${{ steps.release.outputs.tag }}
+      release-version: ${{ steps.release.outputs.version }}
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Python Semantic Release
+        id: release
+        uses: python-semantic-release/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check release
+        if: steps.release.outputs.released == 'false'
+        run: |
+          echo 'No release will be made since there are no release commits. See also Commit Parsers configuration.'
+          exit 1
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install build tools
+        run: |
+          make install-dev
+      - name: Build
+        run: |
+          make build
+      - name: Store the distribution packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-to-github:
+    name: Publish to GitHub
+    needs:
+      - build
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      packages: write
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v3
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Sign the dists with Sigstore
+        uses: sigstore/[email protected]
+        with:
+          inputs: |
+            ./dist/*.tar.gz
+            ./dist/*.whl
+      - name: Upload package distributions to GitHub Releases
+        run: gh release upload ${{needs.build.outputs.release-tag}} ./dist/*
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.REPOSITORY }}
+
+  publish-to-testpypi:
+    name: Publish to TestPyPI
+    needs:
+      - build
+      - publish-to-github
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://pypi.org/p/compliance-to-policy
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download the distributions from release
+        run: gh release download ${{needs.build.outputs.release-tag}} -D dist -p '*.tar.gz' -p '*.whl'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.REPOSITORY }}
+      - name: Publish distribution 📦 to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  test:
+    name: Integration Test
+    needs:
+      - build
+      - publish-to-testpypi
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{needs.build.outputs.release-tag}}
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install
+        run: |
+          version=${{needs.build.outputs.release-version}}
+          version=${version/-rc./rc}
+          count=0
+          while :; do
+            count=$(($count+1))
+            echo "Check if ${version} is available or not ...$count"
+            if pip index versions -i https://test.pypi.org/simple/ compliance-to-policy | grep ${version};then
+              break
+            fi
+            [[ "$count" -gt 5 ]] && echo "Not found ${version}" && exit 1
+            sleep 5
+          done
+          pip index versions -i https://test.pypi.org/simple/ compliance-to-policy | grep ${version}
+          pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple compliance-to-policy==${version}
+          pip install pytest
+      - name: Run test
+        run: make it

.github/workflows/validate.yml

@@ -0,0 +1,47 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: validate
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+    branches:
+      - 'main'
+
+jobs:
+  validate:
+    name: Validate
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with: 
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install for develompemnt
+        run: |
+          make install-dev
+      - name: Unit Test
+        run: |
+          make test
+      - name: Build
+        run: |
+          make build
+      - name: Integration Test
+        run: |
+          pip install ./dist/*.tar.gz
+          make it

Makefile

@@ -1,57 +1,67 @@
-PYTHON := $(shell pwd)/.venv/bin/python
+.PHONY: build
+build:
+	python -m build
 
-.venv:
-	@echo Please create venv firstly
+.PHONY: install
+install:
+	python -m pip install .
 
-build: .venv
-	@$(PYTHON) -m build
+.PHONY: install-dev
+install-dev:
+	python -m pip install ".[dev]"
 
-install: .venv
-	@$(PYTHON) -m pip install .
+# Direct dependency is not allowed for Pypi packaging even if the dependant module is defined as extra dependencies. 
+# Workaround: Move to manual installation by make
+.PHONY: install-detect-descret
+install-detect-descret:
+	python -m pip install detect-secrets@git+https://github.com/ibm/detect-secrets.git@master#egg=detect-secrets
 
-install-dev: .venv
-	@$(PYTHON) -m pip install ".[dev]"
+.PHONY: uninstall
+uninstall:
+	python -m pip uninstall compliance-to-policy
 
-uninstall: .venv
-	@$(PYTHON) -m pip uninstall compliance-to-policy
+.PHONY: format
+format:
+	python -m isort .
+	python -m black .
 
-
-format: .venv
-	@$(PYTHON) -m isort .
-	@$(PYTHON) -m black .
-
-lint: .venv
-	@$(PYTHON) -m pylint ./c2p ./tests
+.PHONY: lint
+lint:
+	python -m pylint ./c2p ./tests
 
 .PHONY: docs
-docs: .venv
-	@$(PYTHON) -m mkdocs build
+docs:
+	python -m mkdocs build
 
 .PHONY: gh-pages
- gh-pages: .venv
-	@$(PYTHON) -m mkdocs gh-deploy
+ gh-pages:
+	python -m mkdocs gh-deploy
 
 # make test ARGS="-n 2 --dist loadscope --log-cli-level DEBUG" TARGET="tests/c2p/test_cli.py"
 # TODO: -n 2 (pytest-xdist plugin) results in no logs displayed.
+.PHONY: test
 test: ARGS ?= 
 test: TARGET ?= tests/
-test: .venv test-plugin
-	@OUTPUT_PATH=/dev/null $(PYTHON) -m pytest $(ARGS) $(TARGET)
+test: test-plugin
+	@OUTPUT_PATH=/dev/null python -m pytest $(ARGS) $(TARGET)
 
+.PHONY: test-plugin
 test-plugin: ARGS ?= 
 test-plugin: TARGET ?= plugins_public/tests/
-test-plugin: .venv
-	@OUTPUT_PATH=/dev/null $(PYTHON) -m pytest $(ARGS) $(TARGET)
-
-# After published, the branch must be merged first-forwardly. TODO: Integrate with CI
-publish: GIT_TAG ?=
-publish:
-	@toml set --toml-path pyproject.toml project.version $(GIT_TAG)
-	@git add pyproject.toml
-	@git commit -S -s -m "update version to $(GIT_TAG)"
-	@git tag $(GIT_TAG)
-
-clean: .venv
+test-plugin:
+	@OUTPUT_PATH=/dev/null python -m pytest $(ARGS) $(TARGET)
+
+.PHONY: it
+it:
+	python samples_public/kyverno/compliance_to_policy.py
+	python samples_public/kyverno/result_to_compliance.py
+	python samples_public/ocm/compliance_to_policy.py
+	python samples_public/ocm/result_to_compliance.py
+	python samples_public/auditree/compliance_to_policy.py
+	python samples_public/auditree/result_to_compliance.py
+
+.PHONY: clean
+clean:
 	@rm -rf build *.egg-info dist
 	@find ./plugins -type d \( -name '*.egg-info' -o -name 'dist' \) | while read x; do echo $$x; rm -r $$x ; done 
-	@$(PYTHON) -m pyclean -v .
+	python -m pyclean -v .

README.md

@@ -1,4 +1,4 @@
-# <img alt="Logo" width="50px" src="./assets/compliance-to-policy-800x800.PNG" style="vertical-align: middle;" /> Compliance-to-Policy (also known as `C2P`)
+# <img alt="Logo" width="50px" src="https://raw.githubusercontent.com/oscal-compass/compliance-to-policy/main/assets/compliance-to-policy-800x800.PNG" style="vertical-align: middle;" /> Compliance-to-Policy (also known as `C2P`)
 
 Compliance-to-Policy (C2P) is designed to bridge Compliance as Code such as Open Security Controls Assessment Language (OSCAL) and Policy as Code used by Policy Validation Point (PVP). It generates policies in native format of PVP from OSCAL Component Definitions and produces OSCAL Assessment Results from the native assessment results of PVP. C2P can be used both as a command-line tool and a Python library, making it easy and flexible to integrate into your Continuous Compliance pipelines, such as GitHub Actions, Tekton Pipelines, or Agile Authoring Pipelines. It supports multiple PVP engines, including [Kyverno](https://kyverno.io/), [Open Cluster Management Policy Framework](https://open-cluster-management.io/), and the open-source [Auditree](https://auditree.github.io/), through dedicated plugins for each. Custom plugins can be implemented with a small amount of Python code.
 

c2p/tools/viewer/viewer.py

@@ -101,7 +101,7 @@ def get_pvp_rule_pair(rule_id):
                     pvp, rule_set = get_pvp_rule_pair(rule_id)
                     if rule_set != None:
                         rule_result = RuleResult(id=f'{rule_id} ({pvp})', description=rule_set['Check_Description'])
-                        o = find_observation(assessment_results.results[0].observations, rule_set['Check_Id'])
+                        o = find_observation(assessment_results.results[0].observations, rule_set['Rule_Id'])
                         if o != None:
                             for subject in o.subjects:
                                 result = get_prop_value(subject.props, 'result')