diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 64225b9d1b..66f15a728f 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -11,9 +11,6 @@ ADD ./{{ item.name }} ./{{ item.name }} # Hacking botocore (https://github.com/boto/botocore/pull/1990) RUN pip3 install -{%- if stream_test | default(false) -%} -{{ " " }}django-storages[sftp] -{%- endif -%} {%- if s3_test | default(false) -%} {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue {%- endif -%} diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 9470aafa0a..024df4e4cf 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -26,18 +26,6 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} -{% if stream_test | default(false) -%} -REDIRECT_TO_OBJECT_STORAGE = False -DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.PulpSFTPStorage" -MEDIA_ROOT = "" -SFTP_STORAGE_HOST = "ci-sftp" -SFTP_STORAGE_ROOT = "/storage/" -SFTP_STORAGE_PARAMS = { - "username": "foo", - "key_filename": "/keys/id_ed25519", -} -{%- endif %} - {% if s3_test | default(false) %} DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" MEDIA_ROOT = "" diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index a05edc0b7f..b19a2054fd 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -15,14 +15,6 @@ - ssh - ~/.config/pulp_smash - - name: Generate an OpenSSH keypair - community.crypto.openssh_keypair: - path: ssh/id_ed25519 - type: ed25519 - owner: 700 # pulp in the container - become: true - when: stream_test | default(false) - - name: "Generate Pulp Settings" template: src: settings.py.j2 diff --git a/.ci/scripts/changelog.py b/.ci/scripts/changelog.py deleted file mode 100755 index ccdfdcfc3a..0000000000 --- a/.ci/scripts/changelog.py +++ /dev/null @@ -1,68 +0,0 @@ -import re -import os -import requests -from packaging.version import Version -from git import Repo - -repo = Repo(os.getcwd()) -heads = repo.git.ls_remote("--heads", "https://github.com/ansible/galaxy_ng.git").split("\n") -branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] -branches.sort(key=lambda ver: Version(ver), reverse=True) - - -def get_changelog(branch): - """ - Get changelog file for a given branch. - - """ - return requests.get( - f"https://raw.githubusercontent.com/ansible/galaxy_ng/{branch}/CHANGES.rst" - ).text - - -def get_changelog_releases(changelog): - """ - Get all versions in changelog. - - """ - versions = re.findall(r"([0-9]+)\.([0-9]+)\.([0-9]+) \(", changelog) - return {".".join(v) for v in versions} - - -def get_changelog_entry(changelog, version): - """ - Get changelog entry for a given version. - - """ - entries = changelog.split(f"{version} (")[1].split("=====\n") - header = f"{version} ({entries[0]}=====\n" - text = "\n\n\n".join(entries[1].split("\n\n\n")[0:-1]) - return header + text + "\n\n\n" - - -main_changelog = get_changelog("master") -main_entries = get_changelog_releases(main_changelog) -entries_list = list(main_entries) -to_add = {} -for branch in branches: - changelog = get_changelog(branch) - entries = get_changelog_releases(changelog) - for entry in entries.difference(main_entries): - description = get_changelog_entry(changelog, entry) - entries_list.append(entry) - print(description) - to_add[entry] = description - -entries_list.sort(key=lambda ver: Version(ver), reverse=True) -for version in sorted(to_add, key=lambda ver: Version(ver)): - next_version = entries_list[entries_list.index(version) + 1] - new_changelog = main_changelog.split(f"{next_version} (")[0] + to_add[version] - new_changelog = new_changelog + f"{next_version} (" - new_changelog = new_changelog + main_changelog.split(f"{next_version} (")[1] - main_changelog = new_changelog - -with open("CHANGES.rst", "w") as f: - f.write(main_changelog) - -if to_add: - repo.git.commit("-m", "Update Changelog\n\n[noissue]", "CHANGES.rst") diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py new file mode 100755 index 0000000000..bf1e6bc02a --- /dev/null +++ b/.ci/scripts/check_release.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import re +import os +import yaml +from tempfile import TemporaryDirectory +from packaging.version import Version +from git import Repo + +UPSTREAM_REMOTE = "https://github.com/ansible/galaxy_ng.git" +DEFAULT_BRANCH = "master" +RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" +Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"] +Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"] + + +def main(): + """Check which branches need a release.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--branches", + default="supported", + help="A comma separated list of branches to check for releases. Can also use keyword: " + "'supported'. Defaults to 'supported', see `ci_update_branches` in " + "`plugin_template.yml`.", + ) + opts = parser.parse_args() + + with TemporaryDirectory() as d: + # Clone from upstream to ensure we have updated branches & main + repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none") + heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")] + available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] + available_branches.sort(key=lambda ver: Version(ver)) + available_branches.append(DEFAULT_BRANCH) + + branches = opts.branches + if branches == "supported": + with open(f"{d}/template_config.yml", mode="r") as f: + tc = yaml.safe_load(f) + branches = tc["ci_update_branches"] + branches.append(DEFAULT_BRANCH) + else: + branches = branches.split(",") + + if diff := set(branches) - set(available_branches): + print(f"Supplied branches contains non-existent branches! {diff}") + exit(1) + + print(f"Checking for releases on branches: {branches}") + + releases = [] + for branch in branches: + if branch != DEFAULT_BRANCH: + # Check if a Z release is needed + changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") + z_release = False + for change in changes.split("\n"): + # Check each changelog file to make sure everything checks out + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + print( + f"Warning: A non-backported changelog ({change}) is present in the " + f"{branch} release branch!" + ) + elif ext in Z_CHANGELOG_EXTS: + z_release = True + if z_release: + # Blobless clone does not have file contents for Z branches, + # check commit message for last Z bump + git_branch = f"origin/{branch}" + next_version = repo.git.log( + "--oneline", "--grep=Bump to", "-n 1", git_branch, "--", ".bumpversion.cfg" + ).split("to")[-1] + next_version = Version(next_version) + print( + f"A Z-release is needed for {branch}, " + f"New Version: {next_version.base_version}" + ) + releases.append(next_version) + else: + # Check if a Y release is needed + changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") + for change in changes.split("\n"): + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + # We don't put Y release bumps in the commit message, check file instead + # The 'current_version' is always the next version to release + next_version = repo.git.grep( + "current_version", DEFAULT_BRANCH, "--", ".bumpversion.cfg" + ).split("=")[-1] + next_version = Version(next_version) + print( + f"A new Y-release is needed! New Version: {next_version.base_version}" + ) + releases.append(next_version) + break + + if len(releases) == 0: + print("No new releases to perform.") + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index c10c2f4096..3826add61f 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -43,7 +43,11 @@ errors.append(f"{filename}:{nr}: Unreadable requirement {line}") else: if check_prereleases and req.specifier.prereleases: - if req.name != "galaxy-ng-client": + # Do not even think about begging for more exceptions! + if ( + not req.name.startswith("opentelemetry") + and req.name != "galaxy-ng-client" + ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") ops = [op for op, ver in req.specs] spec = str(req.specs) diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py new file mode 100755 index 0000000000..d0658c8622 --- /dev/null +++ b/.ci/scripts/collect_changes.py @@ -0,0 +1,101 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import itertools +import os +import re + +import toml +from git import GitCommandError, Repo +from pkg_resources import parse_version + +# Read Towncrier settings +tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"] + +CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") +START_STRING = tc_settings.get( + "start_string", + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n", +) +TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") + + +NAME_REGEX = r".*" +VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)" +DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" +TITLE_REGEX = ( + "(" + + re.escape( + TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX") + ) + .replace("NAME_REGEX", NAME_REGEX) + .replace("VERSION_REGEX", VERSION_REGEX) + .replace("DATE_REGEX", DATE_REGEX) + + ")" +) + + +def get_changelog(repo, branch): + return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n" + + +def _tokenize_changes(splits): + assert len(splits) % 3 == 0 + for i in range(len(splits) // 3): + title = splits[3 * i] + version = parse_version(splits[3 * i + 1]) + yield [version, title + splits[3 * i + 2]] + + +def split_changelog(changelog): + preamble, rest = changelog.split(START_STRING, maxsplit=1) + split_rest = re.split(TITLE_REGEX, rest) + return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:])) + + +def main(): + repo = Repo(os.getcwd()) + remote = repo.remotes[0] + branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)] + branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) + branches = [ref.name for ref in branches] + + with open(CHANGELOG_FILE, "r") as f: + main_changelog = f.read() + preamble, main_changes = split_changelog(main_changelog) + old_length = len(main_changes) + + for branch in branches: + print(f"Looking at branch {branch}") + try: + changelog = get_changelog(repo, branch) + except GitCommandError: + print("No changelog found on this branch.") + continue + dummy, changes = split_changelog(changelog) + new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True) + # Now remove duplicates (retain the first one) + main_changes = [new_changes[0]] + for left, right in itertools.pairwise(new_changes): + if left[0] != right[0]: + main_changes.append(right) + + new_length = len(main_changes) + if old_length < new_length: + print(f"{new_length - old_length} new versions have been added.") + with open(CHANGELOG_FILE, "w") as fp: + fp.write(preamble) + for change in main_changes: + fp.write(change[1]) + + repo.git.commit("-m", "Update Changelog", "-m" "No-Issue", CHANGELOG_FILE) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/update_ci_branches.py b/.ci/scripts/update_ci_branches.py deleted file mode 100755 index 19c9a0192a..0000000000 --- a/.ci/scripts/update_ci_branches.py +++ /dev/null @@ -1,25 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github galaxy_ng' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import os -import sys -import requests - -branches = sys.argv[1:] - -headers = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github.v3+json", -} - -github_api = "https://api.github.com" -workflow_path = "/actions/workflows/update_ci.yml/dispatches" -url = f"{github_api}/repos/ansible/galaxy_ng{workflow_path}" - -for branch in branches: - print(f"Updating {branch}") - requests.post(url, headers=headers, json={"ref": branch}) diff --git a/.ci/scripts/upper_bound.py b/.ci/scripts/upper_bound.py deleted file mode 100755 index 64d4bbbe6a..0000000000 --- a/.ci/scripts/upper_bound.py +++ /dev/null @@ -1,24 +0,0 @@ -import warnings -from pkg_resources import Requirement - -packages = [] - -try: - with open("requirements.txt", "r") as fd: - for line in fd.readlines(): - if not line.startswith("#"): - req = Requirement.parse(line) - spec = str(req.specs) - if "~=" in spec: - warnings.warn(f"Please avoid using ~= on {req.name}") - continue - if len(req.specs) < 2 and "==" not in spec and "<" not in spec: - packages.append(req.name) -except FileNotFoundError: - # skip this test for plugins that don't use a requirements.txt - pass - -if packages: - raise RuntimeError( - "The following packages are missing upper bound: {}".format(", ".join(packages)) - ) diff --git a/.flake8 b/.flake8 index e61888032c..1cf9cd5170 100644 --- a/.flake8 +++ b/.flake8 @@ -6,6 +6,8 @@ # For more info visit https://github.com/pulp/plugin_template [flake8] exclude = ./docs/*,*/migrations/* +per-file-ignores = */__init__.py: F401 + ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402 max-line-length = 100 diff --git a/.github/template_gitref b/.github/template_gitref index 6f6d5c73f3..0459ba4a60 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-208-ge0fb194 +2021.08.26-241-g9cfc63e diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9a7b2b5bc..3dd6448bce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,11 @@ --- name: Galaxy CI on: {pull_request: {branches: ['**']}, push: {branches: ['**']}} + +concurrency: + group: ${{ github.ref_name }}-${{ github.workflow }} + cancel-in-progress: true + jobs: check_commit: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e364fadb88..36f662ae19 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,13 +4,17 @@ # './plugin-template --github galaxy_ng' to update this file. # # For more info visit https://github.com/pulp/plugin_template -name: "CodeQL" +name: "Galaxy CodeQL" on: workflow_dispatch: schedule: - cron: '37 1 * * 6' +concurrency: + group: ${{ github.ref_name }}-${{ github.workflow }} + cancel-in-progress: true + jobs: analyze: name: Analyze diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 37f7afd1b0..19f2ed39c6 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -36,7 +36,7 @@ jobs: - name: Install python dependencies run: | echo ::group::PYDEPS - pip install bump2version + pip install bump2version jinja2 pyyaml echo ::endgroup:: - name: Setting secrets @@ -47,23 +47,27 @@ jobs: - name: Verify that branch name matches current version string on master branch run: | X_Y_VERSION=$(grep version setup.py | sed -rn 's/version="(.*)\.0((a[0-9]+)|(b[0-9]+))?\.dev",/\1/p' | awk '{$1=$1};1') - if [[ "$X_Y_VERSION" != "${{ github.event.inputs.name }}" ]] + if [[ "$X_Y_VERSION" != "${{ inputs.name }}" ]] then echo "Branch name doesn't match the current version string $X_Y_VERSION." exit 1 fi - - name: Create ${{ github.event.inputs.name }} release branch + - name: Create ${{ inputs.name }} release branch run: | - git checkout -b ${{ github.event.inputs.name }} - git push origin ${{ github.event.inputs.name }} + git checkout -b ${{ inputs.name }} + git push origin ${{ inputs.name }} - name: Bump version on master branch run: | git checkout master bump2version --no-commit minor - - name: Make a PR with version bump + - name: Remove entries from CHANGES directory + run: | + find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + + + - name: Make a PR with version bump and without CHANGES/* uses: peter-evans/create-pull-request@v4 with: token: ${{ secrets.RELEASE_TOKEN }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 41c28a975d..a648905261 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Galaxy Nightly CI/CD +name: Galaxy Nightly CI on: schedule: # * is a special character in YAML so you have to quote this string @@ -14,6 +14,10 @@ on: - cron: '00 3 * * *' workflow_dispatch: +concurrency: + group: ${{ github.ref_name }}-${{ github.workflow }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-latest @@ -161,16 +165,16 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: 1 + fetch-depth: 0 - uses: actions/setup-python@v3 with: - python-version: "3.8" + python-version: "3.11" - name: Install python dependencies run: | echo ::group::PYDEPS - pip install gitpython requests packaging + pip install gitpython toml echo ::endgroup:: - name: Configure Git with ansible name and email @@ -178,23 +182,16 @@ jobs: git config --global user.name 'ansible' git config --global user.email 'ansible-infra@redhat.com' - - name: Changelog history - run: python .ci/scripts/changelog.py + - name: Collect changes from all branches + run: python .ci/scripts/collect_changes.py - name: Create Pull Request uses: peter-evans/create-pull-request@v4 with: token: ${{ secrets.RELEASE_TOKEN }} - committer: ansible - author: ansible title: 'Update Changelog' - body: 'No-Issue' + body: '' branch: 'changelog/update' - base: master - commit-message: | - Update Changelog - - No-Issue delete-branch: true publish: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 108293d514..92505047f1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Release Pipeline +name: Galaxy Release Pipeline on: workflow_dispatch: inputs: @@ -15,7 +15,7 @@ on: required: true before_script: description: | - Bash code to run before script.sh is executed. This should only be used when re-running + Bash code to run before bindings and docs are built. This should only be used when re-running a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst required: false @@ -41,7 +41,7 @@ jobs: - name: Install python dependencies run: | echo ::group::PYDEPS - pip install packaging~=21.3 bandersnatch bump2version gitpython towncrier==19.9.0 wheel + pip install packaging~=21.3 bump2version gitpython towncrier==19.9.0 wheel requests echo ::endgroup:: - name: Configure Git with ansible name and email @@ -65,18 +65,14 @@ jobs: with: name: galaxy_ng.tar path: galaxy_ng.tar - test: - needs: build-artifacts - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: azure - - TEST: s3 + build-bindings-docs: + needs: build-artifacts + runs-on: ubuntu-latest + # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests + env: + TEST: docs steps: - uses: actions/download-artifact@v3 @@ -106,10 +102,8 @@ jobs: echo ::endgroup:: echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - + # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to + # setup the Pulp instance. - name: Before Install run: .github/workflows/scripts/before_install.sh shell: bash @@ -140,75 +134,26 @@ jobs: GITHUB_USER: ${{ github.event.pull_request.user.login }} shell: bash - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - name: Additional before_script run: ${{ github.event.inputs.before_script }} shell: bash - - name: Script - if: ${{ env.TEST != 'generate-bindings' }} - run: .github/workflows/scripts/script.sh + - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - GITHUB_USER: ${{ github.event.pull_request.user.login }} - name: Upload python client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} uses: actions/upload-artifact@v3 with: name: python-client.tar path: python-client.tar - name: Upload python client docs - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} uses: actions/upload-artifact@v3 with: name: python-client-docs.tar path: python-client-docs.tar - - name: Upload ruby client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - name: Logs if: always() run: | @@ -224,7 +169,7 @@ jobs: publish: runs-on: ubuntu-latest - needs: test + needs: build-bindings-docs env: TEST: publish @@ -282,25 +227,3 @@ jobs: - name: Create release on GitHub run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} - - - name: Cleanup repository before making changelog PR - run: rm -rf .lock generation galaxy_ng_client* *-client.tar galaxy_ng.tar todo web *docs.tar - - - name: Stage changelog for master branch - run: python .github/workflows/scripts/stage-changelog-for-default-branch.py ${{ github.event.inputs.release }} - - - name: Create Pull Request for Changelog - uses: peter-evans/create-pull-request@v4 - with: - token: ${{ secrets.RELEASE_TOKEN }} - committer: ansible - author: ansible - branch: changelog/${{ github.event.inputs.release }} - base: master - title: 'Cherry pick ${{ github.event.inputs.release }} changelog' - body: '[noissue]' - commit-message: | - ${{ github.event.inputs.release }} changelog - - [noissue] - delete-branch: true diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index b70309a9de..20929d7f28 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -60,9 +60,6 @@ cd .. git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git -pip install pulp-cli - - # Intall requirements for ansible playbooks pip install docker netaddr boto3 ansible diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index e128733cc6..715bde1836 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -36,7 +36,7 @@ tail -v -n +1 .ci/ansible/Containerfile cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" -SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "generate-bindings" "lowerbounds") +SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds") if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then # Many functional tests require these cmd_prefix dnf install -yq lsof which diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 0b407e1091..cbd4fd9dbe 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -17,14 +17,16 @@ source .github/workflows/scripts/utils.sh export PULP_API_ROOT="/api/galaxy/pulp/" -if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then - cd .. - git clone https://github.com/pulp/pulpcore.git - cd - - pip install -r ../pulpcore/doc_requirements.txt - pip install -r doc_requirements.txt +PIP_REQUIREMENTS=("pulp-cli") +if [[ "$TEST" = "docs" || "$TEST" = "publish" ]] +then + PIP_REQUIREMENTS+=("-r" "doc_requirements.txt") fi +pip install ${PIP_REQUIREMENTS[*]} + + + cd .ci/ansible/ if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py index e715444293..521441cbc5 100755 --- a/.github/workflows/scripts/release.py +++ b/.github/workflows/scripts/release.py @@ -6,53 +6,35 @@ # For more info visit https://github.com/pulp/plugin_template import argparse -import asyncio import re import os -import shutil import textwrap - -from bandersnatch.mirror import BandersnatchMirror -from bandersnatch.master import Master -from bandersnatch.configuration import BandersnatchConfig +import requests from git import Repo - -from packaging.requirements import Requirement from pathlib import Path -async def get_package_from_pypi(package_name, plugin_path): +def get_package_from_pypi(version, plugin_path): """ Download a package from PyPI. - :param name: name of the package to download from PyPI - :return: String path to the package + :param version: version of the package to download from PyPI + :return: True/False if download was successful """ - config = BandersnatchConfig().config - config["mirror"]["master"] = "https://pypi.org" - config["mirror"]["workers"] = "1" - config["mirror"]["directory"] = plugin_path - if not config.has_section("plugins"): - config.add_section("plugins") - config["plugins"]["enabled"] = "blocklist_release\n" - if not config.has_section("allowlist"): - config.add_section("allowlist") - config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n" - config["allowlist"]["packages"] = "\n".join([package_name]) os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) - async with Master("https://pypi.org/") as master: - mirror = BandersnatchMirror(homedir=plugin_path, master=master) - name = Requirement(package_name).name - result = await mirror.synchronize([name]) - package_found = False - - for package in result[name]: - current_path = os.path.join(plugin_path, package) - destination_path = os.path.join(plugin_path, "dist", os.path.basename(package)) - shutil.move(current_path, destination_path) - package_found = True - return package_found + r = requests.get(f"https://pypi.org/pypi/galaxy-ng/{version}/json") + if r.status_code == 200: + metadata = r.json() + for url_data in metadata["urls"]: + filename = url_data["filename"] + r2 = requests.get(url_data["url"]) + if r2.status_code != 200: + raise RuntimeError(f"Failed to download released artifact {filename}") + with open(os.path.join(plugin_path, "dist", filename), "wb") as f: + f.write(r2.content) + return True + return False def create_release_commits(repo, release_version, plugin_path): @@ -74,7 +56,7 @@ def create_release_commits(repo, release_version, plugin_path): # Second commit: release version os.system("bump2version release --allow-dirty") - git.add(f"{plugin_path}/{plugin_name}/*") + git.add(f"{plugin_path}/galaxy_ng/*") git.add(f"{plugin_path}/docs/conf.py") git.add(f"{plugin_path}/setup.py") git.add(f"{plugin_path}/requirements.txt") @@ -93,7 +75,7 @@ def create_release_commits(repo, release_version, plugin_path): if not new_dev_version: raise RuntimeError("Could not detect new dev version ... aborting.") - git.add(f"{plugin_path}/{plugin_name}/*") + git.add(f"{plugin_path}/galaxy_ng/*") git.add(f"{plugin_path}/docs/conf.py") git.add(f"{plugin_path}/setup.py") git.add(f"{plugin_path}/requirements.txt") @@ -132,76 +114,76 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): repo.head.reset(index=True, working_tree=True) # Check if Package is available on PyPI - loop = asyncio.get_event_loop() # noqa - # fmt: off - package_found = asyncio.run( - get_package_from_pypi(f"galaxy-ng=={tag.name}", plugin_path) - ) # noqa - # fmt: on - if not package_found: + if not get_package_from_pypi(tag.name, plugin_path): os.system("python3 setup.py sdist bdist_wheel --python-tag py3") -helper = textwrap.dedent( - """\ - Start the release process. +def main(): + helper = textwrap.dedent( + """\ + Start the release process. - Example: - setup.py on plugin before script: - version="2.0.0.dev" + Example: + setup.py on plugin before script: + version="2.0.0.dev" - $ python .ci/scripts/release.py + $ python .ci/scripts/release.py - setup.py on plugin after script: - version="2.0.1.dev" + setup.py on plugin after script: + version="2.0.1.dev" - """ -) -parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) - -parser.add_argument( - "release_version", - type=str, - help="The version string for the release.", -) - -args = parser.parse_args() - -release_version_arg = args.release_version - -release_path = os.path.dirname(os.path.abspath(__file__)) -plugin_path = release_path.split("/.github")[0] - -plugin_name = "galaxy_ng" -version = None -with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - version = re.split("\"|'", line)[1] - if not version: - raise RuntimeError("Could not detect existing version ... aborting.") -release_version = version.replace(".dev", "") - -print(f"\n\nRepo path: {plugin_path}") -repo = Repo(plugin_path) - -release_commit = None -if release_version != release_version_arg: - # Look for a commit with the requested release version - for commit in repo.iter_commits(): - if f"Release {release_version_arg}\n" in commit.message: - release_commit = commit - release_version = release_version_arg - break + """ + ) + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, description=helper + ) + + parser.add_argument( + "release_version", + type=str, + help="The version string for the release.", + ) + + args = parser.parse_args() + + release_version_arg = args.release_version + + release_path = os.path.dirname(os.path.abspath(__file__)) + plugin_path = release_path.split("/.github")[0] + + version = None + with open(f"{plugin_path}/setup.py") as fp: + for line in fp.readlines(): + if "version=" in line: + version = re.split("\"|'", line)[1] + if not version: + raise RuntimeError("Could not detect existing version ... aborting.") + release_version = version.replace(".dev", "") + + print(f"\n\nRepo path: {plugin_path}") + repo = Repo(plugin_path) + + release_commit = None + if release_version != release_version_arg: + # Look for a commit with the requested release version + for commit in repo.iter_commits(): + if f"Release {release_version_arg}\n" in commit.message: + release_commit = commit + release_version = release_version_arg + break + if not release_commit: + raise RuntimeError( + f"The release version {release_version_arg} does not match the .dev version at " + "HEAD. A release commit for such version does not exist." + ) + if not release_commit: - raise RuntimeError( - f"The release version {release_version_arg} does not match the .dev version at HEAD. " - "A release commit for such version does not exist." - ) - -if not release_commit: - release_commit_sha = create_release_commits(repo, release_version, plugin_path) -else: - release_commit_sha = release_commit.hexsha -create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) + release_commit_sha = create_release_commits(repo, release_version, plugin_path) + else: + release_commit_sha = release_commit.hexsha + create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 8cf742245c..b85717ce6f 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -99,7 +99,7 @@ cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" # check for any uncommitted migrations echo "Checking for uncommitted migrations..." -cmd_user_prefix bash -c "django-admin makemigrations --check --dry-run" +cmd_user_prefix bash -c "django-admin makemigrations galaxy --check --dry-run" # Run unit tests. cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs galaxy_ng.tests.unit" diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py new file mode 100755 index 0000000000..e03c20e792 --- /dev/null +++ b/.github/workflows/scripts/update_backport_labels.py @@ -0,0 +1,55 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import requests +import yaml +import random +import os + + +def random_color(): + """Generates a random 24-bit number in hex""" + color = random.randrange(0, 2**24) + return format(color, "06x") + + +session = requests.Session() +token = os.getenv("GITHUB_TOKEN") + +headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +} +session.headers.update(headers) + +# get all labels from the repository's current state +response = session.get("https://api.github.com/repos/pulp/galaxy_ng/labels", headers=headers) +assert response.status_code == 200 +old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")]) + +# get ci_update_branches from template_config.yml +with open("./template_config.yml", "r") as f: + plugin_template = yaml.safe_load(f) +new_labels = set(["backport-" + x for x in plugin_template["ci_update_branches"]]) + +# delete old labels that are not in new labels +for label in old_labels.difference(new_labels): + response = session.delete( + f"https://api.github.com/repos/pulp/galaxy_ng/labels/{label}", headers=headers + ) + assert response.status_code == 204 + +# create new labels that are not in old labels +for label in new_labels.difference(old_labels): + color = random_color() + response = session.post( + "https://api.github.com/repos/pulp/galaxy_ng/labels", + headers=headers, + json={"name": label, "color": color}, + ) + assert response.status_code == 201 diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh deleted file mode 100755 index bb2afe3dca..0000000000 --- a/.github/workflows/scripts/update_ci.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -if [ ! -d ../plugin_template ]; then - echo "Checking out plugin_template" - git clone https://github.com/pulp/plugin_template.git ../plugin_template -fi - - -if [ ! -f "template_config.yml" ]; then - echo "No template_config.yml detected." - exit 1 -fi - -pushd ../plugin_template -pip install -r test_requirements.txt -./plugin-template --github galaxy_ng -popd - -# Check if only gitref file has changed, so no effect on CI workflows. -if [[ `git diff --name-only` == ".github/template_gitref" ]]; then - echo "CI update has no effect on workflows, skipping PR creation." - git stash - exit 0 -fi - -if [[ `git status --porcelain` ]]; then - git add -A - git commit -m "Update CI files" -m "No-Issue" -else - echo "No updates needed" -fi diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml new file mode 100644 index 0000000000..e661811e57 --- /dev/null +++ b/.github/workflows/update-labels.yml @@ -0,0 +1,39 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github galaxy_ng' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + + +--- +name: Galaxy Update Labels +on: + push: + branches: + - main + paths: + - 'template_config.yml' + +jobs: + update_backport_labels: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v3 + with: + python-version: "3.8" + - name: Configure Git with ansible name and email + run: | + git config --global user.name 'ansible' + git config --global user.email 'ansible-infra@redhat.com' + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install requests pyyaml + echo ::endgroup:: + - uses: actions/checkout@v3 + - name: Update labels + run: | + python3 .github/workflows/scripts/update_backport_labels.py + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 62127bdb64..cd59c2cf0f 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -7,14 +7,9 @@ --- -name: CI Update +name: Galaxy CI Update on: workflow_dispatch: - inputs: - all_branches: - description: "Run on all branches" - default: 'no' - required: false jobs: update: @@ -26,7 +21,9 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: 1 + repository: pulp/plugin_template + path: plugin_template + fetch-depth: 0 - uses: actions/setup-python@v3 with: @@ -42,24 +39,28 @@ jobs: run: | git config --global user.name 'ansible' git config --global user.email 'ansible-infra@redhat.com' - - - name: Set short_ref - id: vars - run: echo short_ref=${GITHUB_REF#refs/*/} >> $GITHUB_OUTPUT + - uses: actions/checkout@v3 + with: + path: galaxy_ng + ref: 'master' + fetch-depth: 0 - name: Run update + working-directory: galaxy_ng run: | - .github/workflows/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh - name: Create Pull Request for CI files uses: peter-evans/create-pull-request@v4 with: token: ${{ secrets.RELEASE_TOKEN }} + path: galaxy_ng committer: ansible author: ansible - title: 'Update CI files from ${{ steps.vars.outputs.short_ref }} branch' + title: 'Update CI files for branch master' body: 'No-Issue' - branch: 'create-pull-request/${{ steps.vars.outputs.short_ref }}/patch' + branch: 'update-ci/master' + base: 'master' commit-message: | Update CI files diff --git a/template_config.yml b/template_config.yml index 1bed3cce4c..c74c6273a0 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-208-g15b1f3d +# generated with plugin_template@2021.08.26-241-g9cfc63e additional_repos: - bindings: true @@ -26,6 +26,8 @@ ci_env: GITHUB_USER: ${{ github.event.pull_request.user.login }} ci_trigger: '{pull_request: {branches: [''**'']}, push: {branches: [''**'']}}' ci_update_branches: [] +ci_update_docs: false +ci_update_release_behavior: null cli_package: pulp-cli cli_repo: https://github.com/pulp/pulp-cli.git core_import_allowed: @@ -38,8 +40,10 @@ deploy_client_to_pypi: false deploy_client_to_rubygems: false deploy_to_pypi: false disabled_redis_runners: [] +doc_requirements_from_pulpcore: false docker_fixtures: false docs_test: false +extra_docs_requirements: [] flake8: false flake8_ignore: [] github_org: ansible @@ -98,7 +102,6 @@ test_lowerbounds: false test_performance: false test_reroute: false test_s3: true -test_stream: false update_github: false use_issue_template: false