From a8bcecc038453bccac417ef675b4922d033277ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Rollin?= Date: Tue, 28 Nov 2023 15:57:23 +0100 Subject: [PATCH] Organize CLI by subcmd (#601) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Clean up `manager.py` by re-organizing it by subcmd. ### Details and comments - [x] ci subcmd - [x] tests subcmd - [x] website subcmd - [x] members subcmd - [x] Update docs - [x] Update tests - [x] Update CI/tox --- Closes #230 --------- Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> --- .github/actions/run-tests/action.yml | 10 +- .../ecosystem-badge-and-stars-update.yml | 4 +- .../workflows/ecosystem-batch-repo-check.yml | 14 +- .../ecosystem-batch-save-temp-res-to-db.yml | 2 +- .../workflows/ecosystem-main_repos_fetch.yml | 2 +- .github/workflows/ecosystem-project-check.yml | 2 +- .github/workflows/ecosystem-recompile.yml | 2 +- .github/workflows/ecosystem-submission.yml | 4 +- .pylintrc | 2 +- CONTRIBUTING.md | 6 +- docs/project_overview.md | 16 +- ecosystem/__init__.py | 1 - ecosystem/cli/__init__.py | 5 + ecosystem/cli/ci.py | 121 ++++++++ ecosystem/cli/members.py | 118 ++++++++ ecosystem/{manager.py => cli/tests.py} | 279 +----------------- ecosystem/cli/website.py | 97 ++++++ manager.py | 31 +- tests/{test_manager.py => test_cli.py} | 40 +-- tox.ini | 2 +- 20 files changed, 426 insertions(+), 332 deletions(-) create mode 100644 ecosystem/cli/__init__.py create mode 100644 ecosystem/cli/ci.py create mode 100644 ecosystem/cli/members.py rename ecosystem/{manager.py => cli/tests.py} (62%) create mode 100644 ecosystem/cli/website.py rename tests/{test_manager.py => test_cli.py} (89%) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index bc399392ec..5baa1ce8cc 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -28,7 +28,7 @@ runs: run: | if [ "${{ inputs.test_type }}" == "standard" ]; then echo "Running repository tests for ${{inputs.repo_url}}..." - python manager.py python_standard_tests ${{inputs.repo_url}} \ + python manager.py tests python_standard_tests ${{inputs.repo_url}} \ --run_name=${{inputs.run_name}} \ --python_version=${{inputs.tox_env}} \ --tier=${{ inputs.tier }} \ @@ -36,7 +36,7 @@ runs: fi if [ "${{ inputs.test_type }}" == "stable" ]; then echo "Running Qiskit stable tests for ${{inputs.repo_url}}..." - python manager.py python_stable_tests ${{inputs.repo_url}} \ + python manager.py tests python_stable_tests ${{inputs.repo_url}} \ --run_name=${{inputs.run_name}} \ --python_version=${{inputs.tox_env}} \ --tier=${{ inputs.tier }} \ @@ -44,7 +44,7 @@ runs: fi if [ "${{ inputs.test_type }}" == "development" ]; then echo "Running Qiskit dev tests for ${{inputs.repo_url}}..." - python manager.py python_dev_tests ${{inputs.repo_url}} \ + python manager.py tests python_dev_tests ${{inputs.repo_url}} \ --run_name=${{inputs.run_name}} \ --python_version=${{inputs.tox_env}} \ --tier=${{ inputs.tier }} \ @@ -52,11 +52,11 @@ runs: fi if [ "${{ inputs.test_type }}" == "lint" ]; then echo "Running lint tests for ${{inputs.repo_url}}..." - python manager.py python_styles_check ${{inputs.repo_url}} --run_name=${{inputs.run_name}} --tier=${{inputs.tier}} --style_type="pylint" + python manager.py tests python_styles_check ${{inputs.repo_url}} --run_name=${{inputs.run_name}} --tier=${{inputs.tier}} --style_type="pylint" fi if [ "${{ inputs.test_type }}" == "coverage" ]; then echo "Running coverage tests for ${{inputs.repo_url}}..." - python manager.py python_coverage ${{inputs.repo_url}} --run_name=${{inputs.run_name}} --tier=${{inputs.tier}} --coverage_type="" + python manager.py tests python_coverage ${{inputs.repo_url}} --run_name=${{inputs.run_name}} --tier=${{inputs.tier}} --coverage_type="" fi shell: bash outputs: diff --git a/.github/workflows/ecosystem-badge-and-stars-update.yml b/.github/workflows/ecosystem-badge-and-stars-update.yml index 22c14d7bd8..943fb08462 100644 --- a/.github/workflows/ecosystem-badge-and-stars-update.yml +++ b/.github/workflows/ecosystem-badge-and-stars-update.yml @@ -27,10 +27,10 @@ jobs: pip install -r requirements.txt - name: Update badges - run: python manager.py update_badges + run: python manager.py members update_badges - name: Update stars - run: python manager.py update_stars + run: python manager.py members update_stars - name: Create PR for stars and badges update id: cpr diff --git a/.github/workflows/ecosystem-batch-repo-check.yml b/.github/workflows/ecosystem-batch-repo-check.yml index 2d955a57fd..98b159ebf9 100644 --- a/.github/workflows/ecosystem-batch-repo-check.yml +++ b/.github/workflows/ecosystem-batch-repo-check.yml @@ -37,7 +37,7 @@ jobs: branch: ${{ steps.vars.outputs.pr_branch_name }} - name: Get project for check id: repos - run: python manager.py expose_all_project_to_actions + run: python manager.py ci expose_all_project_to_actions - name: Launch separate workflows for each project check run: | IFS=',' read -r -a repositories <<< ${{ steps.repos.outputs.repositories }} @@ -45,10 +45,10 @@ jobs: for (( i=0; i<${#repositories[*]}; ++i)); do echo "Launching separate workflow for ${repositories[$i]} in ${tiers[$i]} tier" - python manager.py dispatch_check_workflow \ - --repo_url=${repositories[$i]} \ - --branch_name=${{ steps.vars.outputs.pr_branch_name }} \ - --tier=${tiers[$i]} \ - --token=${{ secrets.GITHUB_TOKEN }} \ - --owner=${{ github.repository_owner }} + python manager.py ci dispatch_check_workflow \ + --repo_url=${repositories[$i]} \ + --branch_name=${{ steps.vars.outputs.pr_branch_name }} \ + --tier=${tiers[$i]} \ + --token=${{ secrets.GITHUB_TOKEN }} \ + --owner=${{ github.repository_owner }} done diff --git a/.github/workflows/ecosystem-batch-save-temp-res-to-db.yml b/.github/workflows/ecosystem-batch-save-temp-res-to-db.yml index 35a9d761ed..9e44b0bfe4 100644 --- a/.github/workflows/ecosystem-batch-save-temp-res-to-db.yml +++ b/.github/workflows/ecosystem-batch-save-temp-res-to-db.yml @@ -26,7 +26,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - name: Merge files and save to db - run: python manager.py process_temp_test_results_files --folder_name=${{ github.event.inputs.batch_folder }} + run: python manager.py tests process_temp_test_results_files --folder_name=${{ github.event.inputs.batch_folder }} - name: Commit changes run: | git pull --rebase --autostash diff --git a/.github/workflows/ecosystem-main_repos_fetch.yml b/.github/workflows/ecosystem-main_repos_fetch.yml index 45ed2f9c51..5f2ad3f85d 100644 --- a/.github/workflows/ecosystem-main_repos_fetch.yml +++ b/.github/workflows/ecosystem-main_repos_fetch.yml @@ -29,7 +29,7 @@ jobs: pip install -r requirements-dev.txt - name: Fetch test results - run: python manager.py fetch_and_update_main_tests_results + run: python manager.py tests fetch_and_update_main_tests_results - name: Create PR for batch checks id: cpr diff --git a/.github/workflows/ecosystem-project-check.yml b/.github/workflows/ecosystem-project-check.yml index 164af5bfaa..0a2b18b395 100644 --- a/.github/workflows/ecosystem-project-check.yml +++ b/.github/workflows/ecosystem-project-check.yml @@ -67,7 +67,7 @@ jobs: logs_link: https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }} - name: Merge files and save to db - run: python manager.py process_temp_test_results_files --folder_name=${{ github.event.client_payload.branch_name }} + run: python manager.py tests process_temp_test_results_files --folder_name=${{ github.event.client_payload.branch_name }} - name: Commit changes run: | diff --git a/.github/workflows/ecosystem-recompile.yml b/.github/workflows/ecosystem-recompile.yml index 1acdee4091..5faa0d0c03 100644 --- a/.github/workflows/ecosystem-recompile.yml +++ b/.github/workflows/ecosystem-recompile.yml @@ -37,7 +37,7 @@ jobs: - name: Recompile and push run: | - python -m manager recompile + python -m manager members recompile git config user.name github-actions git config user.email github-actions@github.com diff --git a/.github/workflows/ecosystem-submission.yml b/.github/workflows/ecosystem-submission.yml index 65cb6a2142..0d04293e24 100644 --- a/.github/workflows/ecosystem-submission.yml +++ b/.github/workflows/ecosystem-submission.yml @@ -40,7 +40,7 @@ jobs: id: parse-issue env: ISSUE_BODY: ${{ github.event.issue.body }} - run: python manager.py parser_issue --body="$ISSUE_BODY" + run: python manager.py ci parser_issue --body="$ISSUE_BODY" - name: Tests stable check id: stable uses: ./.github/actions/run-tests @@ -81,7 +81,7 @@ jobs: SUBMISSION_LABELS: ${{ steps.parse-issue.outputs.SUBMISSION_LABELS }} SUBMISSION_WEBSITE: ${{ steps.parse-issue.outputs.SUBMISSION_WEBSITE }} run: | - python manager.py add_repo_2db --repo_name="$SUBMISSION_NAME" \ + python manager.py members add_repo_2db --repo_name="$SUBMISSION_NAME" \ --repo_link="$SUBMISSION_REPO" \ --repo_description="$SUBMISSION_DESCRIPTION" \ --repo_licence="$SUBMISSION_LICENCE" \ diff --git a/.pylintrc b/.pylintrc index 1e00ce80f5..7adef2b7d4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -612,7 +612,7 @@ max-returns=6 max-statements=50 # Minimum number of public methods for a class (see R0903). -min-public-methods=2 +min-public-methods=1 [EXCEPTIONS] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b4251d1d1c..d2f7688faa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,11 +25,11 @@ pip install -r requirements-dev.txt # Running the tests 1. To run tests against the stable version of qiskit
- python manager.py python_stable_tests --python_version=py39 --run_name="stable" + python manager.py tests python_stable_tests --python_version=py39 --run_name="stable" 2. To run tests against the dev version of qiskit
- python manager.py python_dev_tests --python_version=py39 --run_name="dev" + python manager.py tests python_dev_tests --python_version=py39 --run_name="dev" 3. To run tests within repository
- python manager.py python_standard_tests --python_version=py39 --run_name="standard" + python manager.py tests python_standard_tests --python_version=py39 --run_name="standard" # Performing style checks - Run for style checks diff --git a/docs/project_overview.md b/docs/project_overview.md index 6c72b79df6..4990e35391 100644 --- a/docs/project_overview.md +++ b/docs/project_overview.md @@ -31,12 +31,12 @@ Entrypoint is ``manager.py`` file in the root of repository. Example of commands: ```shell -python manager.py python_dev_tests https://github.com/IceKhan13/demo-implementation --python_version=py39 -python manager.py python_stable_tests https://github.com/IceKhan13/demo-implementation --python_version=py39 +python manager.py tests python_dev_tests https://github.com/IceKhan13/demo-implementation --python_version=py39 +python manager.py tests python_stable_tests https://github.com/IceKhan13/demo-implementation --python_version=py39 ``` or in general ```shell -python manager.py [FLAGS] +python manager.py [FLAGS] ``` ### Adding project to the ecosystem @@ -119,13 +119,13 @@ file. This file should be compiled automatically by an action on pushing to `main`, but you can also compile it locally (e.g. for testing) using ```sh -python -m manager recompile +python -m manager members recompile ``` You shouldn't edit `members.json` manually. If you somehow get a merge conflict in `members.json`, don't try to manually -resolve the conflict. Instead, merge the branch, then run `python -m manager +resolve the conflict. Instead, merge the branch, then run `python -m manager members recompile` and add the file to resolve the conflict. ### Tests @@ -136,21 +136,21 @@ There are 3 type of tests for project: `STANDARD`, `DEV` and `STABLE`. CLI command: ```shell -python manager.py python_standard_tests --run_name= --python_version=py39 --tier= +python manager.py tests python_standard_tests --run_name= --python_version=py39 --tier= ``` `DEV` - runs tests with default requirements for project + dev version of qiskit-terra installed CLI command: ```shell -python manager.py python_dev_tests --run_name= --python_version=py39 --tier= +python manager.py tests python_dev_tests --run_name= --python_version=py39 --tier= ``` `STABLE` - runs tests with default requirements for project + latest stable version of qiskit-terra installed CLI command: ```shell -python manager.py python_stable_tests --run_name= --python_version=py39 --tier= +python manager.py tests python_stable_tests --run_name= --python_version=py39 --tier= ``` You can see full setup on test running in [GitHub Action](https://github.com/qiskit-community/ecosystem/blob/main/.github/actions/run-tests/action.yml) diff --git a/ecosystem/__init__.py b/ecosystem/__init__.py index f0697174d5..a31b58d0ca 100644 --- a/ecosystem/__init__.py +++ b/ecosystem/__init__.py @@ -1,2 +1 @@ """Ecosystem main module.""" -from .manager import Manager diff --git a/ecosystem/cli/__init__.py b/ecosystem/cli/__init__.py new file mode 100644 index 0000000000..ac018e0b0e --- /dev/null +++ b/ecosystem/cli/__init__.py @@ -0,0 +1,5 @@ +"""CLI.""" +from .members import CliMembers +from .website import CliWebsite +from .ci import CliCI +from .tests import CliTests diff --git a/ecosystem/cli/ci.py b/ecosystem/cli/ci.py new file mode 100644 index 0000000000..432ee48321 --- /dev/null +++ b/ecosystem/cli/ci.py @@ -0,0 +1,121 @@ +"""CliCI class for controlling all CLI functions.""" +import os +from typing import Optional + +import requests + +from ecosystem.daos import DAO +from ecosystem.models import Tier +from ecosystem.utils import logger, parse_submission_issue +from ecosystem.utils.utils import set_actions_output + + +class CliCI: + """CliCI class. + Entrypoint for all CLI CI commands. + + Each public method of this class is CLI command + and arguments for method are options/flags for this command. + + Ex: `python manager.py ci parser_issue --body=""` + """ + + def __init__(self, root_path: Optional[str] = None): + """CliCI class.""" + self.current_dir = root_path or os.path.abspath(os.getcwd()) + self.resources_dir = "{}/ecosystem/resources".format(self.current_dir) + self.dao = DAO(path=self.resources_dir) + self.logger = logger + + def dispatch_check_workflow( + self, + repo_url: str, + branch_name: str, + tier: str, + token: str, + owner: str = "qiskit-community", + repo: str = "ecosystem", + ) -> bool: + """Dispatch event to trigger check workflow. + + Args: + repo_url: url of the repo + branch_name: name of the branch + tier: tier of the project + token: token base on the date + owner: "qiskit-community" parameters + repo: "ecosystem" + + Return: true + """ + url = "https://api.github.com/repos/{owner}/{repo}/dispatches".format( + owner=owner, repo=repo + ) + repo_split = repo_url.split("/") + repo_name = repo_split[-1] + + # run each type of tests in same workflow + response = requests.post( + url, + json={ + "event_type": "check_project", + "client_payload": { + "repo_url": repo_url, + "repo_name": repo_name, + "branch_name": branch_name, + "tier": tier, + }, + }, + headers={ + "Authorization": "token {}".format(token), + "Accept": "application/vnd.github.v3+json", + }, + ) + if response.ok: + self.logger.info("Success response on dispatch event. %s", response.text) + else: + self.logger.warning( + "Something wend wrong with dispatch event: %s", response.text + ) + return True + + def expose_all_project_to_actions(self): + """Exposes all project for github actions.""" + repositories = [] + tiers = [] + for tier in Tier.non_main_tiers(): + for repo in self.dao.get_repos_by_tier(tier): + if not repo.skip_tests: + repositories.append(repo.url) + tiers.append(repo.tier) + set_actions_output( + [("repositories", ",".join(repositories)), ("tiers", ",".join(tiers))] + ) + + @staticmethod + def parser_issue(body: str) -> None: + """Command for calling body issue parsing function. + + Args: + body: body of the created issue + + Returns: + logs output + We want to give the result of the parsing issue to the GitHub action + """ + + parsed_result = parse_submission_issue(body) + + to_print = [ + ("SUBMISSION_NAME", parsed_result.name), + ("SUBMISSION_REPO", parsed_result.url), + ("SUBMISSION_DESCRIPTION", parsed_result.description), + ("SUBMISSION_LICENCE", parsed_result.licence), + ("SUBMISSION_CONTACT", parsed_result.contact_info), + ("SUBMISSION_ALTERNATIVES", parsed_result.alternatives), + ("SUBMISSION_AFFILIATIONS", parsed_result.affiliations), + ("SUBMISSION_LABELS", parsed_result.labels), + ("SUBMISSION_WEBSITE", parsed_result.website), + ] + + set_actions_output(to_print) diff --git a/ecosystem/cli/members.py b/ecosystem/cli/members.py new file mode 100644 index 0000000000..2e37cb352d --- /dev/null +++ b/ecosystem/cli/members.py @@ -0,0 +1,118 @@ +"""CliMembers class for controlling all CLI functions.""" +import json +import os +from typing import Optional, Tuple + +import requests + +from ecosystem.daos import DAO +from ecosystem.models import Tier +from ecosystem.models.repository import Repository +from ecosystem.utils import logger + + +class CliMembers: + """CliMembers class. + Entrypoint for all CLI members commands. + + Each public method of this class is CLI command + and arguments for method are options/flags for this command. + + Ex: `python manager.py members recompile --body=""` + """ + + def __init__(self, root_path: Optional[str] = None): + """CliMembers class.""" + self.current_dir = root_path or os.path.abspath(os.getcwd()) + self.resources_dir = "{}/ecosystem/resources".format(self.current_dir) + self.dao = DAO(path=self.resources_dir) + self.logger = logger + + def recompile(self): + """Recompile `members.json` from human-readable files.""" + self.dao.compile_json() + + def add_repo_2db( + self, + repo_name: str, + repo_link: str, + repo_description: str, + repo_licence: str, + repo_contact: str, + repo_alt: str, + repo_affiliations: str, + repo_labels: Tuple[str], + repo_tier: Optional[str] = None, + repo_website: Optional[str] = None, + ) -> None: + """Adds repo to list of entries. + + Args: + repo_name: repo name + repo_link: repo url + repo_description: repo description + repo_contact: repo email + repo_alt: repo alternatives + repo_licence: repo licence + repo_affiliations: repo university, company, ... + repo_labels: comma separated labels + repo_tier: tier for repository + repo_website: link to project website + """ + + new_repo = Repository( + repo_name, + repo_link, + repo_description, + repo_licence, + repo_contact, + repo_alt, + repo_affiliations, + list(repo_labels), + tier=repo_tier or Tier.COMMUNITY, + website=repo_website, + ) + self.dao.write(new_repo) + + def update_badges(self): + """Updates badges for projects.""" + badges_folder_path = "{}/badges".format(self.current_dir) + + for tier in Tier.all(): + for project in self.dao.get_repos_by_tier(tier): + tests_passed = True + for type_test in project.tests_results: + if type_test.test_type == "standard" and not type_test.passed: + tests_passed = False + color = "blueviolet" if tests_passed else "gray" + label = project.name + message = tier + url = ( + f"https://img.shields.io/static/v1?" + f"label={label}&message={message}&color={color}" + ) + + shields_request = requests.get(url) + with open(f"{badges_folder_path}/{project.name}.svg", "wb") as outfile: + outfile.write(shields_request.content) + self.logger.info("Badge for %s has been updated.", project.name) + + def update_stars(self): + """Updates start for repositories.""" + for tier in Tier.all(): + for project in self.dao.get_repos_by_tier(tier): + stars = None + url = project.url[:-1] if project.url[-1] == "/" else project.url + url_chunks = url.split("/") + repo = url_chunks[-1] + user = url_chunks[-2] + + response = requests.get(f"http://api.github.com/repos/{user}/{repo}") + if not response.ok: + self.logger.warning("Bad response for project %s", project.url) + continue + + json_data = json.loads(response.text) + stars = json_data.get("stargazers_count") + self.dao.update(project.url, stars=stars) + self.logger.info("Updating star count for %s: %d", project.url, stars) diff --git a/ecosystem/manager.py b/ecosystem/cli/tests.py similarity index 62% rename from ecosystem/manager.py rename to ecosystem/cli/tests.py index aa53d4f4c8..90ec8adc12 100644 --- a/ecosystem/manager.py +++ b/ecosystem/cli/tests.py @@ -1,23 +1,21 @@ -"""Manager class for controlling all CLI functions.""" +"""CliTests class for controlling all CLI functions.""" import glob import json import os import shutil import uuid -from typing import Optional, List, Tuple, Union +from typing import Optional, List, Union import requests -from jinja2 import Environment, PackageLoader, select_autoescape, FileSystemLoader from ecosystem.daos import DAO from ecosystem.models import TestResult, Tier, TestType -from ecosystem.models.repository import Repository from ecosystem.models.test_results import StyleResult, CoverageResult, Package from ecosystem.runners import PythonTestsRunner from ecosystem.runners.main_repos_report_runner import RepositoryActionStatusRunner from ecosystem.runners.python_styles_runner import PythonStyleRunner from ecosystem.runners.python_coverages_runner import PythonCoverageRunner -from ecosystem.utils import logger, parse_submission_issue +from ecosystem.utils import logger from ecosystem.utils.custom_requests import ( get_dev_qiskit_version, get_stable_qiskit_version, @@ -25,284 +23,23 @@ from ecosystem.utils.utils import set_actions_output -class Manager: - """Manager class. - Entrypoint for all CLI commands. +class CliTests: + """CliTests class. + Entrypoint for all CLI tests commands. Each public method of this class is CLI command and arguments for method are options/flags for this command. - Ex: `python manager.py parser_issue --body=""` + Ex: `python manager.py tests python_stable_tests --body=""` """ def __init__(self, root_path: Optional[str] = None): - """Manager class.""" + """CliTests class.""" self.current_dir = root_path or os.path.abspath(os.getcwd()) self.resources_dir = "{}/ecosystem/resources".format(self.current_dir) - - self.env = Environment( - loader=PackageLoader("ecosystem"), autoescape=select_autoescape() - ) - self.pylintrc_template = self.env.get_template(".pylintrc") - self.coveragerc_template = self.env.get_template(".coveragerc") self.dao = DAO(path=self.resources_dir) self.logger = logger - def recompile(self): - """Recompile `members.json` from human-readable files.""" - self.dao.compile_json() - - def build_website(self): - """Generates the ecosystem web page reading `members.json`.""" - environment = Environment(loader=FileSystemLoader("ecosystem/html_templates/")) - projects = self.dao.storage.read() - projects_sorted = sorted( - projects.items(), - key=lambda item: ( - -item[1].stars if item[1].stars is not None else 0, - item[1].name, - ), - ) - templates = { - "website": environment.get_template("webpage.html.jinja"), - "card": environment.get_template("card.html.jinja"), - "tag": environment.get_template("tag.html.jinja"), - "link": environment.get_template("link.html.jinja"), - } - sections = { - "transpiler_plugin": "", - "provider": "", - "applications": "", - "other": "", - } - - max_chars_description_visible = 400 - min_chars_description_hidden = 100 - count_read_more = 1 - for _, repo in projects_sorted: - # Card tags - tags = "" - for label in repo.labels: - tags += templates["tag"].render(color="purple", title=label, text=label) - - # Card links - links = templates["link"].render(url=repo.url, place="repository") - if repo.website: - links += templates["link"].render(url=repo.website, place="website") - - # Card description - if ( - len(repo.description) - max_chars_description_visible - >= min_chars_description_hidden - ): - description = [ - repo.description[:max_chars_description_visible], - repo.description[max_chars_description_visible:], - ] - id_read_more = str(count_read_more) - count_read_more += 1 - else: - description = [repo.description, ""] - id_read_more = "None" - - # Create the card - card = templates["card"].render( - title=repo.name, - tags=tags, - description_visible=description[0], - description_hidden=description[1], - id_read_more=id_read_more, - links=links, - ) - - # Adding the card to a section - sections[repo.group] += card - - return templates["website"].render( - section_transpiler_plugin_cards=sections["transpiler_plugin"], - section_provider_cards=sections["provider"], - section_applications_cards=sections["applications"], - section_other_cards=sections["other"], - ) - - def dispatch_check_workflow( - self, - repo_url: str, - branch_name: str, - tier: str, - token: str, - owner: str = "qiskit-community", - repo: str = "ecosystem", - ) -> bool: - """Dispatch event to trigger check workflow. - - Args: - repo_url: url of the repo - branch_name: name of the branch - tier: tier of the project - token: token base on the date - owner: "qiskit-community" parameters - repo: "ecosystem" - - Return: true - """ - url = "https://api.github.com/repos/{owner}/{repo}/dispatches".format( - owner=owner, repo=repo - ) - repo_split = repo_url.split("/") - repo_name = repo_split[-1] - - # run each type of tests in same workflow - response = requests.post( - url, - json={ - "event_type": "check_project", - "client_payload": { - "repo_url": repo_url, - "repo_name": repo_name, - "branch_name": branch_name, - "tier": tier, - }, - }, - headers={ - "Authorization": "token {}".format(token), - "Accept": "application/vnd.github.v3+json", - }, - ) - if response.ok: - self.logger.info("Success response on dispatch event. %s", response.text) - else: - self.logger.warning( - "Something wend wrong with dispatch event: %s", response.text - ) - return True - - def expose_all_project_to_actions(self): - """Exposes all project for github actions.""" - repositories = [] - tiers = [] - for tier in Tier.non_main_tiers(): - for repo in self.dao.get_repos_by_tier(tier): - if not repo.skip_tests: - repositories.append(repo.url) - tiers.append(repo.tier) - set_actions_output( - [("repositories", ",".join(repositories)), ("tiers", ",".join(tiers))] - ) - - def update_badges(self): - """Updates badges for projects.""" - badges_folder_path = "{}/badges".format(self.current_dir) - - for tier in Tier.all(): - for project in self.dao.get_repos_by_tier(tier): - tests_passed = True - for type_test in project.tests_results: - if type_test.test_type == "standard" and not type_test.passed: - tests_passed = False - color = "blueviolet" if tests_passed else "gray" - label = project.name - message = tier - url = ( - f"https://img.shields.io/static/v1?" - f"label={label}&message={message}&color={color}" - ) - - shields_request = requests.get(url) - with open(f"{badges_folder_path}/{project.name}.svg", "wb") as outfile: - outfile.write(shields_request.content) - self.logger.info("Badge for %s has been updated.", project.name) - - def update_stars(self): - """Updates start for repositories.""" - for tier in Tier.all(): - for project in self.dao.get_repos_by_tier(tier): - stars = None - url = project.url[:-1] if project.url[-1] == "/" else project.url - url_chunks = url.split("/") - repo = url_chunks[-1] - user = url_chunks[-2] - - response = requests.get(f"http://api.github.com/repos/{user}/{repo}") - if not response.ok: - self.logger.warning("Bad response for project %s", project.url) - continue - - json_data = json.loads(response.text) - stars = json_data.get("stargazers_count") - self.dao.update(project.url, stars=stars) - self.logger.info("Updating star count for %s: %d", project.url, stars) - - @staticmethod - def parser_issue(body: str) -> None: - """Command for calling body issue parsing function. - - Args: - body: body of the created issue - - Returns: - logs output - We want to give the result of the parsing issue to the GitHub action - """ - - parsed_result = parse_submission_issue(body) - - to_print = [ - ("SUBMISSION_NAME", parsed_result.name), - ("SUBMISSION_REPO", parsed_result.url), - ("SUBMISSION_DESCRIPTION", parsed_result.description), - ("SUBMISSION_LICENCE", parsed_result.licence), - ("SUBMISSION_CONTACT", parsed_result.contact_info), - ("SUBMISSION_ALTERNATIVES", parsed_result.alternatives), - ("SUBMISSION_AFFILIATIONS", parsed_result.affiliations), - ("SUBMISSION_LABELS", parsed_result.labels), - ("SUBMISSION_WEBSITE", parsed_result.website), - ] - - set_actions_output(to_print) - - def add_repo_2db( - self, - repo_name: str, - repo_link: str, - repo_description: str, - repo_licence: str, - repo_contact: str, - repo_alt: str, - repo_affiliations: str, - repo_labels: Tuple[str], - repo_tier: Optional[str] = None, - repo_website: Optional[str] = None, - ) -> None: - """Adds repo to list of entries. - - Args: - repo_name: repo name - repo_link: repo url - repo_description: repo description - repo_contact: repo email - repo_alt: repo alternatives - repo_licence: repo licence - repo_affiliations: repo university, company, ... - repo_labels: comma separated labels - repo_tier: tier for repository - repo_website: link to project website - """ - - new_repo = Repository( - repo_name, - repo_link, - repo_description, - repo_licence, - repo_contact, - repo_alt, - repo_affiliations, - list(repo_labels), - tier=repo_tier or Tier.COMMUNITY, - website=repo_website, - ) - self.dao.write(new_repo) - def _save_temp_test_result( self, folder_name: str, diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py new file mode 100644 index 0000000000..5a0888fc8e --- /dev/null +++ b/ecosystem/cli/website.py @@ -0,0 +1,97 @@ +"""CliWebsite class for controlling all CLI functions.""" +import os +from typing import Optional + +from jinja2 import Environment, FileSystemLoader + +from ecosystem.daos import DAO + + +class CliWebsite: + """CliMembers class. + Entrypoint for all CLI website commands. + + Each public method of this class is CLI command + and arguments for method are options/flags for this command. + + Ex: `python manager.py website build_website` + """ + + def __init__(self, root_path: Optional[str] = None): + """CliWebsite class.""" + self.current_dir = root_path or os.path.abspath(os.getcwd()) + self.resources_dir = "{}/ecosystem/resources".format(self.current_dir) + self.dao = DAO(path=self.resources_dir) + + def build_website(self): + """Generates the ecosystem web page reading `members.json`.""" + environment = Environment(loader=FileSystemLoader("ecosystem/html_templates/")) + projects = self.dao.storage.read() + projects_sorted = sorted( + projects.items(), + key=lambda item: ( + -item[1].stars if item[1].stars is not None else 0, + item[1].name, + ), + ) + templates = { + "website": environment.get_template("webpage.html.jinja"), + "card": environment.get_template("card.html.jinja"), + "tag": environment.get_template("tag.html.jinja"), + "link": environment.get_template("link.html.jinja"), + } + sections = { + "transpiler_plugin": "", + "provider": "", + "applications": "", + "other": "", + } + + max_chars_description_visible = 400 + min_chars_description_hidden = 100 + count_read_more = 1 + for _, repo in projects_sorted: + # Card tags + tags = "" + for label in repo.labels: + tags += templates["tag"].render(color="purple", title=label, text=label) + + # Card links + links = templates["link"].render(url=repo.url, place="repository") + if repo.website: + links += templates["link"].render(url=repo.website, place="website") + + # Card description + if ( + len(repo.description) - max_chars_description_visible + >= min_chars_description_hidden + ): + description = [ + repo.description[:max_chars_description_visible], + repo.description[max_chars_description_visible:], + ] + id_read_more = str(count_read_more) + count_read_more += 1 + else: + description = [repo.description, ""] + id_read_more = "None" + + # Create the card + card = templates["card"].render( + title=repo.name, + tags=tags, + description_visible=description[0], + description_hidden=description[1], + id_read_more=id_read_more, + links=links, + ) + + # Adding the card to a section + sections[repo.group] += card + + return templates["website"].render( + section_transpiler_plugin_cards=sections["transpiler_plugin"], + section_provider_cards=sections["provider"], + section_applications_cards=sections["applications"], + section_other_cards=sections["other"], + ) diff --git a/manager.py b/manager.py index 198fb572a8..b59e74d0e4 100644 --- a/manager.py +++ b/manager.py @@ -4,33 +4,50 @@ 1. Run tests within repository. ```shell -python manager.py python_standard_tests https://github.com// --tox_python= +python manager.py tests python_standard_tests https://github.com// --tox_python= ``` 2. Run tests against stable version of Qiskit. ```shell -python manager.py python_stable_tests https://github.com// --tox_python= +python manager.py tests python_stable_tests https://github.com// --tox_python= ``` 3. Run tests against dev version of Qiskit. ```shell -python manager.py python_dev_tests https://github.com// --tox_python= +python manager.py tests python_dev_tests https://github.com// --tox_python= ``` 4. Get parse issue. ```shell -python manager.py parser_issue --body="${{ github.event.issue.body }}" +python manager.py ci parser_issue --body="${{ github.event.issue.body }}" ``` 5. Add repo to jsondb. ```shell -python manager.py add_repo_2db --repo_link="https://github.com//" --repo_author="" ... +python manager.py members add_repo_2db --repo_link="https://github.com//" --repo_author="" ... +``` + +6. Recompile members. +```shell +python manager.py members recompile" +``` + +7. Build website. +```shell +python manager.py website build_website" ``` """ import fire -from ecosystem import Manager +from ecosystem.cli import CliMembers, CliWebsite, CliCI, CliTests if __name__ == "__main__": - fire.Fire(Manager) + fire.Fire( + { + "members": CliMembers, + "tests": CliTests, + "website": CliWebsite, + "ci": CliCI, + } + ) diff --git a/tests/test_manager.py b/tests/test_cli.py similarity index 89% rename from tests/test_manager.py rename to tests/test_cli.py index cb61f59b9b..d4f1b43005 100644 --- a/tests/test_manager.py +++ b/tests/test_cli.py @@ -1,4 +1,4 @@ -"""Tests for manager cli.""" +"""Tests for cli.""" import os import io from unittest import TestCase @@ -7,7 +7,7 @@ import responses from ecosystem.daos import DAO -from ecosystem.manager import Manager +from ecosystem.cli import CliCI, CliTests, CliMembers, CliWebsite from ecosystem.models import TestResult, Tier, TestType from ecosystem.models.repository import Repository from ecosystem.models.test_results import Package @@ -65,8 +65,8 @@ def get_community_fail_repo() -> Repository: ) -class TestManager(TestCase): - """Test class for manager cli.""" +class TestCli(TestCase): + """Test class for cli.""" def setUp(self) -> None: self.path = "../resources" @@ -96,12 +96,12 @@ def _delete_members_json(self): def test_build_website(self): """Test the website builder function.""" - manager = Manager(root_path=f"{os.path.abspath(os.getcwd())}/../") - self.assertIsInstance(manager.build_website(), str) + cli_website = CliWebsite(root_path=f"{os.path.abspath(os.getcwd())}/../") + self.assertIsInstance(cli_website.build_website(), str) def test_parser_issue(self): """Tests issue parsing function. - Function: Manager + Function: Cli -> parser_issue Args: issue_body @@ -110,7 +110,7 @@ def test_parser_issue(self): # Issue 1 captured_output = io.StringIO() with redirect_stdout(captured_output): - Manager.parser_issue(self.issue_body) + CliCI.parser_issue(self.issue_body) output_value = captured_output.getvalue().split("\n") @@ -140,7 +140,7 @@ def test_parser_issue(self): # Issue 2 captured_output = io.StringIO() with redirect_stdout(captured_output): - Manager.parser_issue(self.issue_body_2) + CliCI.parser_issue(self.issue_body_2) output_value = captured_output.getvalue().split("\n") @@ -169,7 +169,7 @@ def test_parser_issue(self): @responses.activate def test_dispatch_repository(self): """Test github dispatch event. - Function: Manager + Function: Cli -> dispatch_check_workflow Args: Infos about repo @@ -188,8 +188,8 @@ def test_dispatch_repository(self): "content_type": "application/json", } ) - manager = Manager(root_path=f"{os.path.abspath(os.getcwd())}/../") - response = manager.dispatch_check_workflow( + cli_ci = CliCI(root_path=f"{os.path.abspath(os.getcwd())}/../") + response = cli_ci.dispatch_check_workflow( repo_url="https://github.com/Qiskit-demo/qiskit-demo", branch_name="awesome_branch", tier="COMMUNITY", @@ -211,14 +211,14 @@ def test_update_badges(self): dao.write(commu_success) dao.write(commu_failed) - manager = Manager(root_path=os.path.join(self.current_dir, "..")) - manager.resources_dir = "../resources" - manager.dao = dao + cli_members = CliMembers(root_path=os.path.join(self.current_dir, "..")) + cli_members.resources_dir = "../resources" + cli_members.dao = dao # create badges - manager.update_badges() + cli_members.update_badges() - badges_folder_path = "{}/badges".format(manager.current_dir) + badges_folder_path = "{}/badges".format(cli_members.current_dir) self.assertTrue( os.path.isfile(f"{badges_folder_path}/{commu_success.name}.svg") ) @@ -240,7 +240,7 @@ def test_update_badges(self): @responses.activate def test_fetch_and_update_main_projects(self): - """Tests manager function for fetching tests results.""" + """Tests cli function for fetching tests results.""" owner = "Qiskit" repos_and_test_names = [ ("qiskit-nature", "Nature%2520Unit%2520Tests"), @@ -283,5 +283,5 @@ def test_fetch_and_update_main_projects(self): } ) - manager = Manager(root_path=f"{os.path.abspath(os.getcwd())}/../") - self.assertIsNone(manager.fetch_and_update_main_tests_results()) + cli_tests = CliTests(root_path=f"{os.path.abspath(os.getcwd())}/../") + self.assertIsNone(cli_tests.fetch_and_update_main_tests_results()) diff --git a/tox.ini b/tox.ini index cb25432a9f..4372a5f5f0 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ commands = black {posargs} ecosystem tests --check allowlist_externals = bash basepython = python3 commands = - bash -ec "python manager.py build_website > website/index.html" + bash -ec "python manager.py website build_website > website/index.html" [testenv:ci] commands =