diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..669f4b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +*~ +\#* +.#* +*.egg-info/ +*.exp +*.lib +*.obj +*.pyc +*.pyd.manifest +*.pyo +.cache +.DS_Store +.idea +.project +.pydevproject +/.settings +build/ +dist/ +__pycache__ +debian +# Temporary dir for the anaconda builds. +misc/installer/anaconda/conda_builds +.vscode/** diff --git a/README.md b/README.md index 89f4168..a965583 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,22 @@ # obspy_github_api Helper routines to interact with obspy/obspy via GitHub API +## Quick start + +The easiest way to use obspy_github_api is via its command line interface. + +```shell script +# Use the magic strings found in issue 101's comments to create a config file +obshub make_config 101 --path obspy_config.json + +# Read a specified option. +obshub read_config_value module_list --path obspy_config.json + +# Use a value in the config in another command line utility. +export BUILDDOCS=`bshub read_config_value module_list --path obspy_config.json` +some-other-command --docs $BUILDDOCS +``` + ## Release Versions Release versions are done from separate branches, see https://github.com/obspy/obspy_github_api/branches. diff --git a/obspy_github_api/__init__.py b/obspy_github_api/__init__.py index 6438f1e..7d41f8e 100644 --- a/obspy_github_api/__init__.py +++ b/obspy_github_api/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- from .obspy_github_api import * -__version__ = '0.0.0.dev' +__version__ = "0.0.0.dev" diff --git a/obspy_github_api/cli.py b/obspy_github_api/cli.py new file mode 100644 index 0000000..990cd58 --- /dev/null +++ b/obspy_github_api/cli.py @@ -0,0 +1,52 @@ +""" +Command line Interface for obspy_github_api +""" +import json +from typing import Optional + +import typer + +from obspy_github_api.obspy_github_api import make_ci_json_config + +app = typer.Typer() + +DEFAULT_CONFIG_PATH = "obspy_config/conf.json" + + +@app.command() +def make_config( + issue_number: int, path: str = DEFAULT_CONFIG_PATH, token: Optional[str] = None +): + """ + Create ObsPy's configuration json file for a particular issue. + + This command parses the comments in an issue's text looking for any magic + strings (defined in ObsPy's issue template) and stores the values assigned + to them to a json file for later use. + + The following names are stored in the config file: + module_list - A string of requested modules separated by commas. + module_list_spaces - A string of requested modules separated by spaces. + docs - True if a doc build is requested. + """ + make_ci_json_config(issue_number, path=path, token=token) + + +@app.command() +def read_config_value(name: str, path: str = DEFAULT_CONFIG_PATH): + """ + Read a value from the configuration file. + """ + with open(path, "r") as fi: + params = json.load(fi) + value = params[name] + print(value) + return value + + +def main(): + app() + + +if __name__ == "__main__": + main() diff --git a/obspy_github_api/obspy_github_api.py b/obspy_github_api/obspy_github_api.py index 985bee4..dae042f 100644 --- a/obspy_github_api/obspy_github_api.py +++ b/obspy_github_api/obspy_github_api.py @@ -1,33 +1,45 @@ # -*- coding: utf-8 -*- +import ast import datetime +import json import os import re import time import warnings +from functools import lru_cache +from pathlib import Path import github3 # regex pattern in comments for requesting a docs build -PATTERN_DOCS_BUILD = r'\+DOCS' +PATTERN_DOCS_BUILD = r"\+DOCS" # regex pattern in comments for requesting tests of specific submodules -PATTERN_TEST_MODULES = r'\+TESTS:([a-zA-Z0-9_\.,]*)' +PATTERN_TEST_MODULES = r"\+TESTS:([a-zA-Z0-9_\.,]*)" -try: - # github API token with "repo.status" access right (if used to set commit - # statuses) or with empty scope; to get around rate limitations - token = os.environ["OBSPY_COMMIT_STATUS_TOKEN"] -except KeyError: - msg = ("Could not get authorization token for ObsPy github API " - "(env variable OBSPY_COMMIT_STATUS_TOKEN)") - warnings.warn(msg) - gh = github3.GitHub() -else: - gh = github3.login(token=token) +@lru_cache() +def get_github_client(token=None): + """ + Returns the github client + + github API token with "repo.status" access right (if used to set commit + statuses) or with empty scope; to get around rate limitations + """ + token = token or os.environ.get("GITHUB_TOKEN", None) + if token is None: + msg = ( + "Could not get authorization token for ObsPy github API " + "(env variable OBSPY_COMMIT_STATUS_TOKEN)" + ) + warnings.warn(msg) + gh = github3.GitHub() + else: + gh = github3.login(token=token) + return gh -def check_specific_module_tests_requested(issue_number): +def check_specific_module_tests_requested(issue_number, token=None): """ Checks if tests of specific modules are requested for given issue number (e.g. by magic string '+TESTS:clients.fdsn,clients.arclink' or '+TESTS:ALL' @@ -39,6 +51,7 @@ def check_specific_module_tests_requested(issue_number): modules for given issue number or ``False`` if no specific tests are requested or ``True`` if all modules should be tested. """ + gh = get_github_client(token) issue = gh.issue("obspy", "obspy", issue_number) modules_to_test = set() @@ -67,18 +80,29 @@ def check_specific_module_tests_requested(issue_number): return modules_to_test -def get_module_test_list(issue_number): +def get_module_test_list( + issue_number, token=None, module_path="./obspy/core/util/base.py" +): """ Gets the list of modules that should be tested for the given issue number. - This needs obspy to be installed, because DEFAULT_MODULES and ALL_MODULES - is used. + + If obspy is installed get DEFAULT_MODULES and ALL_MODULES from + core.util.base, else use `constants_path` to look for the constants file + which contains these lists and no other ObsPy imports. :rtype: list :returns: List of modules names to test for given issue number. """ - from obspy.core.util.base import DEFAULT_MODULES, ALL_MODULES + try: # If ObsPy is installed just use module list from expected place. + from obspy.core.util.base import DEFAULT_MODULES, ALL_MODULES + except (ImportError, ModuleNotFoundError): # Else parse the module. + names = {"DEFAULT_MODULES", "NETWORK_MODULES"} + values = get_values_from_module(module_path, names) + DEFAULT_MODULES = values["DEFAULT_MODULES"] + NETWORK_MODULES = values["NETWORK_MODULES"] + ALL_MODULES = DEFAULT_MODULES + NETWORK_MODULES - modules_to_test = check_specific_module_tests_requested(issue_number) + modules_to_test = check_specific_module_tests_requested(issue_number, token) if modules_to_test is False: return DEFAULT_MODULES @@ -88,13 +112,43 @@ def get_module_test_list(issue_number): return sorted(list(set.union(set(DEFAULT_MODULES), modules_to_test))) -def check_docs_build_requested(issue_number): +def get_values_from_module(node, names): + """ + Get values assigned to specified variables from a python file without + importing it. Only works on variables assigned to simple objects. + + Based on this SO answer: https://stackoverflow.com/a/67692/3645626 + + :rtype: dict + :returns: A dict of {name: value} for specified names. + """ + # Create output dict and specify names to search for. + requested_names = {} if names is None else set(names) + out = {} + + # A path was given, get the ast from it. + if isinstance(node, (str, Path)): + node = ast.parse(open(node).read()) + + # Parse nodes, any assignments to any of requested_names is saved. + if hasattr(node, "body"): + for subnode in node.body: + out.update(get_values_from_module(subnode, names=requested_names)) + elif isinstance(node, ast.Assign): + for name in node.targets: + if isinstance(name, ast.Name) and name.id in requested_names: + out[name.id] = ast.literal_eval(node.value) + return out + + +def check_docs_build_requested(issue_number, token=None): """ Check if a docs build was requested for given issue number (by magic string '+DOCS' anywhere in issue comments). :rtype: bool """ + gh = get_github_client(token) issue = gh.issue("obspy", "obspy", issue_number) if re.search(PATTERN_DOCS_BUILD, issue.body): return True @@ -104,17 +158,18 @@ def check_docs_build_requested(issue_number): return False -def get_pull_requests(state="open", sort="updated", direction="desc"): +def get_pull_requests(state="open", sort="updated", direction="desc", token=None): """ Fetch a list of issue numbers for pull requests recently updated first, along with the PR data. """ + gh = get_github_client(token) repo = gh.repository("obspy", "obspy") prs = repo.pull_requests(state=state, sort=sort, direction=direction) return prs -def get_commit_status(commit, context=None, fork='obspy'): +def get_commit_status(commit, context=None, fork="obspy", token=None): """ Return current commit status. Either for a specific context, or overall. @@ -130,6 +185,7 @@ def get_commit_status(commit, context=None, fork='obspy'): :returns: Current commit status (overall or for specific context) as a string or ``None`` if given context has no status. """ + gh = get_github_client(token) # github3.py seems to lack support for fetching the "current" statuses for # all contexts.. (which is available in "combined status" for an SHA # through github API) @@ -137,8 +193,10 @@ def get_commit_status(commit, context=None, fork='obspy'): commit = repo.commit(commit) statuses = {} for status in commit.statuses(): - if (status.context not in statuses or - status.updated_at > statuses[status.context].updated_at): + if ( + status.context not in statuses + or status.updated_at > statuses[status.context].updated_at + ): statuses[status.context] = status # just return current status for given context @@ -156,27 +214,31 @@ def get_commit_status(commit, context=None, fork='obspy'): return None -def get_commit_time(commit, fork="obspy"): +def get_commit_time(commit, fork="obspy", token=None): """ :rtype: float :returns: Commit timestamp as POSIX timestamp. """ + gh = get_github_client(token) repo = gh.repository(fork, "obspy") commit = repo.commit(commit) - dt = datetime.datetime.strptime(commit.commit.committer["date"], - '%Y-%m-%dT%H:%M:%SZ') + dt = datetime.datetime.strptime( + commit.commit.committer["date"], "%Y-%m-%dT%H:%M:%SZ" + ) return time.mktime(dt.timetuple()) -def get_issue_numbers_that_request_docs_build(verbose=False): +def get_issue_numbers_that_request_docs_build(verbose=False, token=None): """ :rtype: list of int """ - open_prs = get_pull_requests(state="open") + open_prs = get_pull_requests(state="open", token=token) if verbose: - print("Checking the following open PRs if a docs build is requested " - "and needed: {}".format(str(num for num, _ in open_prs))) + print( + "Checking the following open PRs if a docs build is requested " + "and needed: {}".format(str(num for num, _ in open_prs)) + ) todo = [] for pr in open_prs: @@ -187,12 +249,13 @@ def get_issue_numbers_that_request_docs_build(verbose=False): def set_pr_docs_that_need_docs_build( - pr_docs_info_dir="/home/obspy/pull_request_docs", verbose=False): + pr_docs_info_dir="/home/obspy/pull_request_docs", verbose=False, token=None +): """ Relies on a local directory with some files to mark when PR docs have been built etc. """ - prs_todo = get_issue_numbers_that_request_docs_build(verbose=verbose) + prs_todo = get_issue_numbers_that_request_docs_build(verbose=verbose, token=token) for pr in prs_todo: number = pr.number @@ -203,9 +266,10 @@ def set_pr_docs_that_need_docs_build( # need to figure out time of last push from commit details.. -_- time = get_commit_time(commit, fork) if verbose: - print("PR #{} requests a docs build, latest commit {} at " - "{}.".format(number, commit, - str(datetime.fromtimestamp(time)))) + print( + "PR #{} requests a docs build, latest commit {} at " + "{}.".format(number, commit, str(datetime.fromtimestamp(time))) + ) filename = os.path.join(pr_docs_info_dir, str(number)) filename_todo = filename + ".todo" @@ -224,9 +288,12 @@ def set_pr_docs_that_need_docs_build( time_done = os.stat(filename_done).st_atime if time_done > time: if verbose: - print("PR #{} was last built at {} and does not need a " - "new build.".format( - number, str(datetime.fromtimestamp(time_done)))) + print( + "PR #{} was last built at {} and does not need a " + "new build.".format( + number, str(datetime.fromtimestamp(time_done)) + ) + ) continue # ..otherwise touch the .todo file with open(filename_todo, "wb"): @@ -237,9 +304,18 @@ def set_pr_docs_that_need_docs_build( print("Done checking which PRs require a docs build.") -def set_commit_status(commit, status, context, description, - target_url=None, fork="obspy", only_when_changed=True, - only_when_no_status_yet=False, verbose=False): +def set_commit_status( + commit, + status, + context, + description, + target_url=None, + fork="obspy", + only_when_changed=True, + only_when_no_status_yet=False, + verbose=False, + token=None, +): """ :param only_when_changed: Whether to only set a status if the commit status would change (commit statuses can not be updated or deleted and there @@ -250,6 +326,7 @@ def set_commit_status(commit, status, context, description, if status not in ("success", "pending", "error", "failure"): raise ValueError("Invalid status: {}".format(status)) + gh = get_github_client(token) # check current status, only set a status if it would change the current # status.. # (avoid e.g. flooding with "pending" status on continuously breaking docs @@ -261,45 +338,63 @@ def set_commit_status(commit, status, context, description, if only_when_no_status_yet: if current_status is not None: if verbose: - print("Commit {} already has a commit status ({}), " - "skipping.".format(commit, current_status)) + print( + "Commit {} already has a commit status ({}), " + "skipping.".format(commit, current_status) + ) return if only_when_changed: if current_status == status: if verbose: - print("Commit {} status would not change ({}), " - "skipping.".format(commit, current_status)) + print( + "Commit {} status would not change ({}), " + "skipping.".format(commit, current_status) + ) return repo = gh.repository(fork, "obspy") commit = repo.commit(commit) - repo.create_status(sha=commit.sha, state=status, context=context, - description=description, target_url=target_url) + repo.create_status( + sha=commit.sha, + state=status, + context=context, + description=description, + target_url=target_url, + ) if verbose: - print("Set commit {} status (context '{}') to '{}'.".format( - commit.sha, context, status)) + print( + "Set commit {} status (context '{}') to '{}'.".format( + commit.sha, context, status + ) + ) -def set_all_updated_pull_requests_docker_testbot_pending(verbose=False): +def set_all_updated_pull_requests_docker_testbot_pending(verbose=False, token=None): """ Set a status "pending" for all open PRs that have not been processed by docker buildbot yet. """ - open_prs = get_pull_requests(state="open") + + open_prs = get_pull_requests(state="open", token=token) if verbose: - print("Working on PRs: " + ", ".join( - [str(pr.number) for pr in open_prs])) + print("Working on PRs: " + ", ".join([str(pr.number) for pr in open_prs])) for pr in open_prs: set_commit_status( - commit=pr.head.sha, status="pending", context="docker-testbot", + commit=pr.head.sha, + status="pending", + context="docker-testbot", description="docker testbot results not available yet", only_when_no_status_yet=True, - verbose=verbose) + verbose=verbose, + ) def get_docker_build_targets( - context="docker-testbot", branches=["master", "maintenance_1.0.x"], - prs=True): + context="docker-testbot", + branches=["master", "maintenance_1.0.x"], + prs=True, + token=None, +): """ Returns a list of build targets that need a build of a given context. @@ -324,11 +419,12 @@ def get_docker_build_targets( :rtype: string """ if not branches and not prs: - return '' + return "" - status_needs_build = (None, 'pending') + gh = get_github_client(token) + status_needs_build = (None, "pending") targets = [] - repo = gh.repository('obspy', 'obspy') + repo = gh.repository("obspy", "obspy") if branches: for name in branches: @@ -339,16 +435,42 @@ def get_docker_build_targets( continue # branches don't have a PR number, use dummy placeholder 'XXX' so # that variable splitting in bash still works - targets.append('XXX_obspy:{}'.format(sha)) + targets.append("XXX_obspy:{}".format(sha)) if prs: - open_prs = get_pull_requests(state='open') + open_prs = get_pull_requests(state="open") for pr in open_prs: fork = pr.head.user sha = pr.head.sha status = get_commit_status(sha, context=context) if status not in status_needs_build: continue - targets.append('{}_{}:{}'.format(str(pr.number), fork, sha)) + targets.append("{}_{}:{}".format(str(pr.number), fork, sha)) + + return " ".join(targets) - return ' '.join(targets) + +def make_ci_json_config(issue_number, path="obspy_ci_conf.json", token=None): + """ + Make a json file for configuring additional actions in CI. + + Indicates which modules are to be run by tests and if docs are to be built. + """ + # It would be interesting to make this more generic by parsing any magic + # comment string to use for later actions. + module_list = get_module_test_list(issue_number, token=token) + docs = check_docs_build_requested(issue_number, token=token) + + out = dict( + module_list=("obspy." + ",obspy.").join(module_list), + module_list_spaces=" ".join(module_list), + docs=docs, + ) + + # make sure path exists + path = Path(path) + path_dir = path if path.is_dir() else path.parent + path_dir.mkdir(exist_ok=True, parents=True) + + with path.open("w") as fi: + json.dump(out, fi, indent=4) diff --git a/obspy_github_api/tests/test_cli.py b/obspy_github_api/tests/test_cli.py new file mode 100644 index 0000000..82224c2 --- /dev/null +++ b/obspy_github_api/tests/test_cli.py @@ -0,0 +1,49 @@ +""" +Tests for command line interface. +""" +import contextlib +import json +import unittest +import tempfile +from pathlib import Path +from subprocess import run + +import pytest + + +class TestCli: + """" + Test case for command line interface. + """ + + config_dir = tempfile.mkdtemp() + config_path = Path(config_dir) / "conf.json" + pr_number = 100 + + @pytest.fixture(scope="class") + def config_path(self, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("obspy_config") + return Path(tmpdir) / "conf.json" + + @pytest.fixture(scope="class") + def populated_config(self, config_path): + """ Get the config for the test PR. """ + run_str = f"obshub make-config {self.pr_number} --path {config_path}" + run(run_str, shell=True, check=True) + return config_path + + def test_path_exists(self, populated_config): + """The config file should now exist.""" + assert Path(populated_config).exists() + + def test_is_json(self, populated_config): + """Ensue the file created can be read by json module. """ + with Path(populated_config).open("r") as fi: + out = json.load(fi) + assert isinstance(out, dict) + + def test_read_config_value(self, populated_config): + """Ensure the config value is printed to screen""" + run_str = f"obshub read-config-value docs --path {populated_config}" + out = run(run_str, shell=True, capture_output=True) + assert out.stdout.decode("utf8").rstrip() == "False" diff --git a/obspy_github_api/tests/test_obspy_github_api.py b/obspy_github_api/tests/test_obspy_github_api.py index c1169e5..853055c 100644 --- a/obspy_github_api/tests/test_obspy_github_api.py +++ b/obspy_github_api/tests/test_obspy_github_api.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- import mock + from obspy_github_api import ( - check_docs_build_requested, check_specific_module_tests_requested, - get_commit_status, get_commit_time, - get_issue_numbers_that_request_docs_build, get_module_test_list, - ) + check_docs_build_requested, + check_specific_module_tests_requested, + get_commit_status, + get_commit_time, + get_issue_numbers_that_request_docs_build, + get_module_test_list, +) MOCK_DEFAULT_MODULES = ["core", "clients.arclink"] @@ -20,18 +24,20 @@ def test_check_docs_build_requested(): def test_check_specific_module_tests_requested(): assert check_specific_module_tests_requested(100) is False assert check_specific_module_tests_requested(101) is True - assert check_specific_module_tests_requested(102) == ["clients.arclink", - "clients.fdsn"] + assert check_specific_module_tests_requested(102) == [ + "clients.arclink", + "clients.fdsn", + ] -@mock.patch('obspy.core.util.base.DEFAULT_MODULES', MOCK_DEFAULT_MODULES) -@mock.patch('obspy.core.util.base.ALL_MODULES', MOCK_ALL_MODULES) +@mock.patch("obspy.core.util.base.DEFAULT_MODULES", MOCK_DEFAULT_MODULES) +@mock.patch("obspy.core.util.base.ALL_MODULES", MOCK_ALL_MODULES) def test_get_module_test_list(): assert get_module_test_list(100) == MOCK_DEFAULT_MODULES assert get_module_test_list(101) == MOCK_ALL_MODULES assert get_module_test_list(102) == sorted( - set.union(set(MOCK_DEFAULT_MODULES), - ["clients.arclink", "clients.fdsn"])) + set.union(set(MOCK_DEFAULT_MODULES), ["clients.arclink", "clients.fdsn"]) + ) def test_get_commit_status(): @@ -39,12 +45,18 @@ def test_get_commit_status(): sha = "f74e0f5bcf26a47df6138c1ce026d9d14d68c4d7" assert get_commit_status(sha) == "pending" assert get_commit_status(sha, context="docker-testbot") == "pending" - assert get_commit_status( - sha, context="continuous-integration/appveyor/branch") == "success" - assert get_commit_status( - sha, context="continuous-integration/appveyor/pr") == "success" - assert get_commit_status( - sha, context="continuous-integration/travis-ci/pr") == "success" + assert ( + get_commit_status(sha, context="continuous-integration/appveyor/branch") + == "success" + ) + assert ( + get_commit_status(sha, context="continuous-integration/appveyor/pr") + == "success" + ) + assert ( + get_commit_status(sha, context="continuous-integration/travis-ci/pr") + == "success" + ) assert get_commit_status(sha, context="coverage/coveralls") == "failure" diff --git a/setup.py b/setup.py index 65e2e77..fc8c03c 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,34 @@ +""" +Setup for ObsPy's github api package. + +Typically this is just used in CI pipelines. +""" import inspect import os import re +import sys from setuptools import setup +if sys.version_info < (2, 7): + sys.exit("Python < 2.7 is not supported") + INSTALL_REQUIRES = [ - 'github3.py>=1.0.0a1', # works with 1.0.0a4 - # soft dependency on obspy itself, - # for routine `get_module_test_list` - ] + "github3.py>=1.0.0a1", # works with 1.0.0a4 + "typer", + # soft dependency on ObsPy itself, for function `get_module_test_list` + # or the path to obspy.core.utils.base.py can be provided to avoid + # needing to have ObsPy installed. +] -SETUP_DIRECTORY = os.path.dirname(os.path.abspath(inspect.getfile( - inspect.currentframe()))) +SETUP_DIRECTORY = os.path.dirname( + os.path.abspath(inspect.getfile(inspect.currentframe())) +) # get the package version from from the main __init__ file. version_regex_pattern = r"__version__ += +(['\"])([^\1]+)\1" -for line in open(os.path.join(SETUP_DIRECTORY, 'obspy_github_api', - '__init__.py')): - if '__version__' in line: +for line in open(os.path.join(SETUP_DIRECTORY, "obspy_github_api", "__init__.py")): + if "__version__" in line: version = re.match(version_regex_pattern, line).group(2) @@ -27,11 +38,13 @@ def find_packages(): """ modules = [] for dirpath, _, filenames in os.walk( - os.path.join(SETUP_DIRECTORY, "obspy_github_api")): + os.path.join(SETUP_DIRECTORY, "obspy_github_api") + ): if "__init__.py" in filenames: modules.append(os.path.relpath(dirpath, SETUP_DIRECTORY)) return [_i.replace(os.sep, ".") for _i in modules] + setup( name="obspy_github_api", version=version, @@ -41,9 +54,10 @@ def find_packages(): url="https://github.com/obspy/obspy_github_api", download_url="https://github.com/obspy/obspy_github_api.git", install_requires=INSTALL_REQUIRES, + python_requires=">3.5", keywords=["obspy", "github"], packages=find_packages(), - entry_points={}, + entry_points={"console_scripts": ["obshub=obspy_github_api.cli:main"]}, classifiers=[ "Programming Language :: Python", "Development Status :: 4 - Beta", @@ -51,7 +65,6 @@ def find_packages(): "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", - ], - long_description="Helper routines to interact with obspy/obspy via GitHub " - "API", - ) + ], + long_description="Helper routines to interact with obspy/obspy via GitHub " "API", +)