New circleci webhooks

baldrick/__init__.py

@@ -1,6 +1,9 @@
 import os
 
-from baldrick import github  # noqa
+from loguru import logger
+
+from baldrick import github
+from baldrick.github import github_auth  # noqa
 
 __all__ = ['create_app', '__version__']
 
@@ -61,6 +64,13 @@ def create_app(name, register_blueprints=True):
     app.integration_id = int(os.environ['GITHUB_APP_INTEGRATION_ID'])
     app.private_key = os.environ['GITHUB_APP_PRIVATE_KEY']
 
+    try:
+        repos = github_auth.repo_to_installation_id_mapping()
+    except Exception as e:
+        logger.exception("Failed to auth with GitHub")
+    else:
+        logger.info(f"Installed on the following repos {repos}")
+
     app.bot_username = name
 
     if register_blueprints:

baldrick/blueprints/circleci.py

@@ -1,4 +1,7 @@
 import json
+from pprint import pformat
+
+from loguru import logger
 
 from baldrick.github.github_auth import repo_to_installation_id_mapping
 from baldrick.github.github_api import RepoHandler
@@ -52,6 +55,57 @@ def circleci_handler():
     repo_handler = RepoHandler(repo, branch="master", installation=repos[repo])
 
     for handler in CIRCLECI_WEBHOOK_HANDLERS:
-        handler(repo_handler, payload, request.headers)
+        handler(repo_handler, "v1", payload, request.headers, payload["status"], payload["vcs_revision"], payload["build_num"])
+
+    return "CirleCI Webhook Finished"
+
+
+@circleci_blueprint.route('/circleci/v2', methods=['POST'])
+def circleci_new_handler():
+    if not request.data:
+        return "No payload received"
+
+    payload = json.loads(request.data)
+
+    logger.debug(f"Got {pformat(payload)} on /circleci/v2")
+    # Validate we have the keys we need, otherwise ignore the push
+    required_keys = {
+        'job',
+        'pipeline',
+    }
+
+    if not required_keys.issubset(payload.keys()):
+        msg = 'Payload missing {}'.format(' '.join(required_keys - payload.keys()))
+        logger.error(msg)
+        return msg
 
-    return "CirleCI Webhook Finsihed"
+    vcs = payload["pipeline"]["vcs"]
+
+    if vcs["provider_name"] != "github":
+        msg = "Only GitHub repositories are supported."
+        logger.error(msg)
+        return msg
+
+    # Get installation id
+    repos = repo_to_installation_id_mapping()
+
+    repo = vcs["target_repository_url"].removeprefix("https://github.com/")
+
+    if repo not in repos:
+        msg = f"Not installed for {repo}"
+        logger.error(msg)
+        logger.trace(f"Only installed for {repos.keys()}")
+        return msg
+
+    repo_handler = RepoHandler(repo, branch=vcs["branch"], installation=repos[repo])
+
+    for handler in CIRCLECI_WEBHOOK_HANDLERS:
+        handler(repo_handler,
+                "v2",
+                payload,
+                request.headers,
+                payload["job"].get("status"),
+                vcs["revision"],
+                payload["job"]["number"])
+
+    return "CirleCI Webhook Finished"

baldrick/github/github_api.py

@@ -65,9 +65,9 @@ def repo_info(self):
         """
         The return of GET /repos/{org}/{repo}
         """
-        response = requests.get(f"{HOST}/repos/{self.repo}")
+        response = requests.get(f"{HOST}/repos/{self.repo}", headers=self._headers)
         if not response.ok:
-            raise ValueError("Unable to fetch repo information {response.json()}")
+            raise ValueError(f"Unable to fetch repo information {response.json()}")
         return response.json()
 
     @property
@@ -319,7 +319,7 @@ def get_issues(self, state, labels, exclude_pr=True):
         """
         url = f'{HOST}/repos/{self.repo}/issues'
         kwargs = {'state': state, 'labels': labels}
-        r = requests.get(url, kwargs)
+        r = requests.get(url, kwargs, headers=headers)
         result = r.json()
         if exclude_pr:
             issue_list = [d['number'] for d in result

baldrick/github/github_auth.py

@@ -4,6 +4,7 @@
 from collections import defaultdict
 
 import dateutil.parser
+from loguru import logger
 
 import jwt
 
@@ -30,7 +31,7 @@ def get_json_web_token():
 
     # Include a one-minute buffer otherwise token might expire by the time we
     # make the request with the token.
-    if json_web_token_expiry is None or now + ONE_MIN > json_web_token_expiry:
+    if json_web_token is None or json_web_token_expiry is None or now + ONE_MIN > json_web_token_expiry:
 
         json_web_token_expiry = now + TEN_MIN
 
@@ -82,7 +83,8 @@ def get_installation_token(installation):
 
         headers = {}
         headers['Authorization'] = 'Bearer {0}'.format(get_json_web_token())
-        headers['Accept'] = 'application/vnd.github.machine-man-preview+json'
+        headers['Accept'] = 'application/vnd.github+json'
+        headers['X-GitHub-Api-Version'] = "2022-11-28"
 
         url = 'https://api.github.com/app/installations/{0}/access_tokens'.format(installation)
 
@@ -91,7 +93,7 @@ def get_installation_token(installation):
 
         if not req.ok:
             if 'message' in resp:
-                raise Exception(resp['message'])
+                raise Exception(f"{req.status_code} {resp['message']}")
             else:
                 raise Exception("An error occurred when requesting token")
 
@@ -119,10 +121,14 @@ def repo_to_installation_id_mapping():
     url = 'https://api.github.com/app/installations'
     headers = {}
     headers['Authorization'] = 'Bearer {0}'.format(get_json_web_token())
-    headers['Accept'] = 'application/vnd.github.machine-man-preview+json'
+    headers['Accept'] = 'application/vnd.github+json'
+    headers['X-GitHub-Api-Version'] = "2022-11-28"
     resp = requests.get(url, headers=headers)
     payload = resp.json()
 
+    if resp.status_code != 200:
+        raise ValueError(f"{resp.status_code} {payload} in response from GitHub while getting installations")
+
     ids = [p['id'] for p in payload]
 
     repos = {}

baldrick/plugins/circleci_artifacts.py

@@ -6,40 +6,51 @@
 
 
 @circleci_webhook_handler
-def set_commit_status_for_artifacts(repo_handler, payload, headers):
+def set_commit_status_for_artifacts(repo_handler, webhook_version, payload, headers, status, revision, build_number):
+    if webhook_version == "v2" and payload.get("type") != "job-completed":
+        msg = "Ignoring not 'job-completed' webhook."
+        logger.debug(msg)
+        return
 
     ci_config = repo_handler.get_config_value("circleci_artifacts", {})
     if not ci_config.get("enabled", False):
         msg = "Skipping artifact check, disabled in config."
         logger.debug(msg)
         return msg
 
-    logger.info(f"Got CircleCI payload for repo: {payload['username']}/{payload['reponame']}")
-    artifacts = get_artifacts_from_build(payload)
+    repo = repo_handler.repo
+    logger.info(f"Got CircleCI payload for repo: {repo}")
+    artifacts = get_artifacts_from_build(repo, build_number)
 
     # Remove enabled from the config list
     ci_config.pop("enabled", None)
 
     for name, config in ci_config.items():
-        if payload["status"] != "success" and not config.get("report_on_fail", False):
+        logger.debug(f"Job {name=} {config=}")
+        if not config.get("enabled", True) or (status != "success" and not config.get("report_on_fail", False)):
+            continue
+
+        if "url" not in config or "message" not in config:
+            logger.warning(f"Incorrectly configured job {name}, skipping because missing url or message")
             continue
 
         url = get_documentation_url_from_artifacts(artifacts, config['url'])
-        logger.debug(f"Found artifact: {url}")
 
         if url:
+            logger.debug(f"Found artifact: {url}")
             repo_handler.set_status("success",
                                     config["message"],
                                     name,
-                                    payload["vcs_revision"],
+                                    revision,
                                     url)
 
     return "All good"
 
 
-def get_artifacts_from_build(p):  # pragma: no cover
+def get_artifacts_from_build(repo, build_num):  # pragma: no cover
     base_url = "https://circleci.com/api/v1.1"
-    query_url = f"{base_url}/project/github/{p['username']}/{p['reponame']}/{p['build_num']}/artifacts"
+    query_url = f"{base_url}/project/github/{repo}/{build_num}/artifacts"
+    logger.debug(f"Getting build {query_url}")
     response = requests.get(query_url)
     assert response.ok, response.content
     return response.json()