diff --git a/.env-dummy b/.env-dummy index 6c0f5a5..a30302c 100644 --- a/.env-dummy +++ b/.env-dummy @@ -1,6 +1,18 @@ # Go to smee.io to generate a URL here SMEE_URL=https://smee.io/CHANGEME +# Optionally customize redis host (local test server defined in docker-compose.yml) +REDIS_HOST=rq-server + +# Optionally customize redis port +REDIS_PORT=6379 + +# Optionally configure time before jobs are killed and marked failed (in seconds, default 180s) +WORKER_JOB_TIMEOUT=21600 + +# Debug level (one of: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") +SPACKBOT_LOG_LEVEL=WARNING + # You don't need to change this unless you change the docker-compose volumes GITHUB_PRIVATE_KEY=/app/spackbot/spack-bot-develop.private-key.pem diff --git a/.github/workflows/build-deploy.yaml b/.github/workflows/build-deploy.yaml index 57b687d..37ad995 100644 --- a/.github/workflows/build-deploy.yaml +++ b/.github/workflows/build-deploy.yaml @@ -12,19 +12,24 @@ on: jobs: deploy-test-containers: runs-on: ubuntu-latest - name: Build Spackbot Container + strategy: + fail-fast: false + # matrix: [tag, path to Dockerfile, label] + matrix: + dockerfile: [[spack-bot, ./Dockerfile, Spackbot], + [spackbot-workers, ./workers/Dockerfile, "Spackbot Workers"]] + name: Build ${{matrix.dockerfile[2]}} Container steps: - name: Checkout uses: actions/checkout@v2 - name: Build and Run Test Container run: | - docker build -t ghcr.io/spack/spack-bot:latest . - docker tag ghcr.io/spack/spack-bot:latest ghcr.io/spack/spack-bot:${GITHUB_SHA::8} - + docker build -f ${{matrix.dockerfile[1]}} -t ghcr.io/spack/${{matrix.dockerfile[0]}}:latest . + docker tag ghcr.io/spack/${{matrix.dockerfile[0]}}:latest ghcr.io/spack/${{matrix.dockerfile[0]}}:${GITHUB_SHA::8} - name: Login and Deploy Test Container if: (github.event_name != 'pull_request') run: | docker images echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ secrets.GHCR_USERNAME }} --password-stdin - docker push --all-tags ghcr.io/spack/spack-bot + docker push --all-tags ghcr.io/spack/${{matrix.dockerfile[0]}} diff --git a/Dockerfile b/Dockerfile index 2d7c709..689624f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,11 +2,14 @@ FROM python:3.7 EXPOSE 8080 +# dependencies first since they're the slowest COPY requirements.txt . -COPY spackbot /app/spackbot -COPY entrypoint.sh /entrypoint.sh RUN pip3 install -r requirements.txt +# copy app in last so that everything above can be cached +COPY spackbot /app/spackbot +COPY entrypoint.sh /entrypoint.sh + ENV PYTHONPATH "${PYTHONPATH}:/app" CMD ["/bin/bash", "/entrypoint.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index c376ff6..93aa305 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,25 @@ services: context: . dockerfile: smee/Dockerfile + rq-worker: + build: + context: . + dockerfile: workers/Dockerfile + env_file: + - ./.env + deploy: + replicas: 1 + + rq-server: + env_file: + - ./.env + image: redis:alpine + expose: + - ${REDIS_PORT} + volumes: + - redis-data:/data + - redis-conf:/usr/local/etc/redis/redis.conf + spackbot: build: context: . @@ -29,3 +48,7 @@ services: - ./.env links: - smee + +volumes: + redis-data: + redis-conf: diff --git a/entrypoint.sh b/entrypoint.sh index 333da8d..4c49f34 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,13 +1,3 @@ #!/bin/bash -# If we have an ssh key bound, add it -if [[ -f "/root/.ssh/id_rsa" ]]; then - printf "Found id_spackbot to authenticate write...\n" - eval "$(ssh-agent -s)" - ssh-add /root/.ssh/id_rsa -else - printf "No id_spackbot found, will not have full permissions\n" -fi - -ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts exec python3 -m spackbot diff --git a/requirements.txt b/requirements.txt index 977d905..e71bb33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,5 @@ aiohttp gidgethub python_dotenv +rq sh - -# Add these so we don't wait for install -mypy -flake8 -isort diff --git a/spackbot/__main__.py b/spackbot/__main__.py index 2390c66..a945054 100644 --- a/spackbot/__main__.py +++ b/spackbot/__main__.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import logging import os import aiohttp @@ -13,12 +12,12 @@ from gidgethub import aiohttp as gh_aiohttp from .routes import router from .auth import authenticate_installation +from .helpers import get_logger # take environment variables from .env file (if present) load_dotenv() -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("spackbot") +logger = get_logger(__name__) #: Location for authenticatd app to get a token for one of its installations INSTALLATION_TOKEN_URL = "app/installations/{installation_id}/access_tokens" @@ -41,13 +40,19 @@ async def main(request): logger.info(f"Received event {event}") # get an installation token to make a GitHubAPI for API calls - token = await authenticate_installation(event.data) + installation_id = event.data["installation"]["id"] + token = await authenticate_installation(installation_id) + + dispatch_kwargs = { + "installation_id": installation_id, + "token": token, + } async with aiohttp.ClientSession() as session: gh = gh_aiohttp.GitHubAPI(session, REQUESTER, oauth_token=token) # call the appropriate callback for the event - await router.dispatch(event, gh, session=session) + await router.dispatch(event, gh, session=session, **dispatch_kwargs) # return a "Success" return web.Response(status=200) diff --git a/spackbot/auth.py b/spackbot/auth.py index afe5bda..3e5b0a1 100644 --- a/spackbot/auth.py +++ b/spackbot/auth.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import logging import os import re import time @@ -16,9 +15,6 @@ load_dotenv() -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("spackbot") - #: Location for authenticatd app to get a token for one of its installations INSTALLATION_TOKEN_URL = "app/installations/{installation_id}/access_tokens" @@ -79,14 +75,13 @@ async def renew_jwt(): return await _tokens.get_token("JWT", renew_jwt) -async def authenticate_installation(payload): +async def authenticate_installation(installation_id): """Get an installation access token for the application. Renew the JWT if necessary, then use it to get an installation access token from github, if necessary. """ - installation_id = payload["installation"]["id"] async def renew_installation_token(): async with aiohttp.ClientSession() as session: diff --git a/spackbot/comments.py b/spackbot/comments.py index 7505c54..31f3e21 100644 --- a/spackbot/comments.py +++ b/spackbot/comments.py @@ -3,7 +3,9 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import io import random +import traceback import spackbot.helpers as helpers @@ -11,9 +13,13 @@ async def tell_joke(gh): """ Tell a joke to ease the PR tension! """ - joke = await gh.getitem( - "https://official-joke-api.appspot.com/jokes/programming/random" - ) + try: + joke = await gh.getitem( + "https://official-joke-api.appspot.com/jokes/programming/random" + ) + except Exception: + return "To be honest, I haven't heard any good jokes lately." + joke = joke[0] return f"> {joke['setup']}\n *{joke['punchline']}*\n😄️" @@ -57,6 +63,31 @@ def get_style_message(output): """ +def get_style_error_message(e_type, e_value, tb): + """ + Given job failure details, format an error message to post. The + parameters e_type, e_value, and tb (for traceback) should be the same as + returned by sys.exc_info(). + """ + buffer = io.StringIO() + traceback.print_tb(tb, file=buffer) + tb_contents = buffer.getvalue() + buffer.close() + + return f""" +I encountered an error attempting to format style. +
+Details + +```bash +Error: {e_type}, {e_value} +Stack trace: +{tb_contents} +``` +
+""" + + commands_message = f""" You can interact with me in many ways! @@ -70,8 +101,14 @@ def get_style_message(output): If you need help or see there might be an issue with me, open an issue [here](https://github.com/spack/spack-bot/issues) """ -style_message = """ -It looks like you had an issue with style checks! To fix this, you can run: +style_message = f""" +It looks like you had an issue with style checks! I can help with that if you ask me! Just say: + +`{helpers.botname} fix style` + +... and I'll try to fix style and push a commit to your fork with the fix. + +Alternatively, you can run: ```bash $ spack style --fix diff --git a/spackbot/handlers/labels.py b/spackbot/handlers/labels.py index 1e42a6e..e90eaa4 100644 --- a/spackbot/handlers/labels.py +++ b/spackbot/handlers/labels.py @@ -4,10 +4,10 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re -import logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("spackbot") +import spackbot.helpers as helpers + +logger = helpers.get_logger(__name__) #: ``label_patterns`` maps labels to patterns that tell us to apply the labels. @@ -163,7 +163,18 @@ async def add_labels(event, gh): attr_matches = [] # Pattern matches for for each attribute are or'd together for attr, patterns in pattern_dict.items(): - attr_matches.append(any(p.search(file[attr]) for p in patterns)) + # 'patch' is an example of an attribute that is not required to + # appear in response when listing pull request files. See here: + # + # https://docs.github.com/en/rest/pulls/pulls#list-pull-requests-files + # + # If we don't get some attribute in the response, no labels that + # depend on finding a match in that attribute should be added. + attr_matches.append( + any(p.search(file[attr]) for p in patterns) + if attr in file + else False + ) # If all attributes have at least one pattern match, we add the label if all(attr_matches): labels.append(label) diff --git a/spackbot/handlers/pipelines.py b/spackbot/handlers/pipelines.py index 85f1ce2..8934f2a 100644 --- a/spackbot/handlers/pipelines.py +++ b/spackbot/handlers/pipelines.py @@ -3,14 +3,13 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import logging import os import urllib.parse import spackbot.helpers as helpers import aiohttp -logger = logging.getLogger(__name__) +logger = helpers.get_logger(__name__) # We can only make the request with a GITLAB TOKEN GITLAB_TOKEN = os.environ.get("GITLAB_TOKEN") @@ -50,7 +49,7 @@ async def run_pipeline(event, gh): # We need the branch name plus number to assemble the GitLab CI branch = pr["head"]["ref"] - branch = f"github/pr{number}_{branch}" + branch = f"pr{number}_{branch}" branch = urllib.parse.quote_plus(branch) url = f"{helpers.gitlab_spack_project_url}/pipeline?ref={branch}" diff --git a/spackbot/handlers/reviewers.py b/spackbot/handlers/reviewers.py index 813412b..6eb0a43 100644 --- a/spackbot/handlers/reviewers.py +++ b/spackbot/handlers/reviewers.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import logging import os import re @@ -13,7 +12,7 @@ import spackbot.comments as comments from gidgethub import BadRequest -logger = logging.getLogger(__name__) +logger = helpers.get_logger(__name__) async def parse_maintainers_from_patch(gh, pull_request): diff --git a/spackbot/handlers/style.py b/spackbot/handlers/style.py index 8c08f1a..347ecfb 100644 --- a/spackbot/handlers/style.py +++ b/spackbot/handlers/style.py @@ -3,14 +3,17 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os + import spackbot.comments as comments import spackbot.helpers as helpers -from sh.contrib import git -import logging -import os -import sh -logger = logging.getLogger(__name__) +from spackbot.workers import fix_style_task, report_style_failure, work_queue + +# If we don't provide a timeout, the default in RQ is 180 seconds +WORKER_JOB_TIMEOUT = int(os.environ.get("WORKER_JOB_TIMEOUT", "21600")) + +logger = helpers.get_logger(__name__) async def style_comment(event, gh): @@ -31,90 +34,32 @@ async def style_comment(event, gh): await gh.post(comments_url, {}, data={"body": comments.style_message}) -def is_up_to_date(output): - """ - A commit can fail if there are no changes! - """ - return "branch is up to date" in output - - -async def fix_style(event, gh): +async def fix_style(event, gh, *args, **kwargs): """ - Respond to a request to fix style. - We first retrieve metadata about the pull request. If the request comes - from anyone with write access to the repository, we commit, and we commit - under the identity of the original person that opened the PR. + Respond to a request to fix style by placing a task in the work queue """ - pr = await gh.getitem(event.data["issue"]["pull_request"]["url"]) - - # Get the sender of the PR - do they have write? - sender = event.data["sender"]["login"] - repository = event.data["repository"] - collaborators_url = repository["collaborators_url"] - - # If they don't have write, we don't allow the command - if not await helpers.found(gh.getitem(collaborators_url, {"collaborator": sender})): - logger.info(f"Not found: {sender}") - return f"Sorry {sender}, I cannot do that for you. Only users with write can make this request!" - - # Tell the user the style fix is going to take a minute or two - message = "Let me see if I can fix that for you! This might take a moment..." - await gh.post(event.data["issue"]["comments_url"], {}, data={"body": message}) - - # Get the username of the original committer - user = pr["user"]["login"] - - # We need the user id if the user is before July 18. 2017 - email = await helpers.get_user_email(gh, user) - - # We need to use the git url with ssh - branch = pr["head"]["ref"] - full_name = pr["head"]["repo"]["full_name"] - fork_url = f"git@github.com:{full_name}.git" - - # At this point, we can clone the repository and make the change - with helpers.temp_dir() as cwd: - - # Clone a fresh spack develop to use for spack style - git("clone", helpers.spack_upstream, "spack-develop") - - spack = sh.Command(f"{cwd}/spack-develop/bin/spack") - - # clone the develop repository to another folder for our PR - git("clone", "spack-develop", "spack") - - os.chdir("spack") - git("config", "user.name", user) - git("config", "user.email", email) - - # This will authenticate the push with the added ssh credentials - git("remote", "add", "upstream", helpers.spack_upstream) - git("remote", "set-url", "origin", fork_url) - - # we're on upstream/develop. Fetch and check out just the PR branch - git("fetch", "origin", f"{branch}:{branch}") - git("checkout", branch) - - # Save the message for the user - res, err = helpers.run_command(spack, ["--color", "never", "style", "--fix"]) - message = comments.get_style_message(res) - - # Commit (allow for no changes) - res, err = helpers.run_command( - git, - ["commit", "-a", "-m", f"[spackbot] updating style on behalf of {user}"], - ) - - # Continue differently if the branch is up to date or not - if is_up_to_date(res): - message += "\nI wasn't able to make any further changes, but please see the message above for remaining issues you can fix locally!" - return message - message += "\nI've updated the branch with isort fixes." - - # Finally, try to push, update the message if permission not allowed - try: - git("push", "origin", branch) - except Exception: - message += "\n\nBut it looks like I'm not able to push to your branch. 😭️ Did you check maintainer can edit when you opened the PR?" - - return message + installation_id = None + + if "installation_id" in kwargs: + installation_id = kwargs["installation_id"] + + job_metadata = { + # This object is attached to job, so we can e.g. access from within the + # job's on_failure callback. + "post_comments_url": event.data["issue"]["comments_url"], + "token": None, + } + + if "token" in kwargs: + job_metadata["token"] = kwargs["token"] + + task_q = work_queue.get_queue() + fix_style_job = task_q.enqueue( + fix_style_task, + event, + installation_id, + job_timeout=WORKER_JOB_TIMEOUT, + meta=job_metadata, + on_failure=report_style_failure, + ) + logger.info(f"Fix style job enqueued: {fix_style_job.id}") diff --git a/spackbot/helpers.py b/spackbot/helpers.py index ff83ea7..38a0636 100644 --- a/spackbot/helpers.py +++ b/spackbot/helpers.py @@ -4,23 +4,24 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from io import StringIO +import aiohttp import contextlib import gidgethub +import json import logging import os import re import tempfile -from gidgethub import aiohttp from datetime import datetime +from io import StringIO +from sh import ErrorReturnCode +from urllib.request import HTTPHandler, Request, build_opener + """Shared function helpers that can be used across routes" """ -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("spackbot") - spack_develop_url = "https://github.com/spack/spack" spack_gitlab_url = "https://gitlab.spack.io" spack_upstream = "git@github.com:spack/spack" @@ -32,12 +33,36 @@ # Bot name can be modified in the environment botname = os.environ.get("SPACKBOT_NAME", "@spackbot") -logging.info(f"bot name is {botname}") # Aliases for spackbot so spackbot doesn't respond to himself aliases = ["spack-bot", "spackbot", "spack-bot-develop", botname] alias_regex = "(%s)" % "|".join(aliases) +__spackbot_log_level = None +__supported_log_levels = ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") + + +def get_logger(name): + global __spackbot_log_level + + if not __spackbot_log_level: + __spackbot_log_level = os.environ.get("SPACKBOT_LOG_LEVEL", "INFO").upper() + + if __spackbot_log_level not in __supported_log_levels: + # Logging not yet configured, so just print this warning + print( + f"WARNING: Unknown log level {__spackbot_log_level}, using INFO instead." + ) + __spackbot_log_level = "INFO" + + logging.basicConfig(level=__spackbot_log_level) + + return logging.getLogger(name) + + +logger = get_logger(__name__) +logger.info(f"bot name is {botname}") + async def list_packages(): """ @@ -46,7 +71,7 @@ async def list_packages(): # Don't provide endpoint with credentials! async with aiohttp.ClientSession() as session: async with session.get( - "https://spack.github.io/packages/data/packages.json" + "https://packages.spack.io/data/packages.json" ) as response: response = await response.json() @@ -112,7 +137,17 @@ def run_command(control, cmd, ok_codes=None): ok_codes = ok_codes or [0, 1] res = StringIO() err = StringIO() - control(*cmd, _out=res, _err=err, _ok_code=ok_codes) + + try: + control(*cmd, _out=res, _err=err, _ok_code=ok_codes) + except ErrorReturnCode as inst: + logger.error(f"cmd {cmd} exited non-zero") + logger.error(f"stdout from {cmd}:") + logger.error(res.getvalue()) + logger.error(f"stderr from {cmd}:") + logger.error(err.getvalue()) + raise inst + return res.getvalue(), err.getvalue() @@ -130,3 +165,55 @@ async def found(coroutine): if e.status_code == 404: return None raise + + +def synchronous_http_request(url, data=None, token=None): + """ + Makes synchronous http request to the provided url, using the token for + authentication. + + Args: + + url: the target of the http request + data: optional dictionary containing request payload data. After stringify + and utf-8 encoding, this is passed directly to urllib.request.Request + constructor. + token: optional, the value to use as the bearer in the auth header + + Returns: + + http response or None if request could not be made + + TODO: The on_failure callback provided at job scheduling time is not + TODO: getting called when it is defined as async. So this is a synchronous + TODO: way using only standard lib calls to do what we do everywhere else by + TODO: awaiting gh api methods. Need to figure out if that is a bug, by design, + TODO: or if I was just doing it wrong. + """ + if not url: + logger.error("No url provided") + return None + + headers = {} + + if token: + headers["Authorization"] = f"Bearer {token}" + + if data: + data = json.dumps(data).encode("utf-8") + + request = Request( + url, + data=data, + headers=headers, + ) + + opener = build_opener(HTTPHandler) + response = opener.open(request) + response_code = response.getcode() + + logger.debug( + f"synchronous_http_request sent request to {url}, response code: {response_code}" + ) + + return response diff --git a/spackbot/routes.py b/spackbot/routes.py index d00f859..d266be7 100644 --- a/spackbot/routes.py +++ b/spackbot/routes.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import logging import re from gidgethub import sansio @@ -16,8 +15,7 @@ from gidgethub import routing from typing import Any -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("spackbot") +logger = helpers.get_logger(__name__) class SpackbotRouter(routing.Router): @@ -94,7 +92,7 @@ async def add_comments(event, gh, *args, session, **kwargs): elif re.search(f"{helpers.botname} fix style", comment, re.IGNORECASE): logger.debug("Responding to request to fix style") - message = await handlers.fix_style(event, gh) + message = await handlers.fix_style(event, gh, *args, **kwargs) # @spackbot commands OR @spackbot help elif re.search(f"{helpers.botname} (commands|help)", comment, re.IGNORECASE): diff --git a/spackbot/workers.py b/spackbot/workers.py new file mode 100644 index 0000000..6027214 --- /dev/null +++ b/spackbot/workers.py @@ -0,0 +1,198 @@ +import os + +import aiohttp +from gidgethub import aiohttp as gh_aiohttp +from sh.contrib import git +import sh + +from redis import Redis +from rq import Queue + +import spackbot.comments as comments +import spackbot.helpers as helpers +from .auth import authenticate_installation, REQUESTER + +logger = helpers.get_logger(__name__) + +REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") +REDIS_PORT = int(os.environ.get("REDIS_PORT", "6379")) + + +class WorkQueue: + def __init__(self): + logger.info(f"WorkQueue creating redis connection ({REDIS_HOST}, {REDIS_PORT})") + self.redis_conn = Redis(host=REDIS_HOST, port=REDIS_PORT) + # Name of queue workers use is defined in "workers/entrypoint.sh" + self.task_q = Queue(name="tasks", connection=self.redis_conn) + + def get_queue(self): + return self.task_q + + +work_queue = WorkQueue() + + +def is_up_to_date(output): + """ + A commit can fail if there are no changes! + """ + return "nothing to commit" in output + + +def report_style_failure(job, connection, type, value, traceback): + """ + Get the api token from the job metadata, use it to post a comment on + the PR containing the excepttion encountered and stack trace. + + """ + user_msg = comments.get_style_error_message(type, value, traceback) + + token = None + if "token" in job.meta: + token = job.meta["token"] + + url = job.meta["post_comments_url"] + data = {"body": user_msg} + + helpers.synchronous_http_request(url, data=data, token=token) + logger.error(user_msg) + + +async def fix_style_task(event, installation_id=None): + """ + We first retrieve metadata about the pull request. If the request comes + from anyone with write access to the repository, we commit, and we commit + under the identity of the original person that opened the PR. + """ + logger.debug(f"fix_style_task, installation_id = {installation_id}") + + token = None + + if installation_id: + token = await authenticate_installation(installation_id) + + if not token: + logger.error("fix_style_task() unable to authenticate installation") + return + + async with aiohttp.ClientSession() as session: + gh = gh_aiohttp.GitHubAPI(session, REQUESTER, oauth_token=token) + + pr_url = event.data["issue"]["pull_request"]["url"] + + pr = await gh.getitem(pr_url) + + logger.debug("GitHub PR") + logger.debug(pr) + + # Get the sender of the PR - do they have write? + sender = event.data["sender"]["login"] + repository = event.data["repository"] + collaborators_url = repository["collaborators_url"] + author = pr["user"]["login"] + + logger.debug( + f"sender = {sender}, repo = {repository}, collabs_url = {collaborators_url}" + ) + + # If they didn't create the PR and don't have write, we don't allow the command + if sender != author and not await helpers.found( + gh.getitem(collaborators_url, {"collaborator": sender}) + ): + msg = f"Sorry {sender}, I cannot do that for you. Only {author} and users with write can make this request!" + await gh.post(event.data["issue"]["comments_url"], {}, data={"body": msg}) + return + + # Tell the user the style fix is going to take a minute or two + message = "Let me see if I can fix that for you!" + await gh.post(event.data["issue"]["comments_url"], {}, data={"body": message}) + + # Get the username of the original committer + user = pr["user"]["login"] + + # We need the user id if the user is before July 18, 2017. See note about why + # here: + # + # https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-email-preferences/setting-your-commit-email-address + # + email = await helpers.get_user_email(gh, user) + + # We need to use the git url with ssh + branch = pr["head"]["ref"] + full_name = pr["head"]["repo"]["full_name"] + fork_url = f"git@github.com:{full_name}.git" + + logger.info( + f"fix_style_task, user = {user}, email = {email}, fork = {fork_url}, branch = {branch}\n" + ) + + # At this point, we can clone the repository and make the change + with helpers.temp_dir() as cwd: + + # Clone a fresh spack develop to use for spack style + git.clone(helpers.spack_upstream, "spack-develop") + + spack = sh.Command(f"{cwd}/spack-develop/bin/spack") + + # clone the develop repository to another folder for our PR + git.clone("spack-develop", "spack") + + os.chdir("spack") + + git.config("user.name", user) + git.config("user.email", email) + + # This will authenticate the push with the added ssh credentials + git.remote("add", "upstream", helpers.spack_upstream) + git.remote("set-url", "origin", fork_url) + + # we're on upstream/develop. Fetch just the PR branch + helpers.run_command(git, ["fetch", "origin", f"{branch}:{branch}"]) + + # check out the PR branch + helpers.run_command(git, ["checkout", branch]) + + # Run the style check and save the message for the user + check_dir = os.getcwd() + res, err = helpers.run_command( + spack, ["--color", "never", "style", "--fix", "--root", check_dir] + ) + logger.debug("spack style [output]") + logger.debug(res) + logger.debug("spack style [error]") + logger.debug(err) + + message = comments.get_style_message(res) + + # Commit (allow for no changes) + res, err = helpers.run_command( + git, + [ + "commit", + "-a", + "-m", + f"[{helpers.botname}] updating style on behalf of {user}", + ], + ) + + # Continue differently if the branch is up to date or not + if is_up_to_date(res): + logger.info("Unable to make any further changes") + message += "\nI wasn't able to make any further changes, but please see the message above for remaining issues you can fix locally!" + await gh.post( + event.data["issue"]["comments_url"], {}, data={"body": message} + ) + return + + message += "\n\nI've updated the branch with isort fixes." + + # Finally, try to push, update the message if permission not allowed + try: + helpers.run_command(git, ["push", "origin", branch]) + except Exception: + logger.error("Unable to push to branch") + message += "\n\nBut it looks like I'm not able to push to your branch. 😭️ Did you check maintainer can edit when you opened the PR?" + + await gh.post( + event.data["issue"]["comments_url"], {}, data={"body": message} + ) diff --git a/workers/Dockerfile b/workers/Dockerfile new file mode 100644 index 0000000..77ce1d5 --- /dev/null +++ b/workers/Dockerfile @@ -0,0 +1,34 @@ +FROM python:3.7 + +COPY workers/requirements.txt /source/requirements.txt + +RUN pip3 install --upgrade pip setuptools wheel && \ + pip3 install -r /source/requirements.txt + +# make the worker trust GitHub's host key (and verify it) +# If GitHub's fingerprint changes, update the code below with a new one: +# https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints +# and rebuild this container. +RUN ssh-keyscan -t rsa github.com 2>/dev/null > github.key \ + && fingerprint=$(ssh-keygen -lf ./github.key | grep -o 'SHA256:[^ ]*') \ + && if [ "$fingerprint" != "SHA256:nThbg6kXUpJWGl7E1IGOCspRomTxdCARLviKw6E5SY8" ]; \ + then echo "GitHub key is invalid!" && exit 1; \ + fi \ + && mkdir -p /root/.ssh \ + && cat ./github.key >> /root/.ssh/known_hosts + +# use identity file in /git_rsa (the mount point for our public/private keys) +RUN \ + echo "Host github.com" >> /root/.ssh/config \ + && echo "IdentityFile /git_rsa/id_rsa" >> /root/.ssh/config + +# ensure /root/.ssh has correct permissions +RUN chmod -R go-rwx /root/.ssh +RUN mkdir -p /git_rsa && chmod -R go-rwx /git_rsa + +# copy app in last so that everything above can be cached +COPY workers/entrypoint.sh /source/entrypoint.sh +COPY spackbot /source/spackbot + +WORKDIR /source +ENTRYPOINT ["/bin/bash", "/source/entrypoint.sh"] diff --git a/workers/entrypoint.sh b/workers/entrypoint.sh new file mode 100644 index 0000000..cea9a6a --- /dev/null +++ b/workers/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Define REDIS_HOST and REDIS_PORT in .env file or k8s deployment. The queue +# from which workers take jobs is given the name "tasks" here. +rq worker -u redis://${REDIS_HOST}:${REDIS_PORT} --with-scheduler tasks diff --git a/workers/requirements.txt b/workers/requirements.txt new file mode 100644 index 0000000..3ebe3e7 --- /dev/null +++ b/workers/requirements.txt @@ -0,0 +1,10 @@ +aiohttp +gidgethub +python_dotenv +rq +sh + +# Add these so we don't wait for install +mypy +flake8 +isort