diff --git a/.env-dummy b/.env-dummy index ad1afc3..7b3f67d 100644 --- a/.env-dummy +++ b/.env-dummy @@ -7,8 +7,22 @@ REDIS_HOST=rq-server # Optionally customize redis port REDIS_PORT=6379 -# Optionally customize the name of the task queue -TASK_QUEUE_NAME=devtasks +# Base of pr binaries mirror +PR_BINARIES_MIRROR_BASE_URL=s3://spack-binaries-prs/testing + +# Name of expected base branch (we react to PRs merged to this branch) +PR_BINARIES_BASE_BRANCH=develop + +# Number of days to keep shared PR binaries in mirror +SHARED_PR_MIRROR_RETIRE_AFTER_DAYS=7 + +# Optionally customize the name of the task queues +TASK_QUEUE_SHORT=devtasks +TASK_QUEUE_LONG=devtasks_long + +# Set the worker task queue name, should match either the TASK_QUEUE_SHORT or TASK_QUEUE_LONG name +# this is done in the docker-compose.yml +# WORKER_TASK_QUEUE=devtasks # Optionally configure time before jobs are killed and marked failed (in seconds, default 180s) WORKER_JOB_TIMEOUT=21600 diff --git a/docker-compose.yml b/docker-compose.yml index 93aa305..072a264 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,9 @@ services: restart: always env_file: - ./.env + environment: + - TASK_QUEUE_SHORT=devtasks + - TASK_QUEUE_LONG=devtasks_long build: context: . dockerfile: smee/Dockerfile @@ -12,6 +15,23 @@ services: build: context: . dockerfile: workers/Dockerfile + environment: + - TASK_QUEUE_SHORT=devtasks + - TASK_QUEUE_LONG=devtasks_long + - WORKER_TASK_QUEUE=devtasks + env_file: + - ./.env + deploy: + replicas: 1 + + rq-long-worker: + build: + context: . + dockerfile: workers/Dockerfile + environment: + - TASK_QUEUE_SHORT=devtasks + - TASK_QUEUE_LONG=devtasks_long + - WORKER_TASK_QUEUE=devtasks_long env_file: - ./.env deploy: diff --git a/redeploy-spackbot-dev.sh b/redeploy-spackbot-dev.sh new file mode 100755 index 0000000..2c0830d --- /dev/null +++ b/redeploy-spackbot-dev.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e +set -x + +project_root=$(pwd) + +gh_user=kwryankrattiger +image_tag=0.0.1 + +# Rebuild images +docker build -f ${project_root}/Dockerfile -t ghcr.io/${gh_user}/spackbot:${image_tag} ${project_root} +docker build -f ${project_root}/workers/Dockerfile -t ghcr.io/${gh_user}/spackbot-workers:${image_tag} ${project_root} +docker push ghcr.io/${gh_user}/spackbot:${image_tag} +docker push ghcr.io/${gh_user}/spackbot-workers:${image_tag} + +# Rollout with the new containers +kubectl rollout restart -n spack deployments/spackbotdev-spack-io +kubectl rollout restart -n spack deployments/spackbotdev-workers +kubectl rollout restart -n spack deployments/spackbotdev-lworkers diff --git a/requirements.txt b/requirements.txt index eb0583b..4895f60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ gidgethub python_dotenv rq sh +aws diff --git a/spackbot/handlers/__init__.py b/spackbot/handlers/__init__.py index 519f9f2..26fa1e7 100644 --- a/spackbot/handlers/__init__.py +++ b/spackbot/handlers/__init__.py @@ -1,5 +1,10 @@ -from .pipelines import run_pipeline, run_pipeline_rebuild_all # noqa +from .gitlab import ( # noqa + run_pipeline, + run_pipeline_rebuild_all, + close_pr_gitlab_branch, +) from .labels import add_labels # noqa from .reviewers import add_reviewers, add_issue_maintainers # noqa from .reviewers import add_reviewers # noqa from .style import style_comment, fix_style # noqa +from .mirrors import close_pr_mirror # noqa diff --git a/spackbot/handlers/pipelines.py b/spackbot/handlers/gitlab.py similarity index 71% rename from spackbot/handlers/pipelines.py rename to spackbot/handlers/gitlab.py index b0d399f..33019ff 100644 --- a/spackbot/handlers/pipelines.py +++ b/spackbot/handlers/gitlab.py @@ -4,11 +4,13 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spackbot.helpers as helpers +import os from spackbot.workers import ( run_pipeline_task, report_rebuild_failure, - work_queue, + get_queue, + TASK_QUEUE_SHORT, WORKER_JOB_TIMEOUT, ) @@ -25,7 +27,7 @@ async def run_pipeline_rebuild_all(event, gh, **kwargs): "token": kwargs["token"], } - task_q = work_queue.get_queue() + task_q = get_queue(TASK_QUEUE_SHORT) scheduled_job = task_q.enqueue( run_pipeline_task, event, @@ -45,7 +47,7 @@ async def run_pipeline(event, gh, **kwargs): "token": kwargs["token"], } - task_q = work_queue.get_queue() + task_q = get_queue(TASK_QUEUE_SHORT) scheduled_job = task_q.enqueue( run_pipeline_task, event, @@ -54,3 +56,19 @@ async def run_pipeline(event, gh, **kwargs): on_failure=report_rebuild_failure, ) logger.info(f"Run pipeline job enqueued: {scheduled_job.id}") + + +async def close_pr_gitlab_branch(event, gh): + payload = event.data + + pr_number = payload["number"] + pr_branch = payload["pull_request"]["head"]["ref"] + pr_branch_name = f"pr{pr_number}_{pr_branch}" + + url = helpers.gitlab_spack_project_url + url = f"{url}/repository/branches/{pr_branch_name}" + + GITLAB_TOKEN = os.environ.get("GITLAB_TOKEN") + headers = {"PRIVATE-TOKEN": GITLAB_TOKEN} + + await helpers.delete(url, headers=headers) diff --git a/spackbot/handlers/mirrors.py b/spackbot/handlers/mirrors.py new file mode 100644 index 0000000..40b77fb --- /dev/null +++ b/spackbot/handlers/mirrors.py @@ -0,0 +1,108 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spackbot.helpers as helpers +from spackbot.helpers import ( + pr_expected_base, + pr_mirror_base_url, + pr_shared_mirror, + publish_mirror_base_url, +) +from spackbot.workers import ( + copy_pr_mirror, + prune_mirror_duplicates, + update_mirror_index, + delete_pr_mirror, + get_queue, + TASK_QUEUE_LONG, +) + +# If we don't provide a timeout, the default in RQ is 180 seconds +WORKER_JOB_TIMEOUT = 6 * 60 * 60 + +logger = helpers.get_logger(__name__) + + +async def close_pr_mirror(event, gh): + payload = event.data + + # This should only be called when a PR is closed + if not payload["pull_request"]["state"] == "closed": + return + + # Get PR event info + base_branch = payload["pull_request"]["base"]["ref"] + is_merged = payload["pull_request"]["merged"] + pr_number = payload["number"] + pr_branch = payload["pull_request"]["head"]["ref"] + + pr_mirror_url = f"{pr_mirror_base_url}/pr{pr_number}_{pr_branch}" + shared_pr_mirror_url = f"{pr_mirror_base_url}/{pr_shared_mirror}" + + # Get task queue info + ltask_q = get_queue(TASK_QUEUE_LONG) + copy_job = None + job_metadata = { + "type": None, + "pr_number": pr_number, + "pr_branch": pr_branch, + } + + if is_merged and base_branch == pr_expected_base: + logger.info( + f"PR {pr_number}/{pr_branch} merged to develop, graduating binaries" + ) + + # Copy all of the stack binaries from the PR to the shared PR + # mirror. + job_metadata.update({"type": "copy"}) + copy_job = ltask_q.enqueue( + copy_pr_mirror, + pr_mirror_url, + shared_pr_mirror_url, + meta=job_metadata, + job_timeout=WORKER_JOB_TIMEOUT, + ) + logger.info(f"Copy job queued: {copy_job.id}") + + # Prune duplicates that have been published after copy + # since copy may have introduced duplicates for some reason + job_metadata.update({"type": "prune"}) + shared_stack_pr_mirror_url = f"{shared_pr_mirror_url}/{{stack}}" + publish_stack_mirror_url = ( + f"{publish_mirror_base_url}/{{stack}}/{pr_expected_base}" + ) + prune_job = ltask_q.enqueue( + prune_mirror_duplicates, + shared_stack_pr_mirror_url, + publish_stack_mirror_url, + job_timeout=WORKER_JOB_TIMEOUT, + depends_on=copy_job, + meta=job_metadata, + ) + logger.info(f"Pruning job queued: {prune_job.id}") + + # Queue a reindex for the stack mirror to attempt to run after + # prune. + job_metadata.update({"type": "reindex"}) + update_job = ltask_q.enqueue( + update_mirror_index, + shared_stack_pr_mirror_url, + job_timeout=WORKER_JOB_TIMEOUT, + depends_on=prune_job, + meta=job_metadata, + ) + logger.info(f"Reindex job queued: {update_job.id}") + + # Delete the mirror + job_metadata.update({"type": "delete"}) + del_job = ltask_q.enqueue( + delete_pr_mirror, + pr_mirror_url, + meta=job_metadata, + job_timeout=WORKER_JOB_TIMEOUT, + depends_on=copy_job, + ) + logger.info(f"Delete job queued: {del_job.id}") diff --git a/spackbot/handlers/style.py b/spackbot/handlers/style.py index 9c78917..e385677 100644 --- a/spackbot/handlers/style.py +++ b/spackbot/handlers/style.py @@ -9,8 +9,9 @@ from spackbot.workers import ( fix_style_task, report_style_failure, - work_queue, + get_queue, WORKER_JOB_TIMEOUT, + TASK_QUEUE_SHORT, ) @@ -46,7 +47,7 @@ async def fix_style(event, gh, *args, **kwargs): "token": kwargs["token"], } - task_q = work_queue.get_queue() + task_q = get_queue(TASK_QUEUE_SHORT) fix_style_job = task_q.enqueue( fix_style_task, event, diff --git a/spackbot/helpers.py b/spackbot/helpers.py index 37543fd..e8f73be 100644 --- a/spackbot/helpers.py +++ b/spackbot/helpers.py @@ -17,6 +17,7 @@ from io import StringIO from sh import ErrorReturnCode from urllib.request import HTTPHandler, Request, build_opener +from urllib.parse import urlparse """Shared function helpers that can be used across routes" @@ -37,7 +38,16 @@ botname = os.environ.get("SPACKBOT_NAME", "@spackbot") # Bucket where pr binary mirrors live -pr_mirror_bucket = "spack-binaries-prs" +pr_mirror_base_url = os.environ.get( + "PR_BINARIES_MIRROR_BASE_URL", "s3://spack-binaries-prs" +) +shared_pr_mirror_retire_after_days = os.environ.get( + "SHARED_PR_MIRROR_RETIRE_AFTER_DAYS", 7 +) +pr_shared_mirror = "shared_pr_mirror" +pr_expected_base = os.environ.get("PR_BINARIES_BASE_BRANCH", "develop") + +publish_mirror_base_url = "s3://spack-binaries" # Aliases for spackbot so spackbot doesn't respond to himself aliases = ["spack-bot", "spackbot", "spack-bot-develop", botname] @@ -182,6 +192,16 @@ async def post(url, headers): return await response.json() +async def delete(url, headers): + """ + Convenience method to create a new session and make a one-off + delete request, given a url and headers to include in the request. + """ + async with aiohttp.ClientSession() as session: + async with session.delete(url, headers=headers) as response: + return await response.json() + + def synchronous_http_request(url, data=None, token=None): """ Makes synchronous http request to the provided url, using the token for @@ -232,3 +252,23 @@ def synchronous_http_request(url, data=None, token=None): ) return response + + +def s3_parse_url(url, default_bucket="spack-binaries-prs", default_prefix="dummy"): + parsed = { + "bucket": default_bucket, + "prefix": default_prefix, + } + + if type(url) == str: + url = urlparse(url) + + if url.scheme == "s3": + parsed.update( + { + "bucket": url.netloc, + "prefix": url.path.strip("/"), + } + ) + + return parsed diff --git a/spackbot/routes.py b/spackbot/routes.py index 78f54e4..7018d6c 100644 --- a/spackbot/routes.py +++ b/spackbot/routes.py @@ -130,3 +130,13 @@ async def label_pull_requests(event, gh, *args, session, **kwargs): Add labels to PRs based on which files were modified. """ await handlers.add_labels(event, gh) + + +@router.register("pull_request", action="closed") +async def on_closed_pull_request(event, gh, *args, session, **kwargs): + """ + Respond to the pull request closed + """ + await handlers.close_pr_gitlab_branch(event, gh) + + await handlers.close_pr_mirror(event, gh) diff --git a/spackbot/workers.py b/spackbot/workers.py index 09a37b6..27879a0 100644 --- a/spackbot/workers.py +++ b/spackbot/workers.py @@ -8,14 +8,16 @@ import aiohttp import boto3 +from datetime import datetime from gidgethub import aiohttp as gh_aiohttp +import re + from sh.contrib import git import sh from redis import Redis from rq import get_current_job, Queue - import spackbot.comments as comments import spackbot.helpers as helpers from .auth import REQUESTER @@ -24,7 +26,9 @@ REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") REDIS_PORT = int(os.environ.get("REDIS_PORT", "6379")) -TASK_QUEUE_NAME = os.environ.get("TASK_QUEUE_NAME", "tasks") +TASK_QUEUE_SHORT = os.environ.get("TASK_QUEUE_SHORT", "tasks") +TASK_QUEUE_LONG = os.environ.get("TASK_QUEUE_LONG", "tasks_long") +QUERY_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" # If we don't provide a timeout, the default in RQ is 180 seconds WORKER_JOB_TIMEOUT = int(os.environ.get("WORKER_JOB_TIMEOUT", "21600")) @@ -32,19 +36,11 @@ # We can only make the pipeline request with a GITLAB TOKEN GITLAB_TOKEN = os.environ.get("GITLAB_TOKEN") +redis = Redis(host=REDIS_HOST, port=REDIS_PORT) -class WorkQueue: - def __init__(self): - logger.info(f"WorkQueue creating redis connection ({REDIS_HOST}, {REDIS_PORT})") - self.redis_conn = Redis(host=REDIS_HOST, port=REDIS_PORT) - # Name of queue workers use is defined in "workers/entrypoint.sh" - self.task_q = Queue(name=TASK_QUEUE_NAME, connection=self.redis_conn) - - def get_queue(self): - return self.task_q - -work_queue = WorkQueue() +def get_queue(name): + return Queue(name=name, connection=redis) def is_up_to_date(output): @@ -165,13 +161,16 @@ async def run_pipeline_task(event): url = f"{url}&variables[][key]=SPACK_PRUNE_UP_TO_DATE&variables[][value]=False" logger.info( - f"Deleting s3://{helpers.pr_mirror_bucket}/{pr_mirror_key} for rebuild request by {sender}" + f"Deleting {helpers.pr_mirror_base_url}/{pr_mirror_key} for rebuild request by {sender}" ) + pr_url = helpers.s3_parse_url( + f"{helpers.pr_mirror_base_url}/{pr_mirror_key}" + ) # Wipe out PR binary mirror contents s3 = boto3.resource("s3") - bucket = s3.Bucket(helpers.pr_mirror_bucket) - bucket.objects.filter(Prefix=pr_mirror_key).delete() + bucket = s3.Bucket(pr_url.get("bucket")) + bucket.objects.filter(Prefix=pr_url.get("prefix")).delete() headers = {"PRIVATE-TOKEN": GITLAB_TOKEN} @@ -328,3 +327,273 @@ async def fix_style_task(event): await gh.post( event.data["issue"]["comments_url"], {}, data={"body": message} ) + + +async def copy_pr_mirror(pr_mirror_url, shared_pr_mirror_url): + """Copy between the S3 per-pr mirror and the shared pr mirror. + + Parameters: + pr_mirror_url (string): URL to S3 mirror for a PR + shared_pr_mirror_url (string): URL to S3 mirror for shared PR binaries + """ + pr_url = helpers.s3_parse_url(pr_mirror_url) + shared_pr_url = helpers.s3_parse_url(shared_pr_mirror_url) + + s3 = boto3.resource("s3") + pr_bucket_name = pr_url.get("bucket") + pr_bucket = s3.Bucket(pr_bucket_name) + pr_mirror_prefix = pr_url.get("prefix") + + shared_pr_bucket = s3.Bucket(shared_pr_url.get("bucket")) + shared_pr_mirror_prefix = shared_pr_url.get("prefix") + + # Files extensions to copy + extensions = (".spack", ".spec.json", ".spec.yaml", ".spec.json.sig") + + for obj in pr_bucket.objects.filter(Prefix=pr_mirror_prefix): + if obj.key.endswith(extensions): + # Create a new opject replacing the first instance of the pr_mirror_prefix + # with the shared_pr_mirror_prefix. + new_obj = shared_pr_bucket.Object( + obj.key.replace(pr_mirror_prefix, shared_pr_mirror_prefix, 1) + ) + # Copy the PR mirror object to the new object in the shared PR mirror + new_obj.copy( + { + "Bucket": pr_bucket_name, + "Key": obj.key, + } + ) + + +async def delete_pr_mirror(pr_mirror_url): + """Delete a mirror from S3. This routine was written for PR mirrors + but is general enough to be used to delete any S3 mirror. + + Parameters: + pr_mirror_url (string): URL to S3 mirror + """ + pr_url = helpers.s3_parse_url(pr_mirror_url) + + s3 = boto3.resource("s3") + pr_bucket = s3.Bucket(pr_url.get("bucket")) + pr_mirror_prefix = pr_url.get("prefix") + pr_bucket.objects.filter(Prefix=pr_mirror_prefix).delete() + + +def list_ci_stacks(spack_root): + """Loop through the CI stacks in the spack repo. + + Parameters: + spack_root (path): Root of a spack clone + """ + pipeline_root = f"{spack_root}/share/spack/gitlab/cloud_pipelines/stacks/" + for stack in os.listdir(pipeline_root): + if os.path.isfile(f"{pipeline_root}/{stack}/spack.yaml"): + yield stack + + +def hash_from_key(key): + """This works because we guarentee the hash is in the key string. + If this assumption is ever broken, this code will break. + + Parameters: + key (string): File/Object name that contains a spack package concrete + hash. + """ + h = None + # hash is 32 chars long between a "-" and a "." + # examples include: + # linux-ubuntu18.04-x86_64-gcc-8.4.0-armadillo-10.5.0-gq3ijjrtnzgpm4bvuamjr6wa7hzxkypz.spack + # linux-ubuntu18.04-x86_64-gcc-8.4.0-armadillo-10.5.0-gq3ijjrtnzgpm4bvuamjr6wa7hzxkypz.spec.json + h = re.findall(r"-([a-zA-Z0-9]{32,32})\.", key.lower()) + if len(h) > 1: + # Error, multiple matches are ambigious + h = None + elif h: + h = h[0] + return h + + +def check_skip_job(job=None): + """Check if there is another job in the queue that is of the same type. + + Parameters: + job (rq.Job): Job to check (default=rq.get_current_job) + """ + + if not job: + job = get_current_job() + + job_type = job.meta.get("type", "-") + skip = False + logger.debug(f"-- Checking skip job({job.id}): {job_type}") + # Check if another job of this type is queued + queue = get_queue(job.origin) + for _job in queue.jobs: + _job_type = _job.meta["type"] + logger.debug(f"-- job({_job.id}): {_job_type}") + if _job.meta["type"] == job_type: + skip = True + break + + if skip: + logger.debug(f"Skipping {job_type} job") + pr_number = job.meta.get("pr_number", None) + if pr_number: + logger.debug(f"PR: https://github.com/spack/spack/pull/{pr_number}") + + return skip + + +# Prune per stack mirror +async def prune_mirror_duplicates(shared_pr_mirror_url, publish_mirror_url): + """Prune objects from the S3 mirror for shared PR binaries that have been published to the + develop mirror or have expired. + + Parameters: + shared_pr_mirror_url (string): URL to S3 mirror for shared PR binaries + publish_mirror_url (string): URL to S3 mirror for published PR binaries + """ + + # Current job stack + if check_skip_job(): + return + + s3 = boto3.resource("s3") + + with helpers.temp_dir() as cwd: + git.clone( + "--branch", + helpers.pr_expected_base, + "--depth", + 1, + helpers.spack_upstream, + "spack", + ) + + for stack in list_ci_stacks(f"{cwd}/spack"): + shared_pr_url = helpers.s3_parse_url( + shared_pr_mirror_url.format_map({"stack": stack}) + ) + shared_pr_bucket_name = shared_pr_url.get("bucket") + shared_pr_bucket = s3.Bucket(shared_pr_bucket_name) + shared_pr_mirror_prefix = shared_pr_url.get("prefix") + + publish_url = helpers.s3_parse_url( + publish_mirror_url.format_map({"stack": stack}) + ) + publish_bucket = s3.Bucket(publish_url.get("bucket")) + publish_mirror_prefix = publish_url.get("prefix") + + # All of the expected possible spec file extensions + extensions = (".spec.json", ".spec.yaml", ".spec.json.sig") + + # Get the current time for age based pruning + now = datetime.now() + delete_specs = set() + shared_pr_specs = set() + for obj in shared_pr_bucket.objects.filter( + Prefix=shared_pr_mirror_prefix, + ): + # Need to convert from aware to naive time to get delta + last_modified = obj.last_modified.replace(tzinfo=None) + # Prune obj.last_modified > helpers.shared_pr_mirror_retire_after_days + # (default: 7) days to avoid storing cached objects that only + # existed during development. + # Anything older than the retirement age should just be indesciminately + # pruned + if ( + now - last_modified + ).days >= helpers.shared_pr_mirror_retire_after_days: + logger.debug( + f"pr mirror pruning {obj.key} from s3://{shared_pr_bucket_name}: " + "reason(age)" + ) + obj.delete() + + # Grab the hash from the object, to ensure all of the files associated with + # it are also removed. + spec_hash = hash_from_key(obj.key) + if spec_hash: + delete_specs.add(spec_hash) + continue + + if not obj.key.endswith(extensions): + continue + + # Get the hashes in the shared PR bucket. + spec_hash = hash_from_key(obj.key) + if spec_hash: + shared_pr_specs.add(spec_hash) + else: + logger.error( + f"Encountered spec file without hash in name: {obj.key}" + ) + + # Check in the published base branch bucket for duplicates to delete + for obj in publish_bucket.objects.filter( + Prefix=publish_mirror_prefix, + ): + if not obj.key.endswith(extensions): + continue + + spec_hash = hash_from_key(obj.key.lower()) + if spec_hash in shared_pr_specs: + delete_specs.add(spec_hash) + + # Also look at the .spack files for deletion + extensions = (".spack", *extensions) + + # Delete all of the objects with marked hashes + for obj in shared_pr_bucket.objects.filter( + Prefix=shared_pr_mirror_prefix, + ): + if not obj.key.endswith(extensions): + continue + + if hash_from_key(obj.key) in delete_specs: + logger.debug( + f"pr mirror pruning {obj.key} from s3://{shared_pr_bucket_name}: " + "reason(published)" + ) + obj.delete() + + +# Upate index per stack mirror +async def update_mirror_index(base_mirror_url): + """Use spack buildcache command to update index for each Spack CI stack mirror. + + Parameters: + base_mirror_url (string): Base URL to S3 mirror with the format placeholder {stack} + where the stack name will go in the URL. + """ + + # Current job stack + if check_skip_job(): + return + + with helpers.temp_dir() as cwd: + git.clone( + "--branch", + helpers.pr_expected_base, + "--depth", + 1, + helpers.spack_upstream, + "spack", + ) + spack = sh.Command(f"{cwd}/spack/bin/spack") + + for stack in list_ci_stacks(f"{cwd}/spack"): + stack_mirror_url = base_mirror_url.format_map({"stack": stack}) + print(f"Updating binary index at {stack_mirror_url}") + helpers.run_command( + spack, + [ + "-d", + "buildcache", + "update-index", + "--mirror-url", + f"{stack_mirror_url}", + ], + ) diff --git a/workers/Dockerfile b/workers/Dockerfile index 77ce1d5..7ce4dbe 100644 --- a/workers/Dockerfile +++ b/workers/Dockerfile @@ -5,6 +5,10 @@ COPY workers/requirements.txt /source/requirements.txt RUN pip3 install --upgrade pip setuptools wheel && \ pip3 install -r /source/requirements.txt +COPY workers/install_aws.sh /source/install_aws.sh + +RUN . /source/install_aws.sh + # make the worker trust GitHub's host key (and verify it) # If GitHub's fingerprint changes, update the code below with a new one: # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints diff --git a/workers/entrypoint.sh b/workers/entrypoint.sh index b743ed4..52a3b95 100644 --- a/workers/entrypoint.sh +++ b/workers/entrypoint.sh @@ -4,4 +4,4 @@ # deployment. REDIS_HOST and REDIS_PORT define the hostname/ip and port # of the redis instance, while TASK_QUEUE_NAME defines the name of the # queue used for communication between the webservice and workers. -rq worker -u redis://${REDIS_HOST}:${REDIS_PORT} --with-scheduler ${TASK_QUEUE_NAME} +rq worker -u redis://${REDIS_HOST}:${REDIS_PORT} --with-scheduler ${WORKER_TASK_QUEUE} diff --git a/workers/install_aws.sh b/workers/install_aws.sh new file mode 100644 index 0000000..1c7fc8c --- /dev/null +++ b/workers/install_aws.sh @@ -0,0 +1,39 @@ +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + +echo "-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF2Cr7UBEADJZHcgusOJl7ENSyumXh85z0TRV0xJorM2B/JL0kHOyigQluUG +ZMLhENaG0bYatdrKP+3H91lvK050pXwnO/R7fB/FSTouki4ciIx5OuLlnJZIxSzx +PqGl0mkxImLNbGWoi6Lto0LYxqHN2iQtzlwTVmq9733zd3XfcXrZ3+LblHAgEt5G +TfNxEKJ8soPLyWmwDH6HWCnjZ/aIQRBTIQ05uVeEoYxSh6wOai7ss/KveoSNBbYz +gbdzoqI2Y8cgH2nbfgp3DSasaLZEdCSsIsK1u05CinE7k2qZ7KgKAUIcT/cR/grk +C6VwsnDU0OUCideXcQ8WeHutqvgZH1JgKDbznoIzeQHJD238GEu+eKhRHcz8/jeG +94zkcgJOz3KbZGYMiTh277Fvj9zzvZsbMBCedV1BTg3TqgvdX4bdkhf5cH+7NtWO +lrFj6UwAsGukBTAOxC0l/dnSmZhJ7Z1KmEWilro/gOrjtOxqRQutlIqG22TaqoPG +fYVN+en3Zwbt97kcgZDwqbuykNt64oZWc4XKCa3mprEGC3IbJTBFqglXmZ7l9ywG +EEUJYOlb2XrSuPWml39beWdKM8kzr1OjnlOm6+lpTRCBfo0wa9F8YZRhHPAkwKkX +XDeOGpWRj4ohOx0d2GWkyV5xyN14p2tQOCdOODmz80yUTgRpPVQUtOEhXQARAQAB +tCFBV1MgQ0xJIFRlYW0gPGF3cy1jbGlAYW1hem9uLmNvbT6JAlQEEwEIAD4WIQT7 +Xbd/1cEYuAURraimMQrMRnJHXAUCXYKvtQIbAwUJB4TOAAULCQgHAgYVCgkICwIE +FgIDAQIeAQIXgAAKCRCmMQrMRnJHXJIXEAChLUIkg80uPUkGjE3jejvQSA1aWuAM +yzy6fdpdlRUz6M6nmsUhOExjVIvibEJpzK5mhuSZ4lb0vJ2ZUPgCv4zs2nBd7BGJ +MxKiWgBReGvTdqZ0SzyYH4PYCJSE732x/Fw9hfnh1dMTXNcrQXzwOmmFNNegG0Ox +au+VnpcR5Kz3smiTrIwZbRudo1ijhCYPQ7t5CMp9kjC6bObvy1hSIg2xNbMAN/Do +ikebAl36uA6Y/Uczjj3GxZW4ZWeFirMidKbtqvUz2y0UFszobjiBSqZZHCreC34B +hw9bFNpuWC/0SrXgohdsc6vK50pDGdV5kM2qo9tMQ/izsAwTh/d/GzZv8H4lV9eO +tEis+EpR497PaxKKh9tJf0N6Q1YLRHof5xePZtOIlS3gfvsH5hXA3HJ9yIxb8T0H +QYmVr3aIUes20i6meI3fuV36VFupwfrTKaL7VXnsrK2fq5cRvyJLNzXucg0WAjPF +RrAGLzY7nP1xeg1a0aeP+pdsqjqlPJom8OCWc1+6DWbg0jsC74WoesAqgBItODMB +rsal1y/q+bPzpsnWjzHV8+1/EtZmSc8ZUGSJOPkfC7hObnfkl18h+1QtKTjZme4d +H17gsBJr+opwJw/Zio2LMjQBOqlm3K1A4zFTh7wBC7He6KPQea1p2XAMgtvATtNe +YLZATHZKTJyiqA== +=vYOk +-----END PGP PUBLIC KEY BLOCK-----" > aws.pgp.keyfile.txt +gpg --import aws.pgp.keyfile.txt +curl -o awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig +gpg --verify awscliv2.sig awscliv2.zip + +unzip awscliv2.zip +./aws/install + +rm awscliv2.sig awscliv2.zip