diff --git a/.github/workflows/custom_docker_builds.yml b/.github/workflows/custom_docker_builds.yml index 242fa5a96..778da17e2 100644 --- a/.github/workflows/custom_docker_builds.yml +++ b/.github/workflows/custom_docker_builds.yml @@ -27,7 +27,7 @@ jobs: - python-aws-bash include: - docker-image: gh-gl-sync - image-tags: ghcr.io/spack/ci-bridge:0.0.31 + image-tags: ghcr.io/spack/ci-bridge:0.0.32 - docker-image: gitlab-api-scrape image-tags: ghcr.io/spack/gitlab-api-scrape:0.0.2 - docker-image: ci-key-rotate diff --git a/images/gh-gl-sync/SpackCIBridge.py b/images/gh-gl-sync/SpackCIBridge.py index 070a260ea..1d6a48272 100644 --- a/images/gh-gl-sync/SpackCIBridge.py +++ b/images/gh-gl-sync/SpackCIBridge.py @@ -3,7 +3,6 @@ import argparse import atexit import base64 -import boto3 from datetime import datetime, timedelta, timezone import dateutil.parser from github import Github @@ -20,7 +19,7 @@ class SpackCIBridge(object): def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_project="", - disable_status_post=True, sync_draft_prs=False, pr_mirror_bucket=None, + disable_status_post=True, sync_draft_prs=False, main_branch=None, prereq_checks=[]): self.gitlab_repo = gitlab_repo self.github_project = github_project @@ -34,7 +33,6 @@ def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_pro self.post_status = not disable_status_post self.sync_draft_prs = sync_draft_prs - self.pr_mirror_bucket = pr_mirror_bucket self.main_branch = main_branch self.currently_running_sha = None self.currently_running_url = None @@ -250,36 +248,6 @@ def gitlab_shallow_fetch(self): fetch_args = ["git", "fetch", "-q", "--depth=1", "gitlab"] subprocess.run(fetch_args, check=True, stdout=subprocess.PIPE).stdout - def get_synced_prs(self): - """Return a list of PR branches that already exist on GitLab.""" - self.get_gitlab_pr_branches() - synced_prs = [] - for line in self.gitlab_pr_output.split(b"\n"): - if line.find(b"gitlab/") == -1: - continue - synced_pr = line.strip().replace(b"gitlab/", b"", 1).decode("utf-8") - synced_prs.append(synced_pr) - print("Synced PRs:") - for pr in synced_prs: - print(" {0}".format(pr)) - return synced_prs - - def get_prs_to_delete(self, open_prs, synced_prs): - """Find PRs that have already been synchronized to GitLab that are no longer open on GitHub. - Return a list of strings in the format of ": {1}".format(branch, sha)) - def delete_pr_mirrors(self, closed_refspecs): - if closed_refspecs: - s3 = boto3.resource("s3") - bucket = s3.Bucket(self.pr_mirror_bucket) - - print("Deleting mirrors for closed PRs:") - for refspec in closed_refspecs: - pr_mirror_key = refspec[1:] - print(" deleting {0}".format(pr_mirror_key)) - bucket.objects.filter(Prefix=pr_mirror_key).delete() - def sync(self): """Synchronize pull requests from GitHub as branches on GitLab.""" @@ -601,13 +558,6 @@ def sync(self): # Get tags on GitHub. tags = self.list_github_tags() - # Retrieve PRs that have already been synced to GitLab. - synced_prs = self.get_synced_prs() - - # Find closed PRs that are currently synced. - # These will be deleted from GitLab. - closed_refspecs = self.get_prs_to_delete(all_open_prs["pr_strings"], synced_prs) - # Get refspecs for open PRs and protected branches. open_refspecs, fetch_refspecs = self.get_open_refspecs(open_prs) self.update_refspecs_for_protected_branches(protected_branches, open_refspecs, fetch_refspecs) @@ -616,16 +566,11 @@ def sync(self): # Sync open GitHub PRs and protected branches to GitLab. self.fetch_github_branches(fetch_refspecs) self.build_local_branches(open_prs, protected_branches) - if open_refspecs or closed_refspecs: + if open_refspecs: print("Syncing to GitLab") - push_args = ["git", "push", "--porcelain", "-f", "gitlab"] + closed_refspecs + open_refspecs + push_args = ["git", "push", "--porcelain", "-f", "gitlab"] + open_refspecs subprocess.run(push_args, check=True) - # Clean up per-PR dedicated mirrors for any closed PRs - if self.pr_mirror_bucket: - print('Cleaning up per-PR mirrors for closed PRs') - self.delete_pr_mirrors(closed_refspecs) - # Post pipeline status to GitHub for each open PR, if enabled if self.post_status: print('Posting pipeline status for open PRs and protected branches') @@ -667,7 +612,6 @@ def sync(self): github_project=args.github_project, disable_status_post=args.disable_status_post, sync_draft_prs=args.sync_draft_prs, - pr_mirror_bucket=args.pr_mirror_bucket, main_branch=args.main_branch, prereq_checks=args.prereq_check) bridge.setup_ssh(ssh_key_base64) diff --git a/images/gh-gl-sync/entrypoint.sh b/images/gh-gl-sync/entrypoint.sh deleted file mode 100644 index b9d1c5f40..000000000 --- a/images/gh-gl-sync/entrypoint.sh +++ /dev/null @@ -1,7 +0,0 @@ -export GITLAB_SSH_KEY_BASE64="$( base64 < /secrets/gitlab-ssh-key )" -export GITHUB_TOKEN="$( cat /secrets/github-access-token )" -# TODO: update these repos after testing -python3 /scripts/SpackCIBridge.py \ - 'scottwittenburg/gitlab-ci-tests' \ - 'ssh.gitlab.spack.io' \ - 'scott/gitlab-ci-tests' \ diff --git a/images/gh-gl-sync/test_SpackCIBridge.py b/images/gh-gl-sync/test_SpackCIBridge.py index 5c2020f93..f733841d4 100644 --- a/images/gh-gl-sync/test_SpackCIBridge.py +++ b/images/gh-gl-sync/test_SpackCIBridge.py @@ -136,30 +136,6 @@ def test_list_github_protected_branches(capfd): assert expected in out -def test_get_synced_prs(capfd): - """Test the get_synced_prs method.""" - bridge = SpackCIBridge.SpackCIBridge() - bridge.get_gitlab_pr_branches = lambda *args: None - bridge.gitlab_pr_output = b""" - gitlab/pr1_example - gitlab/pr2_another_try - """ - assert bridge.get_synced_prs() == ["pr1_example", "pr2_another_try"] - out, err = capfd.readouterr() - assert out == "Synced PRs:\n pr1_example\n pr2_another_try\n" - - -def test_get_prs_to_delete(capfd): - """Test the get_prs_to_delete method.""" - open_prs = ["pr3_try_this", "pr4_new_stuff"] - synced_prs = ["pr1_first_try", "pr2_different_approach", "pr3_try_this"] - bridge = SpackCIBridge.SpackCIBridge() - closed_refspecs = bridge.get_prs_to_delete(open_prs, synced_prs) - assert closed_refspecs == [":pr1_first_try", ":pr2_different_approach"] - out, err = capfd.readouterr() - assert out == "Synced Closed PRs:\n pr1_first_try\n pr2_different_approach\n" - - def test_get_open_refspecs(): """Test the get_open_refspecs and update_refspecs_for_protected_branches methods.""" open_prs = { diff --git a/k8s/custom/gh-gl-sync/cron-jobs.yaml b/k8s/custom/gh-gl-sync/cron-jobs.yaml index 0b9be1e88..6521f3265 100644 --- a/k8s/custom/gh-gl-sync/cron-jobs.yaml +++ b/k8s/custom/gh-gl-sync/cron-jobs.yaml @@ -15,7 +15,7 @@ spec: restartPolicy: Never containers: - name: sync - image: ghcr.io/spack/ci-bridge:0.0.31 + image: ghcr.io/spack/ci-bridge:0.0.32 imagePullPolicy: IfNotPresent resources: requests: diff --git a/k8s/spack/spackbot-spack-io/deployments.yaml b/k8s/spack/spackbot-spack-io/deployments.yaml index 976ba69b5..f61321e55 100644 --- a/k8s/spack/spackbot-spack-io/deployments.yaml +++ b/k8s/spack/spackbot-spack-io/deployments.yaml @@ -37,8 +37,10 @@ spec: value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com - name: REDIS_PORT value: "6379" - - name: TASK_QUEUE_NAME + - name: TASK_QUEUE_SHORT value: "tasksproduction" + - name: TASK_QUEUE_LONG + value: "tasksproduction_ltask" - name: WORKER_JOB_TIMEOUT value: "600" - name: PYTHONUNBUFFERED @@ -97,9 +99,9 @@ spec: memory: 1G # Mount secrets to non-existing location volumeMounts: - - mountPath: "/git_rsa" - name: spack-bot-idrsa - readOnly: true + - mountPath: "/git_rsa" + name: spack-bot-idrsa + readOnly: true env: - name: SPACKBOT_LOG_LEVEL value: "INFO" @@ -107,8 +109,93 @@ spec: value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com - name: REDIS_PORT value: "6379" - - name: TASK_QUEUE_NAME + - name: TASK_QUEUE_SHORT value: "tasksproduction" + - name: TASK_QUEUE_LONG + value: "tasksproduction_ltask" + - name: WORKER_TASK_QUEUE + value: "tasksproduction" + - name: WORKER_JOB_TIMEOUT + value: "600" + - name: PYTHONUNBUFFERED + value: "1" + - name: GITHUB_APP_IDENTIFIER + value: "123749" + - name: GITHUB_APP_REQUESTER + value: "spackbot" + - name: GITLAB_TOKEN + valueFrom: + secretKeyRef: + name: spack-bot-credentials + key: gitlab_token + - name: GITHUB_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: spack-bot-credentials + key: github_private_key + - name: GITHUB_WEBHOOK_SECRET + valueFrom: + secretKeyRef: + name: spack-bot-credentials + key: github_webhook_secret + volumes: + - name: spack-bot-idrsa + secret: + secretName: spack-bot-idrsa + defaultMode: 0600 + nodeSelector: + spack.io/node-pool: base + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spackbot-lworkers + namespace: spack + labels: + app: spackbot-lworkers + svc: workers +spec: + selector: + matchLabels: + app: spackbot-lworkers + svc: workers + # replicas cannot exceed 1 until + # rq jobs can be properly sync'd + replicas: 1 + template: + metadata: + labels: + app: spackbot-lworkers + svc: workers + spec: + serviceAccountName: spackbot-spack-io + containers: + - name: long-task-worker + image: "ghcr.io/spack/spackbot-workers:latest" + imagePullPolicy: Always + resources: + requests: + cpu: 1500m + memory: 1G + # Mount secrets to non-existing location + volumeMounts: + - mountPath: "/git_rsa" + name: spack-bot-idrsa + readOnly: true + env: + - name: SPACKBOT_LOG_LEVEL + value: "INFO" + - name: REDIS_HOST + value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com + - name: REDIS_PORT + value: "6379" + - name: TASK_QUEUE_SHORT + value: "tasksproduction" + - name: TASK_QUEUE_LONG + value: "tasksproduction_ltask" + - name: WORKER_TASK_QUEUE + value: "tasksproduction_ltask" - name: WORKER_JOB_TIMEOUT value: "600" - name: PYTHONUNBUFFERED @@ -133,9 +220,9 @@ spec: name: spack-bot-credentials key: github_webhook_secret volumes: - - name: spack-bot-idrsa - secret: - secretName: spack-bot-idrsa - defaultMode: 0600 + - name: spack-bot-idrsa + secret: + secretName: spack-bot-idrsa + defaultMode: 0600 nodeSelector: spack.io/node-pool: base diff --git a/k8s/spack/spackbot-spack-io/service-accounts.yaml b/k8s/spack/spackbot-spack-io/service-accounts.yaml index 382ed7538..a5d46a6e5 100644 --- a/k8s/spack/spackbot-spack-io/service-accounts.yaml +++ b/k8s/spack/spackbot-spack-io/service-accounts.yaml @@ -5,4 +5,4 @@ metadata: name: spackbot-spack-io namespace: spack annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::588562868276:role/DeleteObjectsFromBucketSpackBinariesPRs + eks.amazonaws.com/role-arn: arn:aws:iam::588562868276:role/FullCRUDAccessToBucketSpackBinariesPRs diff --git a/k8s/spack/spackbotdev-spack-io/README.md b/k8s/spack/spackbotdev-spack-io/README.md index e2f9ff490..642b930c8 100644 --- a/k8s/spack/spackbotdev-spack-io/README.md +++ b/k8s/spack/spackbotdev-spack-io/README.md @@ -18,7 +18,8 @@ Now the development/test work cycle proceeds like this: 3. edit `k8s/spack/spackbotdev-spack-io/deployments` and update the `image` of the `spec[template][spec][containers]` elements to refer to the approprate test image tags 4. run `kubectl apply -f k8s/spack/spackbotdev-spack-io/deployments` -Now you can make a new PR from your fork of `spack-test/spack` or comment on some existing one. +Now you can make a new PR from your fork of `spack-test/spack` or comment on some existing one. Make sure to force push the current develop to `spack-test/spack` before testing to +avoid accidently tagging all maintainers in spack on the new/updated PRs. Once you are happy with your changes to `spack/spackbot`, you can merge your PR on `spack/spackbot`, and updating `main` branch there will trigger new images to be tagged `latest` and pushed. Once the tag is updated in the registry, undo your edits to the `image` tags in the development version of spackbot (replace tags with the values in production spackbot). diff --git a/k8s/spack/spackbotdev-spack-io/deployments.yaml b/k8s/spack/spackbotdev-spack-io/deployments.yaml index 5d6bb108a..8452e3ff5 100644 --- a/k8s/spack/spackbotdev-spack-io/deployments.yaml +++ b/k8s/spack/spackbotdev-spack-io/deployments.yaml @@ -23,7 +23,7 @@ spec: - name: web image: "ghcr.io/spack/spack-bot:latest" # Can use some other image for testing, see README.md - # image: "ghcr.io/scottwittenburg/spackbot:0.0.93" + #image: "ghcr.io/kwryankrattiger/spackbot:0.0.1" imagePullPolicy: Always resources: requests: @@ -39,8 +39,14 @@ spec: value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com - name: REDIS_PORT value: "6379" - - name: TASK_QUEUE_NAME + - name: PR_BINARIES_MIRROR_BASE_URL + value: "s3://spack-binaries-prs/testing" + - name: SHARED_PR_BINARIES_RETIRE_AFTER_DAYS + value: "9999" + - name: TASK_QUEUE_SHORT value: "tasksdevelop" + - name: TASK_QUEUE_LONG + value: "tasksdevelop_ltask" - name: WORKER_JOB_TIMEOUT value: "600" - name: GITLAB_SPACK_PROJECT_URL @@ -97,7 +103,7 @@ spec: - name: worker image: "ghcr.io/spack/spackbot-workers:latest" # Can use some other image for testing, see README.md - # image: "ghcr.io/scottwittenburg/spackbot-workers:0.0.93" + #image: "ghcr.io/kwryankrattiger/spackbot-workers:0.0.1" imagePullPolicy: Always resources: requests: @@ -105,17 +111,25 @@ spec: memory: 1G # Mount secrets to non-existing location volumeMounts: - - mountPath: "/git_rsa" - name: spack-bot-dev-idrsa - readOnly: true + - mountPath: "/git_rsa" + name: spack-bot-dev-idrsa + readOnly: true env: + - name: PR_BINARIES_MIRROR_BASE_URL + value: "s3://spack-binaries-prs/testing" + - name: SHARED_PR_BINARIES_RETIRE_AFTER_DAYS + value: "9999" - name: SPACKBOT_LOG_LEVEL value: "DEBUG" - name: REDIS_HOST value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com - name: REDIS_PORT value: "6379" - - name: TASK_QUEUE_NAME + - name: TASK_QUEUE_SHORT + value: "tasksdevelop" + - name: TASK_QUEUE_LONG + value: "tasksdevelop_ltask" + - name: WORKER_TASK_QUEUE value: "tasksdevelop" - name: WORKER_JOB_TIMEOUT value: "600" @@ -145,10 +159,99 @@ spec: name: spack-bot-dev-credentials key: github_webhook_secret volumes: - - name: spack-bot-dev-idrsa - secret: - secretName: spack-bot-dev-idrsa - defaultMode: 0600 + - name: spack-bot-dev-idrsa + secret: + secretName: spack-bot-dev-idrsa + defaultMode: 0600 + nodeSelector: + spack.io/node-pool: base + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spackbotdev-lworkers + namespace: spack + labels: + app: spackbotdev-lworkers + svc: workers +spec: + selector: + matchLabels: + app: spackbotdev-lworkers + svc: workers + # replicas cannot exceed 1 until + # rq jobs can be properly sync'd + replicas: 1 + template: + metadata: + labels: + app: spackbotdev-lworkers + svc: workers + spec: + serviceAccountName: spackbotdev-spack-io + containers: + - name: long-task-worker + image: "ghcr.io/spack/spackbot-workers:latest" + # Can use some other image for testing, see README.md + #image: "ghcr.io/kwryankrattiger/spackbot-workers:0.0.1" + imagePullPolicy: Always + resources: + requests: + cpu: 900m + memory: 1G + # Mount secrets to non-existing location + volumeMounts: + - mountPath: "/git_rsa" + name: spack-bot-dev-idrsa + readOnly: true + env: + - name: PR_BINARIES_MIRROR_BASE_URL + value: "s3://spack-binaries-prs/testing" + - name: SHARED_PR_BINARIES_RETIRE_AFTER_DAYS + value: "9999" + - name: SPACKBOT_LOG_LEVEL + value: "DEBUG" + - name: REDIS_HOST + value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com + - name: REDIS_PORT + value: "6379" + - name: TASK_QUEUE_SHORT + value: "tasksdevelop" + - name: TASK_QUEUE_LONG + value: "tasksdevelop_ltask" + - name: WORKER_TASK_QUEUE + value: "tasksdevelop_ltask" + - name: WORKER_JOB_TIMEOUT + value: "600" + - name: PYTHONUNBUFFERED + value: "1" + - name: GITHUB_APP_IDENTIFIER + value: "126480" + - name: GITHUB_APP_REQUESTER + value: "spack-test" + - name: SPACKBOT_NAME + value: "@spackbot-test" + - name: GITLAB_TOKEN + valueFrom: + secretKeyRef: + name: spack-bot-dev-credentials + key: gitlab_token + - name: GITHUB_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: spack-bot-dev-credentials + key: github_private_key + - name: GITHUB_WEBHOOK_SECRET + valueFrom: + secretKeyRef: + name: spack-bot-dev-credentials + key: github_webhook_secret + volumes: + - name: spack-bot-dev-idrsa + secret: + secretName: spack-bot-dev-idrsa + defaultMode: 0600 nodeSelector: spack.io/node-pool: base diff --git a/k8s/spack/spackbotdev-spack-io/service-accounts.yaml b/k8s/spack/spackbotdev-spack-io/service-accounts.yaml index 8f25f7863..b76c540b9 100644 --- a/k8s/spack/spackbotdev-spack-io/service-accounts.yaml +++ b/k8s/spack/spackbotdev-spack-io/service-accounts.yaml @@ -5,4 +5,4 @@ metadata: name: spackbotdev-spack-io namespace: spack annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::588562868276:role/DeleteObjectsFromBucketSpackBinariesPRs + eks.amazonaws.com/role-arn: arn:aws:iam::588562868276:role/FullCRUDAccessToBucketSpackBinariesPRs