Add worker deployments for spackbotdev

.github/workflows/custom_docker_builds.yml

@@ -27,7 +27,7 @@ jobs:
           - python-aws-bash
         include:
           - docker-image: gh-gl-sync
-            image-tags: ghcr.io/spack/ci-bridge:0.0.31
+            image-tags: ghcr.io/spack/ci-bridge:0.0.32
           - docker-image: gitlab-api-scrape
             image-tags: ghcr.io/spack/gitlab-api-scrape:0.0.2
           - docker-image: ci-key-rotate

images/gh-gl-sync/SpackCIBridge.py

@@ -3,7 +3,6 @@
 import argparse
 import atexit
 import base64
-import boto3
 from datetime import datetime, timedelta, timezone
 import dateutil.parser
 from github import Github
@@ -20,7 +19,7 @@
 class SpackCIBridge(object):
 
     def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_project="",
-                 disable_status_post=True, sync_draft_prs=False, pr_mirror_bucket=None,
+                 disable_status_post=True, sync_draft_prs=False,
                  main_branch=None, prereq_checks=[]):
         self.gitlab_repo = gitlab_repo
         self.github_project = github_project
@@ -34,7 +33,6 @@ def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_pro
 
         self.post_status = not disable_status_post
         self.sync_draft_prs = sync_draft_prs
-        self.pr_mirror_bucket = pr_mirror_bucket
         self.main_branch = main_branch
         self.currently_running_sha = None
         self.currently_running_url = None
@@ -250,36 +248,6 @@ def gitlab_shallow_fetch(self):
         fetch_args = ["git", "fetch", "-q", "--depth=1", "gitlab"]
         subprocess.run(fetch_args, check=True, stdout=subprocess.PIPE).stdout
 
-    def get_synced_prs(self):
-        """Return a list of PR branches that already exist on GitLab."""
-        self.get_gitlab_pr_branches()
-        synced_prs = []
-        for line in self.gitlab_pr_output.split(b"\n"):
-            if line.find(b"gitlab/") == -1:
-                continue
-            synced_pr = line.strip().replace(b"gitlab/", b"", 1).decode("utf-8")
-            synced_prs.append(synced_pr)
-        print("Synced PRs:")
-        for pr in synced_prs:
-            print("    {0}".format(pr))
-        return synced_prs
-
-    def get_prs_to_delete(self, open_prs, synced_prs):
-        """Find PRs that have already been synchronized to GitLab that are no longer open on GitHub.
-        Return a list of strings in the format of ":<branch_name" that will be used
-        to delete these branches from GitLab.
-        """
-        prs_to_delete = []
-        for synced_pr in synced_prs:
-            if synced_pr not in open_prs:
-                prs_to_delete.append(synced_pr)
-        print("Synced Closed PRs:")
-        closed_refspecs = []
-        for pr in prs_to_delete:
-            print("    {0}".format(pr))
-            closed_refspecs.append(":{0}".format(pr))
-        return closed_refspecs
-
     def get_open_refspecs(self, open_prs):
         """Return lists of refspecs for fetch and push given a list of open PRs."""
         print("Building initial lists of refspecs to fetch and push")
@@ -550,17 +518,6 @@ def create_status_for_commit(self, sha, branch, state, target_url, description):
             print(e_inst)
         print("  {0} -> {1}".format(branch, sha))
 
-    def delete_pr_mirrors(self, closed_refspecs):
-        if closed_refspecs:
-            s3 = boto3.resource("s3")
-            bucket = s3.Bucket(self.pr_mirror_bucket)
-
-            print("Deleting mirrors for closed PRs:")
-            for refspec in closed_refspecs:
-                pr_mirror_key = refspec[1:]
-                print("    deleting {0}".format(pr_mirror_key))
-                bucket.objects.filter(Prefix=pr_mirror_key).delete()
-
     def sync(self):
         """Synchronize pull requests from GitHub as branches on GitLab."""
 
@@ -601,13 +558,6 @@ def sync(self):
             # Get tags on GitHub.
             tags = self.list_github_tags()
 
-            # Retrieve PRs that have already been synced to GitLab.
-            synced_prs = self.get_synced_prs()
-
-            # Find closed PRs that are currently synced.
-            # These will be deleted from GitLab.
-            closed_refspecs = self.get_prs_to_delete(all_open_prs["pr_strings"], synced_prs)
-
             # Get refspecs for open PRs and protected branches.
             open_refspecs, fetch_refspecs = self.get_open_refspecs(open_prs)
             self.update_refspecs_for_protected_branches(protected_branches, open_refspecs, fetch_refspecs)
@@ -616,16 +566,11 @@ def sync(self):
             # Sync open GitHub PRs and protected branches to GitLab.
             self.fetch_github_branches(fetch_refspecs)
             self.build_local_branches(open_prs, protected_branches)
-            if open_refspecs or closed_refspecs:
+            if open_refspecs:
                 print("Syncing to GitLab")
-                push_args = ["git", "push", "--porcelain", "-f", "gitlab"] + closed_refspecs + open_refspecs
+                push_args = ["git", "push", "--porcelain", "-f", "gitlab"] + open_refspecs
                 subprocess.run(push_args, check=True)
 
-            # Clean up per-PR dedicated mirrors for any closed PRs
-            if self.pr_mirror_bucket:
-                print('Cleaning up per-PR mirrors for closed PRs')
-                self.delete_pr_mirrors(closed_refspecs)
-
             # Post pipeline status to GitHub for each open PR, if enabled
             if self.post_status:
                 print('Posting pipeline status for open PRs and protected branches')
@@ -667,7 +612,6 @@ def sync(self):
                            github_project=args.github_project,
                            disable_status_post=args.disable_status_post,
                            sync_draft_prs=args.sync_draft_prs,
-                           pr_mirror_bucket=args.pr_mirror_bucket,
                            main_branch=args.main_branch,
                            prereq_checks=args.prereq_check)
     bridge.setup_ssh(ssh_key_base64)

images/gh-gl-sync/test_SpackCIBridge.py

@@ -136,30 +136,6 @@ def test_list_github_protected_branches(capfd):
     assert expected in out
 
 
-def test_get_synced_prs(capfd):
-    """Test the get_synced_prs method."""
-    bridge = SpackCIBridge.SpackCIBridge()
-    bridge.get_gitlab_pr_branches = lambda *args: None
-    bridge.gitlab_pr_output = b"""
-  gitlab/pr1_example
-  gitlab/pr2_another_try
-    """
-    assert bridge.get_synced_prs() == ["pr1_example", "pr2_another_try"]
-    out, err = capfd.readouterr()
-    assert out == "Synced PRs:\n    pr1_example\n    pr2_another_try\n"
-
-
-def test_get_prs_to_delete(capfd):
-    """Test the get_prs_to_delete method."""
-    open_prs = ["pr3_try_this", "pr4_new_stuff"]
-    synced_prs = ["pr1_first_try", "pr2_different_approach", "pr3_try_this"]
-    bridge = SpackCIBridge.SpackCIBridge()
-    closed_refspecs = bridge.get_prs_to_delete(open_prs, synced_prs)
-    assert closed_refspecs == [":pr1_first_try", ":pr2_different_approach"]
-    out, err = capfd.readouterr()
-    assert out == "Synced Closed PRs:\n    pr1_first_try\n    pr2_different_approach\n"
-
-
 def test_get_open_refspecs():
     """Test the get_open_refspecs and update_refspecs_for_protected_branches methods."""
     open_prs = {

k8s/custom/gh-gl-sync/cron-jobs.yaml

@@ -15,7 +15,7 @@ spec:
           restartPolicy: Never
           containers:
           - name: sync
-            image: ghcr.io/spack/ci-bridge:0.0.31
+            image: ghcr.io/spack/ci-bridge:0.0.32
             imagePullPolicy: IfNotPresent
             resources:
               requests:

k8s/spack/spackbot-spack-io/deployments.yaml

@@ -37,8 +37,10 @@ spec:
           value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com
         - name: REDIS_PORT
           value: "6379"
-        - name: TASK_QUEUE_NAME
+        - name: TASK_QUEUE_SHORT
           value: "tasksproduction"
+        - name: TASK_QUEUE_LONG
+          value: "tasksproduction_ltask"
         - name: WORKER_JOB_TIMEOUT
           value: "600"
         - name: PYTHONUNBUFFERED
@@ -97,18 +99,103 @@ spec:
             memory: 1G
         # Mount secrets to non-existing location
         volumeMounts:
-          - mountPath: "/git_rsa"
-            name: spack-bot-idrsa
-            readOnly: true
+        - mountPath: "/git_rsa"
+          name: spack-bot-idrsa
+          readOnly: true
         env:
         - name: SPACKBOT_LOG_LEVEL
           value: "INFO"
         - name: REDIS_HOST
           value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com
         - name: REDIS_PORT
           value: "6379"
-        - name: TASK_QUEUE_NAME
+        - name: TASK_QUEUE_SHORT
           value: "tasksproduction"
+        - name: TASK_QUEUE_LONG
+          value: "tasksproduction_ltask"
+        - name: WORKER_TASK_QUEUE
+          value: "tasksproduction"
+        - name: WORKER_JOB_TIMEOUT
+          value: "600"
+        - name: PYTHONUNBUFFERED
+          value: "1"
+        - name: GITHUB_APP_IDENTIFIER
+          value: "123749"
+        - name: GITHUB_APP_REQUESTER
+          value: "spackbot"
+        - name: GITLAB_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: spack-bot-credentials
+              key: gitlab_token
+        - name: GITHUB_PRIVATE_KEY
+          valueFrom:
+            secretKeyRef:
+              name: spack-bot-credentials
+              key: github_private_key
+        - name: GITHUB_WEBHOOK_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: spack-bot-credentials
+              key: github_webhook_secret
+      volumes:
+      - name: spack-bot-idrsa
+        secret:
+          secretName: spack-bot-idrsa
+          defaultMode: 0600
+      nodeSelector:
+        spack.io/node-pool: base
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spackbot-lworkers
+  namespace: spack
+  labels:
+    app: spackbot-lworkers
+    svc: workers
+spec:
+  selector:
+    matchLabels:
+      app: spackbot-lworkers
+      svc: workers
+  # replicas cannot exceed 1 until
+  # rq jobs can be properly sync'd
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: spackbot-lworkers
+        svc: workers
+    spec:
+      serviceAccountName: spackbot-spack-io
+      containers:
+      - name: long-task-worker
+        image: "ghcr.io/spack/spackbot-workers:latest"
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 1500m
+            memory: 1G
+        # Mount secrets to non-existing location
+        volumeMounts:
+        - mountPath: "/git_rsa"
+          name: spack-bot-idrsa
+          readOnly: true
+        env:
+        - name: SPACKBOT_LOG_LEVEL
+          value: "INFO"
+        - name: REDIS_HOST
+          value: pr-binary-graduation-task-queue.cev8lh.ng.0001.use1.cache.amazonaws.com
+        - name: REDIS_PORT
+          value: "6379"
+        - name: TASK_QUEUE_SHORT
+          value: "tasksproduction"
+        - name: TASK_QUEUE_LONG
+          value: "tasksproduction_ltask"
+        - name: WORKER_TASK_QUEUE
+          value: "tasksproduction_ltask"
         - name: WORKER_JOB_TIMEOUT
           value: "600"
         - name: PYTHONUNBUFFERED
@@ -133,9 +220,9 @@ spec:
               name: spack-bot-credentials
               key: github_webhook_secret
       volumes:
-        - name: spack-bot-idrsa
-          secret:
-            secretName: spack-bot-idrsa
-            defaultMode: 0600
+      - name: spack-bot-idrsa
+        secret:
+          secretName: spack-bot-idrsa
+          defaultMode: 0600
       nodeSelector:
         spack.io/node-pool: base

k8s/spack/spackbot-spack-io/service-accounts.yaml

@@ -5,4 +5,4 @@ metadata:
   name: spackbot-spack-io
   namespace: spack
   annotations:
-    eks.amazonaws.com/role-arn: arn:aws:iam::588562868276:role/DeleteObjectsFromBucketSpackBinariesPRs
+    eks.amazonaws.com/role-arn: arn:aws:iam::588562868276:role/FullCRUDAccessToBucketSpackBinariesPRs

k8s/spack/spackbotdev-spack-io/README.md

@@ -18,7 +18,8 @@ Now the development/test work cycle proceeds like this:
 3. edit `k8s/spack/spackbotdev-spack-io/deployments` and update the `image` of the `spec[template][spec][containers]` elements to refer to the approprate test image tags
 4. run `kubectl apply -f k8s/spack/spackbotdev-spack-io/deployments`
 
-Now you can make a new PR from your fork of `spack-test/spack` or comment on some existing one.
+Now you can make a new PR from your fork of `spack-test/spack` or comment on some existing one. Make sure to force push the current develop to `spack-test/spack` before testing to
+avoid accidently tagging all maintainers in spack on the new/updated PRs.
 
 Once you are happy with your changes to `spack/spackbot`, you can merge your PR on `spack/spackbot`, and updating `main` branch there will trigger new images to be tagged `latest` and pushed.  Once the tag is updated in the registry, undo your edits to the `image` tags in the development version of spackbot (replace tags with the values in production spackbot).