feat: improve garm-provider dev setup and adjust docs for new garm v0.1.5

.gitignore

@@ -3,8 +3,7 @@
 .idea
 /bin/
 /hack/bin/garm-provider-k8s
-/hack/local-development/kubernetes/configmap.yaml
-/hack/local-development/kubernetes/garm-operator-all.yaml
+/hack/local-development/kubernetes/garm-operator-crs.yaml
 
 # Ignore all files in the tmp directory
 # primarly used during the release process

.gitleaksignore

@@ -0,0 +1,3 @@
+# only used for local development
+hack/local-development/kubernetes/garm-operator-all.yaml:generic-api-key:3385
+

DEVELOPMENT.md

@@ -18,107 +18,88 @@ To start developing, you need the following tools installed:
 
 ## Getting Started
 
-1. Get yourself a GitHub PAT for development purposes with access to an Organization where the runners will be registered.
-
-1. To configure `garm` you have to generate a new `config.toml` with your previous created PAT.
-   You can use the `make template` target in the root directory of this repository to generate a new `configMap` which contains a valid `config.toml` for `garm`
+1. Get yourself a GitHub PAT for development purposes with access to a Repository where the runners will be 
+   registered. Your PAT needs the following permissions:
+   If you'll use a PAT (classic), you'll have to grant access for the following scopes. See official [cloudbase/garm](https://github.com/cloudbase/garm/blob/main/doc/github_credentials.md) docs for more information.
+
+   * ```public_repo``` - for access to a repository
+   * ```repo``` - for access to a private repository
+   * ```admin:org``` - if you plan on using this with an organization to which you have access
+   * ```manage_runners:enterprise``` - if you plan to use garm at the enterprise level
+   * ```admin:repo_hook``` - if you want to allow GARM to install webhooks on repositories (optional)
+   * ```admin:org_hook``` - if you want to allow GARM to install webhooks on organizations (optional)
+
+Fine grained PATs are also supported as long as you grant the required privileges:
+
+   * **Repository permissions**:
+     * `Administration: Read & write` - needed to generate JIT config/registration token, remove runners, etc.
+     * `Metadata: Read-only` - automatically enabled by above
+     * `Webhooks: Read & write` - needed to install webhooks on repositories
+  * **Organization permissions**:
+     * `Self-hosted runners: Read & write` - needed to manage runners in an organization
+     * `Webhooks: Read & write` - needed to install webhooks on organizations
+
+1. To spin up GitHub Action Runners with `garm`, the `garm-operator` needs some CRs which can be created by the 
+   following command.
+   You can use the `make template` target in the root directory of this repository to generate a new 
+   `garm-operator-crs.yaml` which contains all CRs required for `garm-operator`.
 
       ```bash
-      GARM_GITHUB_OAUTH_TOKEN=ghp_QVK50pUo25QsJBy5DfA95EyUCzAbG20Q1NP2 make template
+      GARM_GITHUB_ORGANIZATION=my-github-org \
+      GARM_GITHUB_REPOSITORY=my-github-repo \
+      GARM_GITHUB_TOKEN=gha_testtoken \
+      GARM_GITHUB_WEBHOOK_SECRET=supersecret \
+      make template
       ```
 
-1. Start the development environment by running `make tilt-up` in the root directory of this repository.
+1. Start the development environment by running
+   ```bash
+   $ make tilt-up
+   ```
+   in the root directory of this repository.
 
    This will start a local Kubernetes cluster using `kind` (`kind get clusters` will show you a `garm` cluster) and deploy a garm-server with an already registered `garm-provider-k8s`.
    The `make tilt-up` command will give you the URL to the local tilt environment.
 
-1. As we also written the [garm-operator](https://github.com/mercedes-benz/garm-operator) to manage
+2. As we also written the [garm-operator](https://github.com/mercedes-benz/garm-operator) to manage
    `garm` resources (like `Pools`, `Organizations` and so on) we are using the `garm-operator` to bootstrap the local development setup. 
 
-   You will notice that in the `garm-operator-system` namespace the `garm-operator` is running.
-
-1. You are now able to create the `garm` resources by either creating them via `garm-cli` in the `garm-server` container (in `garm-server` namespace) or via the `garm-operator` by applying the following manifest files to your local cluster. 
+   You will notice that in the `garm-operator-system` namespace the `garm-operator` is running, and your previously 
+   created CRs are applied.
+   ```bash
+   $ kubectl get garm -n garm-operator-system
+   ```
+   ``` 
+   NAME                                                                  ID                                     VERSION   AGE
+   garmserverconfig.garm-operator.mercedes-benz.com/garm-server-config   65cc57ab-3a6c-48b9-9565-af7332e65f32   v0.1.5    10m
 
-      **organization:**
-      ```yaml
-      apiVersion: garm-operator.mercedes-benz.com/v1alpha1
-      kind: Organization
-      metadata:
-        labels:
-          app.kubernetes.io/name: organization
-          app.kubernetes.io/instance: organization-sample
-          app.kubernetes.io/part-of: garm-operator
-        name: your-org # this organization must exist and the PAT must have access to it
-        namespace: garm-operator-system
-      spec:
-        webhookSecretRef:
-          key: "webhookSecret"
-          name: "org-webhook-secret"
-        credentialsName: "github-pat"
-      ---
-      apiVersion: v1
-      kind: Secret
-      metadata:
-        name: org-webhook-secret
-        namespace: garm-operator-system
-      data:
-        webhookSecret: bXlzZWNyZXQ=
-      ```
-
-      **image:**
-      ```yaml
-      apiVersion: garm-operator.mercedes-benz.com/v1alpha1
-      kind: Image
-      metadata:
-        labels:
-          app.kubernetes.io/name: image
-          app.kubernetes.io/instance: image-sample
-          app.kubernetes.io/part-of: garm-operator
-        name: runner-default
-        namespace: garm-operator-system
-      spec:
-        tag: localhost:5000/runner:linux-ubuntu-22.04-x86_64
-      ```
-
-      **pool:**
-      ```yaml
-      apiVersion: garm-operator.mercedes-benz.com/v1alpha1
-      kind: Pool
-      metadata:
-        labels:
-          app.kubernetes.io/instance: pool-sample
-          app.kubernetes.io/name: pool
-          app.kubernetes.io/part-of: garm-operator
-        name: k8s-pool
-        namespace: garm-operator-system
-      spec:
-        githubScopeRef:
-          apiGroup: garm-operator.mercedes-benz.com
-          kind: Organization
-          name: your-org
-        enabled: true
-        extraSpecs: "{}"
-        flavor: medium
-        githubRunnerGroup: ""
-        imageName: runner-default
-        maxRunners: 4
-        minIdleRunners: 2
-        osArch: amd64
-        osType: linux
-        providerName: kubernetes_external # this is the name defined in your garm server
-        runnerBootstrapTimeout: 20
-        runnerPrefix: ""
-        tags:
-          - linux
-          - kubernetes
-      ```
-
-      Now you should be able to see two pods in the `runner` namespace:
-      ```bash
-      $ kubectl get pods -n runner
-      NAME                READY   STATUS    RESTARTS   AGE
-      garm-hoyjldsegfal   1/1     Running   0          7s
-      garm-hpohj7g2b4df   1/1     Running   0          7s
-      ```
-
-1. Time to start developing. 🎉
+   NAME                                                          ID    READY   AUTHTYPE   GITHUBENDPOINT   AGE
+   githubcredential.garm-operator.mercedes-benz.com/github-pat   3     True    pat        github           28m
+
+   NAME                                                    URL                      READY   AGE
+   githubendpoint.garm-operator.mercedes-benz.com/github   https://api.github.com   True    28m
+
+   NAME                                                   TAG                                              AGE
+   image.garm-operator.mercedes-benz.com/runner-default   localhost:5000/runner:linux-ubuntu-22.04-arm64   10m
+
+   NAME                                                        ID                                     MINIDLERUNNERS   MAXRUNNERS   READY   AGE
+   pool.garm-operator.mercedes-benz.com/kubernetes-pool-repo   8d8699c3-0acb-4fd5-9f5a-8b60b7504eca   1                2            True    10m
+
+   NAME                                                        ID                                     READY   AGE
+   repository.garm-operator.mercedes-benz.com/test-workflows   3cd28551-18f8-430d-a0c8-49cf24dd7355   True    10m
+
+   NAME                                                           ID                                     POOL                   GARM RUNNER STATUS   PROVIDER RUNNER STATUS   AGE
+   runner.garm-operator.mercedes-benz.com/garm-k8s-vi5mdto8lnav   9e475cad-0e22-4db2-b53b-3d8cb184439e   kubernetes-pool-repo   running              idle                     5m46s
+
+   ```
+3. Also in the `runner` namespace, you should see created runner pods by the `garm-provider-k8s`.
+   ```bash
+   $ kubectl get pods -n runner
+   ```
+   ```
+   NAME                    READY   STATUS    RESTARTS   AGE
+   garm-k8s-vi5mdto8lnav   1/1     Running   0          6m14s
+   ```
+
+4. Everytime you now change some code, tilt will automatically build a new container image with the `garm-server` and
+   `garm-provider-k8s` binary and update the image tag in the k8s Deployment manifest. Time to start developing! 🎉

Makefile

@@ -106,15 +106,27 @@ docker-build-upstream-runner: ## Build the used runner image
 
 .PHONY: template
 template: ## Create the necessary configmap for the local development
-ifeq ($(GARM_GITHUB_OAUTH_TOKEN),)
-	$(error GARM_GITHUB_OAUTH_TOKEN is undefined)
+ifeq ($(GARM_GITHUB_TOKEN),)
+	$(error GARM_GITHUB_TOKEN is undefined)
+endif
+ifeq ($(GARM_GITHUB_REPOSITORY),)
+	$(error GARM_GITHUB_REPOSITORY is undefined)
+endif
+ifeq ($(GARM_GITHUB_WEBHOOK_SECRET),)
+	$(error GARM_GITHUB_WEBHOOK_SECRET is undefined)
+endif
+ifeq ($(GARM_GITHUB_ORGANIZATION),)
+	$(error GARM_GITHUB_ORGANIZATION is undefined)
 endif
-	GARM_GITHUB_NAME=$(GARM_GITHUB_NAME) \
-	GARM_GITHUB_OAUTH_TOKEN=$(GARM_GITHUB_OAUTH_TOKEN) \
 	GARM_GITHUB_BASE_URL=$(GARM_GITHUB_BASE_URL) \
 	GARM_GITHUB_API_BASE_URL=$(GARM_GITHUB_API_BASE_URL) \
 	GARM_GITHUB_UPLOAD_BASE_URL=$(GARM_GITHUB_UPLOAD_BASE_URL) \
-	envsubst < hack/local-development/kubernetes/configmap-envsubst.yaml > hack/local-development/kubernetes/configmap.yaml
+	GARM_GITHUB_REPOSITORY=$(GARM_GITHUB_REPOSITORY) \
+	GARM_GITHUB_ORGANIZATION=$(GARM_GITHUB_ORGANIZATION) \
+	GARM_GITHUB_TOKEN=$(shell echo -n $(GARM_GITHUB_TOKEN) | base64) \
+	GARM_GITHUB_WEBHOOK_SECRET=$(shell echo -n $(GARM_GITHUB_WEBHOOK_SECRET) | base64) \
+	RUNNER_IMAGE=$(shell echo "localhost:5000/runner:linux-ubuntu-22.04-$(shell uname -m)") \
+	envsubst < hack/local-development/kubernetes/garm-operator-crs-envsubst.yaml > hack/local-development/kubernetes/garm-operator-crs.yaml
 
 .PHONY: prepare-operator
 prepare-operator: ## Prepare garm-operator for local development

Tiltfile

@@ -1,30 +1,17 @@
 # SPDX-License-Identifier: MIT
 
-allow_k8s_contexts('garm')
+allow_k8s_contexts('kind-garm')
 
 # we need cert-manager for the garm-operator
 load('ext://cert_manager', 'deploy_cert_manager')
 
 # we use the `cert_manager` extension to deploy cert-manager into the kind-cluster
 # as the plugin has already well written readiness checks we can use it to wait for
 deploy_cert_manager(
-    kind_cluster_name='garm', # just for security reasons ;-)
-    version='v1.12.0' # the version of cert-manager to deploy
+    kind_cluster_name='kind-garm', # just for security reasons ;-)
+    version='v1.15.3' # the version of cert-manager to deploy
 )
 
-# prepare garm-operator
-local_resource(
-    "prepare garm-operator",
-    cmd="make prepare-operator",
-    auto_init=True,
-    trigger_mode=TRIGGER_MODE_MANUAL,
-    labels=["garm-operator"],
-    deps=["."]
-)
-
-k8s_yaml('hack/local-development/kubernetes/garm-operator-all.yaml')
-
-
 # build garm-provider-k8s binary with 'make build copy'
 local_resource(
     "build provider",
@@ -57,3 +44,60 @@ local_resource(
 # take care of the kubernetes manifests where garm with the provider binary is deployed
 templated_yaml = kustomize('hack/local-development/kubernetes')
 k8s_yaml(templated_yaml)
+
+k8s_resource(
+    "garm-server",
+    objects=[
+        'garm-server:Namespace:default',
+        'runner:Namespace:default',
+        'garm-server:ServiceAccount:garm-server',
+        'garm-provider-k8s:ClusterRole:default',
+        'garm-provider-k8s:RoleBinding:runner',
+        'garm-configuration:ConfigMap:garm-server',
+        'garm-kubernetes-provider-config:ConfigMap:garm-server',
+        'garm-data:PersistentVolumeClaim:garm-server',
+        'garm-home:PersistentVolumeClaim:garm-server'
+
+    ],
+    labels=["garm-server"],
+)
+
+# deploy the garm-operator and CRs
+k8s_yaml('hack/local-development/kubernetes/garm-operator-all.yaml')
+
+k8s_yaml('hack/local-development/kubernetes/garm-operator-crs.yaml')
+
+k8s_resource(
+    "garm-operator-controller-manager",
+    objects=[
+        'garm-operator-system:Namespace:default',
+        'enterprises.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'garmserverconfigs.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'githubcredentials.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'githubendpoints.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'images.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'organizations.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'pools.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'repositories.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'runners.garm-operator.mercedes-benz.com:CustomResourceDefinition:default',
+        'garm-operator-controller-manager:ServiceAccount:garm-operator-system',
+        'garm-operator-leader-election-role:Role:garm-operator-system',
+        'garm-operator-manager-role:Role:garm-operator-system',
+        'garm-operator-manager-role:ClusterRole:default',
+        'garm-operator-leader-election-rolebinding:RoleBinding:garm-operator-system',
+        'garm-operator-manager-rolebinding:RoleBinding:garm-operator-system',
+        'garm-operator-kube-state-metrics-config:ConfigMap:garm-operator-system',
+        'garm-operator-serving-cert:Certificate:garm-operator-system',
+        'garm-operator-selfsigned-issuer:Issuer:garm-operator-system',
+        'garm-operator-validating-webhook-configuration:ValidatingWebhookConfiguration:default',
+        'garm-server-config:GarmServerConfig:garm-operator-system',
+        'github:GitHubEndpoint:garm-operator-system',
+        'github-pat:GitHubCredential:garm-operator-system',
+        'github-pat:Secret:garm-operator-system',
+        'test-workflows:Repository:garm-operator-system',
+        'repo-webhook-secret:Secret:garm-operator-system',
+        'runner-default:Image:garm-operator-system',
+        'kubernetes-pool-repo:Pool:garm-operator-system'
+    ],
+    labels=["operator"],
+)

hack/Dockerfile

@@ -3,16 +3,16 @@
 FROM golang:1.22.5 AS build
 
 ARG garm_repo=https://github.com/cloudbase/garm
-ARG garm_repo_ref=v0.1.4
+ARG garm_repo_ref=v0.1.5
 
 # build garm binary
 # primarly used to get the binary build for
 # the local CPU architecture
 RUN git clone $garm_repo garm_repo && \
   cd garm_repo && \
   git checkout $garm_repo_ref && \
-  CGO_ENABLED=1 go install -ldflags "-linkmode external -extldflags '-static' -X main.Version=$garm_repo_ref" ./cmd/garm && \
-  CGO_ENABLED=1 go install -ldflags "-linkmode external -extldflags '-static' -X main.Version=$garm_repo_ref" ./cmd/garm-cli
+  CGO_ENABLED=1 go install -ldflags "-linkmode external -extldflags '-static' -X github.com/cloudbase/garm/util/appdefaults.Version=$garm_repo_ref" ./cmd/garm && \
+  CGO_ENABLED=1 go install -ldflags "-linkmode external -extldflags '-static' -X github.com/cloudbase/garm/util/appdefaults.Version=$garm_repo_ref" ./cmd/garm-cli
 
 
 FROM alpine:3.18

hack/local-development/kubernetes/configmap-envsubst.yaml → hack/local-development/kubernetes/configmap.yaml

@@ -42,13 +42,6 @@ data:
     config_file = "/opt/garm/provider-config/garm-provider-k8s-config.yaml"
     provider_executable = "/opt/garm/bin/garm-provider-k8s"
     environment_variables = ["KUBERNETES_"]
-
-    [[github]]
-    name = "$GARM_GITHUB_NAME"
-    oauth2_token = "$GARM_GITHUB_OAUTH_TOKEN"
-    base_url = "$GARM_GITHUB_BASE_URL"
-    api_base_url = "$GARM_GITHUB_API_BASE_URL"
-    upload_base_url = "$GARM_GITHUB_UPLOAD_BASE_URL"
 ---
 apiVersion: v1
 kind: ConfigMap