upgraded tekton api usage from alpha to beta

.github/workflows/e2e.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        kubernetes: ["1.16", "1.17", "1.18"]
+        kubernetes: ["1.17", "1.18", "1.19"]
     steps:
       - name: Install python dependencies
         run: sudo apt-get update && sudo apt-get install -y python3-setuptools python3-pip

build/defs/kustomize.build_defs

@@ -46,6 +46,12 @@ $(exe {CONFIG.KUSTOMIZE_TOOL}) build . > $HOME/$OUTS
 
     _kube_score(name, kustomized_rule, kube_score_ignored_tests)
 
+    sh_cmd(
+        name = f"{name}_push",
+        data = [kustomized_rule],
+        cmd = f"trap cleanup 1 2 3 6; cleanup() {{ set +x; }}; set -x; kubectl apply \\\$@ -f $(out_location {kustomized_rule})",
+    )
+
     return kustomized_rule
 
 def _replace_images_cmd(name: str, images: list):

cmd/dracon/cmd/run.go

@@ -49,8 +49,8 @@ var runCmd = &cobra.Command{
 		patches, err := template.LoadPatchYAMLFiles(pipelineOpts.ExtraPatchesPath)
 
 		// append PipelineResources
-		pipelineResourceDocs, err := template.GeneratePipelineResourceDocs()
-		files["draconPipelineResources"] = pipelineResourceDocs
+		// pipelineResourceDocs, err := template.GeneratePipelineResourceDocs()
+		// files["draconPipelineResources"] = pipelineResourceDocs
 
 		resDocs, err := template.PatchFileYAMLs(files, patches)
 		if err != nil {
@@ -69,7 +69,7 @@ var runCmd = &cobra.Command{
 		}
 
 		for _, doc := range resDocs["PipelineRun"] {
-			err = kubernetes.Apply(string(doc), &kubernetes.KubectlOpts{
+			err = kubernetes.Create(string(doc), &kubernetes.KubectlOpts{
 				Namespace: kubernetesNamespace,
 				Context:   kubernetesContext,
 			})

docs/designs/kustomize-support.md

@@ -25,14 +25,14 @@ patches:
 - path: patches/my-repository.yaml
   target:
     group: dracon
-    version: v1alpha1
+    version: v1beta1
     kind: PipelineResource
     name: "{{.RunID}}-git-github-oauth2-proxy"
 # note: this above patch doesn't currently read well so we may need to reconsider how we define pipelines.
 
 # ./patches/my-repository.yaml
 ---
-apiVersion: dracon/v1alpha1
+apiVersion: dracon/v1beta1
 kind: PipelineResource
 metadata:
   name: "{{.RunID}}-git-github-oauth2-proxy"

docs/getting-started/tutorials/running-demo-against-private-repository.md

@@ -26,8 +26,8 @@
    2. Name the resources consistently, we've opted for `gitssh-<repository_domain>-<repository_path_kebab_separated>`, examples:
       - `github.com/thought-machine/dracon` -> `gitssh-github-thought-machine-dracon`
       - `github.com/tektoncd/pipeline` -> `gitssh-github-tektoncd-pipeline`
-   3. Set the `tekton.dev/v1alpha1, PipelineRun` resource's `spec.serviceAccountName` to the `v1, ServiceAccount` you just added.
-   4. Remove the previous `tekton.dev/v1alpha1, PipelineResource`.
+   3. Set the `tekton.dev/v1beta1, PipelineRun` resource's `spec.serviceAccountName` to the `v1, ServiceAccount` you just added.
+   4. Remove the previous `tekton.dev/v1beta1, PipelineResource`.
 
       ```yaml
       ---
@@ -57,7 +57,7 @@
         - name: gitssh-github-tektoncd-pipeline
       ---
       # git+ssh config: pipeline resource
-      apiVersion: tekton.dev/v1alpha1
+      apiVersion: tekton.dev/v1beta1
       kind: PipelineResource
       metadata:
         name: "{{.RunID}}-gitssh-github-tektoncd-pipeline"

docs/getting-started/tutorials/running-demo-against-public-repository.md

@@ -18,15 +18,15 @@
    ```bash
    $ cp -r "${PWD}/dracon/examples/pipelines/mixed-lang-project" "${PWD}"
    ```
-3. Update the `tekton.dev/v1alpha1, PipelineResource` in `pipeline-run.yaml`:
+3. Update the `tekton.dev/v1beta1, PipelineResource` in `pipeline-run.yaml`:
 
    1. Set `spec.params[0].value` to your desired git revision/branch.
    2. Set `spec.params[1].value` to your desired git public git url.
 
       ```yaml
       ---
       # git+https config
-      apiVersion: dracon/v1alpha1
+      apiVersion: dracon/v1beta1
       kind: PipelineResource
       metadata:
         name: "{{.RunID}}-git-github-oauth2_proxy"

examples/git-ssh.pipeline-run.yaml

@@ -25,7 +25,7 @@ secrets:
 - name: gitssh-github-tektoncd-pipeline
 ---
 # git+ssh config: pipeline resource
-apiVersion: tekton.dev/v1alpha1
+apiVersion: tekton.dev/v1beta1
 kind: PipelineResource
 metadata:
   name: "{{.RunID}}-gitssh-github-tektoncd-pipeline"
@@ -39,7 +39,7 @@ spec:
     value: [email protected]:tektoncd/pipeline.git
 ---
 # pipelinerun
-apiVersion: tekton.dev/v1alpha1
+apiVersion: tekton.dev/v1beta1
 kind: PipelineRun
 metadata:
   name: "demo-{{.RunID}}"

examples/pipelines/golang-project-new/BUILD

@@ -0,0 +1,21 @@
+subinclude("//build/defs:kustomize")
+
+kustomized_config(
+    name = "dev",
+    srcs = [
+        "elasticsearch-consumer.yaml",
+        "enricher.yaml",
+        "git-source.yaml",
+        "gosec-producer.yaml",
+        "kustomization.yaml",
+        "pipeline.yaml",
+        "pipeline-run.yaml",
+        "//third_party/k8s:tektoncd_catalog_git-clone",
+    ],
+    images = [
+        "//consumers/elasticsearch_c:dracon-consumer-elasticsearch",
+        "//cmd/enricher:dracon-enricher",
+        "//source/git:dracon-source-git",
+        "//producers/golang_gosec:dracon-producer-gosec",
+    ],
+)

examples/pipelines/golang-project-new/elasticsearch-consumer.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: golang-project-elasticsearch-consumer
+  labels: {}
+spec:
+  params: 
+  - name: dracon-scan-time
+    type: string
+  steps:
+  # run elasticsearch consumer
+  - name: run-elasticsearch-consumer
+    image: index.docker.io/thoughtmachine/dracon-consumer-elasticsearch:latest
+    env: 
+    - name: DRACON_SCAN_TIME
+      value: "$(params.dracon-start-time)" 
+    - name: ELASTICSEARCH_URL
+      value: http://elasticsearch.dracon.svc:9200}
+    command: ["/consume"]
+    args: [
+      "-in", "$(workspaces.source.path)/.dracon/enriched",
+      "-es-index", "dracon"
+    ]
+  workspaces:
+  - name: source
+    mountPath: /workspace/source

examples/pipelines/golang-project-new/enricher.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: golang-project-enricher
+  labels: {}
+spec:
+  resources: {inputs: [], outputs: []}
+  # inputs: {resources: []}
+  # outputs: {resources: []}
+  steps:
+  # run enricher
+  - name: run-enricher
+    image: index.docker.io/thoughtmachine/dracon-enricher:latest
+    env:
+    - name: ENRICHER_READ_PATH
+      value: $(workspaces.source.path)/.dracon/raw
+    - name: ENRICHER_WRITE_PATH
+      value: $(workspaces.source.path)/.dracon/enriched
+    - name: ENRICHER_DB_CONNECTION
+      value: "postgresql://dracon:[email protected]?sslmode=disable"
+    command: ["/enricher"]
+  workspaces:
+  - name: source
+    mountPath: /workspace/source

examples/pipelines/golang-project-new/git-source.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: golang-project-git-source
+  labels: {}
+spec:
+  resources: {inputs: [], outputs: []}
+  # inputs: {resources: [{name: git-source, type: git}]}
+  # outputs: {resources: [{name: source, type: storage}]}
+  steps:
+  - name: env
+    image: index.docker.io/thoughtmachine/dracon-source-git:latest
+    command: ["env"]
+  - name: git-source
+    image: index.docker.io/thoughtmachine/dracon-source-git:latest
+    command: ["/git.sh"]
+  workspaces:
+  - name: source
+    mountPath: /workspace/source

examples/pipelines/golang-project-new/gosec-producer.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: golang-project-gosec-producer
+  labels: {}
+spec:
+  params: []
+  resources: {inputs: [], outputs: []}
+  steps:
+  # run gosec
+  - name: run-gosec
+    image: securego/gosec
+    command: ["sh"]
+    args: ["-c",
+      "gosec -fmt=json -out=/scratch/gosec.tool_out $(workspaces.source.path)/... || true"
+    ]
+    volumeMounts: 
+    - name: producer-scratch
+      mountPath: /scratch
+  # parse results
+  - name: parse-gosec
+    image: index.docker.io/thoughtmachine/dracon-producer-gosec:latest
+    command: ["/parse"]
+    args: [
+      "-in=/scratch/gosec.tool_out",
+      "-out=$(workspaces.source.path)/.dracon/raw/gosec.pb"
+    ]
+    volumeMounts: 
+    - name: producer-scratch
+      mountPath: /scratch
+  volumes:
+  - name: producer-scratch
+    emptyDir: {}
+  workspaces:
+  - name: source
+    mountPath: /workspace/source

examples/pipelines/golang-project-new/kustomization.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: dracon
+
+commonLabels:
+  app: dracon
+  app.kubernetes.io/component: "pipeline"
+  dracon.thoughtmachine.io/pipeline: "golang-project"
+
+resources:
+- elasticsearch-consumer.yaml
+- enricher.yaml
+- git-source.yaml
+- gosec-producer.yaml
+- pipeline-run.yaml
+- pipeline.yaml
+- third_party/k8s/git-clone.yaml
+
+patches:
+- patch: |-
+    - op: move
+      from: /metadata/name
+      path: /metadata/generateName
+  target:
+      kind: PipelineRun

examples/pipelines/golang-project-new/pipeline-run.yaml

@@ -0,0 +1,42 @@
+# ---
+# # git+https config
+# apiVersion: tekton.dev/v1alpha1
+# kind: PipelineResource
+# metadata:
+#   name: "{{.RunID}}-git-github-oauth2-proxy"
+#   labels: {}
+# spec:
+#   type: git
+#   params:
+#   - name: revision
+#     value: master
+#   - name: url
+#     value: https://github.com/pusher/oauth2_proxy.git
+---
+# # pipelinerun
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: "golang-project-"
+  # name: "golang-project-{{.RunID}}"
+  labels:
+    project: dracon
+spec:
+  pipelineRef: {name: golang-project}
+  podTemplate: {}
+  # serviceAccount: git-github-oauth2-proxy # replace with desired git source service account (if needed)
+  timeout: 1h0m0s
+  params:
+  - name: repo-url
+    value: https://github.com/pusher/oauth2_proxy.git
+  - name: branch-name
+    value: master
+  workspaces:
+  - name: source # this workspace name must be declared in the Pipeline
+    volumeClaimTemplate:
+      spec:
+        accessModes:
+          - ReadWriteOnce # access mode may affect how you can use this volume in parallel tasks
+        resources:
+          requests:
+            storage: 1Gi

examples/pipelines/golang-project-new/pipeline.yaml

@@ -0,0 +1,55 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: golang-project
+  labels: {}
+spec:
+  params:
+  - name: repo-url
+    type: string
+    description: The git repository URL to clone from.
+  - name: branch-name
+    type: string
+    description: The git branch to clone.
+  workspaces:
+  - name: source
+  tasks:
+  - name: fetch-source
+    taskRef: {name: git-clone}
+    workspaces:
+    - name: output
+      workspace: source
+    params:
+    - name: url
+      value: $(params.repo-url)
+    - name: revision
+      value: $(params.branch-name)
+  # stage 1 - archive source
+  - name: annotate-source
+    runAfter: [fetch-source]
+    taskRef: {name: golang-project-git-source}
+    workspaces:
+    - name: source
+      workspace: source
+  # stage 2 - run tools
+  - name: gosec-producer
+    runAfter: [annotate-source]
+    taskRef: {name: golang-project-gosec-producer}
+    workspaces:
+    - name: source
+      workspace: source
+  # stage 3 - enrichment
+  - name: enricher
+    runAfter: [gosec-producer]
+    taskRef: {name: golang-project-enricher}
+    workspaces:
+    - name: source
+      workspace: source
+  # stage 4 - consumers
+  - name: elasticsearch-consumer
+    runAfter: [enricher]
+    taskRef: {name: golang-project-elasticsearch-consumer}
+    workspaces:
+    - name: source
+      workspace: source

examples/pipelines/golang-project/elasticsearch-consumer.yaml

@@ -1,11 +1,12 @@
 ---
-apiVersion: dracon/v1alpha1
+apiVersion: dracon/v1beta1
 kind: Consumer
 metadata:
   name: golang-project-elasticsearch-consumer
   labels: {}
 spec:
-  inputs: {resources: [], params: []}
+  params: []
+  resources: {inputs: [], outputs: []}
   steps:
   # run elasticsearch consumer
   - name: run-elasticsearch-consumer