diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 54ec4dc..9133727 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - kubernetes: ["1.16", "1.17", "1.18"] + kubernetes: ["1.17", "1.18", "1.19"] steps: - name: Install python dependencies run: sudo apt-get update && sudo apt-get install -y python3-setuptools python3-pip diff --git a/build/defs/kustomize.build_defs b/build/defs/kustomize.build_defs index a7ce143..bf349c7 100644 --- a/build/defs/kustomize.build_defs +++ b/build/defs/kustomize.build_defs @@ -46,6 +46,12 @@ $(exe {CONFIG.KUSTOMIZE_TOOL}) build . > $HOME/$OUTS _kube_score(name, kustomized_rule, kube_score_ignored_tests) + sh_cmd( + name = f"{name}_push", + data = [kustomized_rule], + cmd = f"trap cleanup 1 2 3 6; cleanup() {{ set +x; }}; set -x; kubectl apply \\\$@ -f $(out_location {kustomized_rule})", + ) + return kustomized_rule def _replace_images_cmd(name: str, images: list): diff --git a/cmd/dracon/cmd/run.go b/cmd/dracon/cmd/run.go index e4d318b..cd6e3ab 100644 --- a/cmd/dracon/cmd/run.go +++ b/cmd/dracon/cmd/run.go @@ -49,8 +49,8 @@ var runCmd = &cobra.Command{ patches, err := template.LoadPatchYAMLFiles(pipelineOpts.ExtraPatchesPath) // append PipelineResources - pipelineResourceDocs, err := template.GeneratePipelineResourceDocs() - files["draconPipelineResources"] = pipelineResourceDocs + // pipelineResourceDocs, err := template.GeneratePipelineResourceDocs() + // files["draconPipelineResources"] = pipelineResourceDocs resDocs, err := template.PatchFileYAMLs(files, patches) if err != nil { @@ -69,7 +69,7 @@ var runCmd = &cobra.Command{ } for _, doc := range resDocs["PipelineRun"] { - err = kubernetes.Apply(string(doc), &kubernetes.KubectlOpts{ + err = kubernetes.Create(string(doc), &kubernetes.KubectlOpts{ Namespace: kubernetesNamespace, Context: kubernetesContext, }) diff --git a/docs/designs/kustomize-support.md b/docs/designs/kustomize-support.md index 99b7777..c76381c 100644 --- a/docs/designs/kustomize-support.md +++ b/docs/designs/kustomize-support.md @@ -25,14 +25,14 @@ patches: - path: patches/my-repository.yaml target: group: dracon - version: v1alpha1 + version: v1beta1 kind: PipelineResource name: "{{.RunID}}-git-github-oauth2-proxy" # note: this above patch doesn't currently read well so we may need to reconsider how we define pipelines. # ./patches/my-repository.yaml --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: PipelineResource metadata: name: "{{.RunID}}-git-github-oauth2-proxy" diff --git a/docs/getting-started/tutorials/running-demo-against-private-repository.md b/docs/getting-started/tutorials/running-demo-against-private-repository.md index 2a7b5f9..84280bd 100644 --- a/docs/getting-started/tutorials/running-demo-against-private-repository.md +++ b/docs/getting-started/tutorials/running-demo-against-private-repository.md @@ -26,8 +26,8 @@ 2. Name the resources consistently, we've opted for `gitssh--`, examples: - `github.com/thought-machine/dracon` -> `gitssh-github-thought-machine-dracon` - `github.com/tektoncd/pipeline` -> `gitssh-github-tektoncd-pipeline` - 3. Set the `tekton.dev/v1alpha1, PipelineRun` resource's `spec.serviceAccountName` to the `v1, ServiceAccount` you just added. - 4. Remove the previous `tekton.dev/v1alpha1, PipelineResource`. + 3. Set the `tekton.dev/v1beta1, PipelineRun` resource's `spec.serviceAccountName` to the `v1, ServiceAccount` you just added. + 4. Remove the previous `tekton.dev/v1beta1, PipelineResource`. ```yaml --- @@ -57,7 +57,7 @@ - name: gitssh-github-tektoncd-pipeline --- # git+ssh config: pipeline resource - apiVersion: tekton.dev/v1alpha1 + apiVersion: tekton.dev/v1beta1 kind: PipelineResource metadata: name: "{{.RunID}}-gitssh-github-tektoncd-pipeline" diff --git a/docs/getting-started/tutorials/running-demo-against-public-repository.md b/docs/getting-started/tutorials/running-demo-against-public-repository.md index c2dcabe..570703d 100644 --- a/docs/getting-started/tutorials/running-demo-against-public-repository.md +++ b/docs/getting-started/tutorials/running-demo-against-public-repository.md @@ -18,7 +18,7 @@ ```bash $ cp -r "${PWD}/dracon/examples/pipelines/mixed-lang-project" "${PWD}" ``` -3. Update the `tekton.dev/v1alpha1, PipelineResource` in `pipeline-run.yaml`: +3. Update the `tekton.dev/v1beta1, PipelineResource` in `pipeline-run.yaml`: 1. Set `spec.params[0].value` to your desired git revision/branch. 2. Set `spec.params[1].value` to your desired git public git url. @@ -26,7 +26,7 @@ ```yaml --- # git+https config - apiVersion: dracon/v1alpha1 + apiVersion: dracon/v1beta1 kind: PipelineResource metadata: name: "{{.RunID}}-git-github-oauth2_proxy" diff --git a/examples/git-ssh.pipeline-run.yaml b/examples/git-ssh.pipeline-run.yaml index 75a036a..238dec0 100644 --- a/examples/git-ssh.pipeline-run.yaml +++ b/examples/git-ssh.pipeline-run.yaml @@ -25,7 +25,7 @@ secrets: - name: gitssh-github-tektoncd-pipeline --- # git+ssh config: pipeline resource -apiVersion: tekton.dev/v1alpha1 +apiVersion: tekton.dev/v1beta1 kind: PipelineResource metadata: name: "{{.RunID}}-gitssh-github-tektoncd-pipeline" @@ -39,7 +39,7 @@ spec: value: git@github.com:tektoncd/pipeline.git --- # pipelinerun -apiVersion: tekton.dev/v1alpha1 +apiVersion: tekton.dev/v1beta1 kind: PipelineRun metadata: name: "demo-{{.RunID}}" diff --git a/examples/pipelines/golang-project-new/BUILD b/examples/pipelines/golang-project-new/BUILD new file mode 100644 index 0000000..4a83df0 --- /dev/null +++ b/examples/pipelines/golang-project-new/BUILD @@ -0,0 +1,21 @@ +subinclude("//build/defs:kustomize") + +kustomized_config( + name = "dev", + srcs = [ + "elasticsearch-consumer.yaml", + "enricher.yaml", + "git-source.yaml", + "gosec-producer.yaml", + "kustomization.yaml", + "pipeline.yaml", + "pipeline-run.yaml", + "//third_party/k8s:tektoncd_catalog_git-clone", + ], + images = [ + "//consumers/elasticsearch_c:dracon-consumer-elasticsearch", + "//cmd/enricher:dracon-enricher", + "//source/git:dracon-source-git", + "//producers/golang_gosec:dracon-producer-gosec", + ], +) diff --git a/examples/pipelines/golang-project-new/elasticsearch-consumer.yaml b/examples/pipelines/golang-project-new/elasticsearch-consumer.yaml new file mode 100644 index 0000000..f556bcd --- /dev/null +++ b/examples/pipelines/golang-project-new/elasticsearch-consumer.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: golang-project-elasticsearch-consumer + labels: {} +spec: + params: + - name: dracon-scan-time + type: string + steps: + # run elasticsearch consumer + - name: run-elasticsearch-consumer + image: index.docker.io/thoughtmachine/dracon-consumer-elasticsearch:latest + env: + - name: DRACON_SCAN_TIME + value: "$(params.dracon-start-time)" + - name: ELASTICSEARCH_URL + value: http://elasticsearch.dracon.svc:9200} + command: ["/consume"] + args: [ + "-in", "$(workspaces.source.path)/.dracon/enriched", + "-es-index", "dracon" + ] + workspaces: + - name: source + mountPath: /workspace/source diff --git a/examples/pipelines/golang-project-new/enricher.yaml b/examples/pipelines/golang-project-new/enricher.yaml new file mode 100644 index 0000000..1e5c3de --- /dev/null +++ b/examples/pipelines/golang-project-new/enricher.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: golang-project-enricher + labels: {} +spec: + resources: {inputs: [], outputs: []} + # inputs: {resources: []} + # outputs: {resources: []} + steps: + # run enricher + - name: run-enricher + image: index.docker.io/thoughtmachine/dracon-enricher:latest + env: + - name: ENRICHER_READ_PATH + value: $(workspaces.source.path)/.dracon/raw + - name: ENRICHER_WRITE_PATH + value: $(workspaces.source.path)/.dracon/enriched + - name: ENRICHER_DB_CONNECTION + value: "postgresql://dracon:dracon@dracon-enrichment-db.dracon.svc?sslmode=disable" + command: ["/enricher"] + workspaces: + - name: source + mountPath: /workspace/source diff --git a/examples/pipelines/golang-project-new/git-source.yaml b/examples/pipelines/golang-project-new/git-source.yaml new file mode 100644 index 0000000..38cdacf --- /dev/null +++ b/examples/pipelines/golang-project-new/git-source.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: golang-project-git-source + labels: {} +spec: + resources: {inputs: [], outputs: []} + # inputs: {resources: [{name: git-source, type: git}]} + # outputs: {resources: [{name: source, type: storage}]} + steps: + - name: env + image: index.docker.io/thoughtmachine/dracon-source-git:latest + command: ["env"] + - name: git-source + image: index.docker.io/thoughtmachine/dracon-source-git:latest + command: ["/git.sh"] + workspaces: + - name: source + mountPath: /workspace/source diff --git a/examples/pipelines/golang-project-new/gosec-producer.yaml b/examples/pipelines/golang-project-new/gosec-producer.yaml new file mode 100644 index 0000000..e515a8f --- /dev/null +++ b/examples/pipelines/golang-project-new/gosec-producer.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: golang-project-gosec-producer + labels: {} +spec: + params: [] + resources: {inputs: [], outputs: []} + steps: + # run gosec + - name: run-gosec + image: securego/gosec + command: ["sh"] + args: ["-c", + "gosec -fmt=json -out=/scratch/gosec.tool_out $(workspaces.source.path)/... || true" + ] + volumeMounts: + - name: producer-scratch + mountPath: /scratch + # parse results + - name: parse-gosec + image: index.docker.io/thoughtmachine/dracon-producer-gosec:latest + command: ["/parse"] + args: [ + "-in=/scratch/gosec.tool_out", + "-out=$(workspaces.source.path)/.dracon/raw/gosec.pb" + ] + volumeMounts: + - name: producer-scratch + mountPath: /scratch + volumes: + - name: producer-scratch + emptyDir: {} + workspaces: + - name: source + mountPath: /workspace/source diff --git a/examples/pipelines/golang-project-new/kustomization.yaml b/examples/pipelines/golang-project-new/kustomization.yaml new file mode 100644 index 0000000..e895ef4 --- /dev/null +++ b/examples/pipelines/golang-project-new/kustomization.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: dracon + +commonLabels: + app: dracon + app.kubernetes.io/component: "pipeline" + dracon.thoughtmachine.io/pipeline: "golang-project" + +resources: +- elasticsearch-consumer.yaml +- enricher.yaml +- git-source.yaml +- gosec-producer.yaml +- pipeline-run.yaml +- pipeline.yaml +- third_party/k8s/git-clone.yaml + +patches: +- patch: |- + - op: move + from: /metadata/name + path: /metadata/generateName + target: + kind: PipelineRun diff --git a/examples/pipelines/golang-project-new/pipeline-run.yaml b/examples/pipelines/golang-project-new/pipeline-run.yaml new file mode 100644 index 0000000..86f0cf3 --- /dev/null +++ b/examples/pipelines/golang-project-new/pipeline-run.yaml @@ -0,0 +1,42 @@ +# --- +# # git+https config +# apiVersion: tekton.dev/v1alpha1 +# kind: PipelineResource +# metadata: +# name: "{{.RunID}}-git-github-oauth2-proxy" +# labels: {} +# spec: +# type: git +# params: +# - name: revision +# value: master +# - name: url +# value: https://github.com/pusher/oauth2_proxy.git +--- +# # pipelinerun +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: "golang-project-" + # name: "golang-project-{{.RunID}}" + labels: + project: dracon +spec: + pipelineRef: {name: golang-project} + podTemplate: {} + # serviceAccount: git-github-oauth2-proxy # replace with desired git source service account (if needed) + timeout: 1h0m0s + params: + - name: repo-url + value: https://github.com/pusher/oauth2_proxy.git + - name: branch-name + value: master + workspaces: + - name: source # this workspace name must be declared in the Pipeline + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce # access mode may affect how you can use this volume in parallel tasks + resources: + requests: + storage: 1Gi diff --git a/examples/pipelines/golang-project-new/pipeline.yaml b/examples/pipelines/golang-project-new/pipeline.yaml new file mode 100644 index 0000000..7565de6 --- /dev/null +++ b/examples/pipelines/golang-project-new/pipeline.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: golang-project + labels: {} +spec: + params: + - name: repo-url + type: string + description: The git repository URL to clone from. + - name: branch-name + type: string + description: The git branch to clone. + workspaces: + - name: source + tasks: + - name: fetch-source + taskRef: {name: git-clone} + workspaces: + - name: output + workspace: source + params: + - name: url + value: $(params.repo-url) + - name: revision + value: $(params.branch-name) + # stage 1 - archive source + - name: annotate-source + runAfter: [fetch-source] + taskRef: {name: golang-project-git-source} + workspaces: + - name: source + workspace: source + # stage 2 - run tools + - name: gosec-producer + runAfter: [annotate-source] + taskRef: {name: golang-project-gosec-producer} + workspaces: + - name: source + workspace: source + # stage 3 - enrichment + - name: enricher + runAfter: [gosec-producer] + taskRef: {name: golang-project-enricher} + workspaces: + - name: source + workspace: source + # stage 4 - consumers + - name: elasticsearch-consumer + runAfter: [enricher] + taskRef: {name: golang-project-elasticsearch-consumer} + workspaces: + - name: source + workspace: source diff --git a/examples/pipelines/golang-project/elasticsearch-consumer.yaml b/examples/pipelines/golang-project/elasticsearch-consumer.yaml index 036653d..0c7e7d8 100644 --- a/examples/pipelines/golang-project/elasticsearch-consumer.yaml +++ b/examples/pipelines/golang-project/elasticsearch-consumer.yaml @@ -1,11 +1,12 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Consumer metadata: name: golang-project-elasticsearch-consumer labels: {} spec: - inputs: {resources: [], params: []} + params: [] + resources: {inputs: [], outputs: []} steps: # run elasticsearch consumer - name: run-elasticsearch-consumer diff --git a/examples/pipelines/golang-project/enricher.yaml b/examples/pipelines/golang-project/enricher.yaml index 37457e2..b93cd3c 100644 --- a/examples/pipelines/golang-project/enricher.yaml +++ b/examples/pipelines/golang-project/enricher.yaml @@ -1,12 +1,13 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Enricher metadata: name: golang-project-enricher labels: {} spec: - inputs: {resources: []} - outputs: {resources: []} + resources: {inputs: [], outputs: []} + # inputs: {resources: []} + # outputs: {resources: []} steps: # run enricher - name: run-enricher diff --git a/examples/pipelines/golang-project/git-source.yaml b/examples/pipelines/golang-project/git-source.yaml index 48bf857..03f8ed2 100644 --- a/examples/pipelines/golang-project/git-source.yaml +++ b/examples/pipelines/golang-project/git-source.yaml @@ -1,13 +1,22 @@ --- -apiVersion: tekton.dev/v1alpha1 +apiVersion: tekton.dev/v1beta1 kind: Task metadata: name: golang-project-git-source labels: {} spec: - inputs: {resources: [{name: git-source, type: git}]} - outputs: {resources: [{name: source, type: storage}]} + results: + - name: dracon-scan-time + description: "The scan time that dracon started at" + resources: {inputs: [], outputs: []} + # inputs: {resources: [{name: git-source, type: git}]} + # outputs: {resources: [{name: source, type: storage}]} steps: - name: git-source image: index.docker.io/thoughtmachine/dracon-source-git:latest command: ["/git.sh"] + - name: print-dracon-scan-time + image: bash:latest + script: | + #!/usr/bin/env bash + date --rfc-3339=seconds | tee $(results.dracon-scan-time.path) diff --git a/examples/pipelines/golang-project/gosec-producer.yaml b/examples/pipelines/golang-project/gosec-producer.yaml index de766af..b91ef5b 100644 --- a/examples/pipelines/golang-project/gosec-producer.yaml +++ b/examples/pipelines/golang-project/gosec-producer.yaml @@ -1,12 +1,14 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Producer metadata: name: golang-project-gosec-producer labels: {} spec: - inputs: {resources: [], params: []} - outputs: {resources: []} + params: [] + resources: {inputs: [], outputs: []} + # inputs: {resources: [], params: []} + # outputs: {resources: []} volumes: [] steps: # run gosec diff --git a/examples/pipelines/golang-project/pipeline-run.yaml b/examples/pipelines/golang-project/pipeline-run.yaml index bf3bbe6..3745dc2 100644 --- a/examples/pipelines/golang-project/pipeline-run.yaml +++ b/examples/pipelines/golang-project/pipeline-run.yaml @@ -1,6 +1,6 @@ --- # git+https config -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: PipelineResource metadata: name: "{{.RunID}}-git-github-oauth2-proxy" @@ -14,7 +14,7 @@ spec: value: https://github.com/pusher/oauth2_proxy.git --- # pipelinerun -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: PipelineRun metadata: name: "golang-project-{{.RunID}}" diff --git a/examples/pipelines/golang-project/pipeline.yaml b/examples/pipelines/golang-project/pipeline.yaml index 6769744..592bd0a 100644 --- a/examples/pipelines/golang-project/pipeline.yaml +++ b/examples/pipelines/golang-project/pipeline.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Pipeline metadata: name: golang-project @@ -30,5 +30,7 @@ spec: - name: elasticsearch-consumer runAfter: [enricher] taskRef: {name: golang-project-elasticsearch-consumer} - params: [] + params: + - name: dracon-start-time + value: "$(tasks.fetch-source.results.dracon-start-time)" resources: {inputs: [], outputs: []} diff --git a/examples/pipelines/mixed-lang-project/bandit-producer.yaml b/examples/pipelines/mixed-lang-project/bandit-producer.yaml index 258f0dc..2971f56 100644 --- a/examples/pipelines/mixed-lang-project/bandit-producer.yaml +++ b/examples/pipelines/mixed-lang-project/bandit-producer.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Producer metadata: name: mixed-lang-project-bandit-producer diff --git a/examples/pipelines/mixed-lang-project/elasticsearch-consumer.yaml b/examples/pipelines/mixed-lang-project/elasticsearch-consumer.yaml index d20ccec..b011a09 100644 --- a/examples/pipelines/mixed-lang-project/elasticsearch-consumer.yaml +++ b/examples/pipelines/mixed-lang-project/elasticsearch-consumer.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Consumer metadata: name: mixed-lang-project-elasticsearch-consumer diff --git a/examples/pipelines/mixed-lang-project/enricher.yaml b/examples/pipelines/mixed-lang-project/enricher.yaml index 7719d90..4cae544 100644 --- a/examples/pipelines/mixed-lang-project/enricher.yaml +++ b/examples/pipelines/mixed-lang-project/enricher.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Enricher metadata: name: mixed-lang-project-enricher diff --git a/examples/pipelines/mixed-lang-project/git-source.yaml b/examples/pipelines/mixed-lang-project/git-source.yaml index 2e62312..3506b1f 100644 --- a/examples/pipelines/mixed-lang-project/git-source.yaml +++ b/examples/pipelines/mixed-lang-project/git-source.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1alpha1 +apiVersion: tekton.dev/v1beta1 kind: Task metadata: name: mixed-lang-project-git-source diff --git a/examples/pipelines/mixed-lang-project/gosec-producer.yaml b/examples/pipelines/mixed-lang-project/gosec-producer.yaml index 07173b4..c97b1f5 100644 --- a/examples/pipelines/mixed-lang-project/gosec-producer.yaml +++ b/examples/pipelines/mixed-lang-project/gosec-producer.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Producer metadata: name: mixed-lang-project-gosec-producer diff --git a/examples/pipelines/mixed-lang-project/pipeline-run.yaml b/examples/pipelines/mixed-lang-project/pipeline-run.yaml index 89a3c0d..a9c45b4 100644 --- a/examples/pipelines/mixed-lang-project/pipeline-run.yaml +++ b/examples/pipelines/mixed-lang-project/pipeline-run.yaml @@ -1,6 +1,6 @@ --- # git+https config -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: PipelineResource metadata: name: "{{.RunID}}-git-github-thoughtmachine-please" @@ -14,7 +14,7 @@ spec: value: https://github.com/thought-machine/please.git --- # pipelinerun -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: PipelineRun metadata: name: "mixed-lang-project-{{.RunID}}" diff --git a/examples/pipelines/mixed-lang-project/pipeline.yaml b/examples/pipelines/mixed-lang-project/pipeline.yaml index a723da9..2055e56 100644 --- a/examples/pipelines/mixed-lang-project/pipeline.yaml +++ b/examples/pipelines/mixed-lang-project/pipeline.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Pipeline metadata: name: mixed-lang-project diff --git a/examples/pipelines/mixed-lang-project/spotbugs-producer.yaml b/examples/pipelines/mixed-lang-project/spotbugs-producer.yaml index 0542ae5..20add90 100644 --- a/examples/pipelines/mixed-lang-project/spotbugs-producer.yaml +++ b/examples/pipelines/mixed-lang-project/spotbugs-producer.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Producer metadata: name: mixed-lang-project-spotbugs-producer diff --git a/examples/pipelines/python-project/bandit-producer.yaml b/examples/pipelines/python-project/bandit-producer.yaml index cc84c5e..82b3a6c 100644 --- a/examples/pipelines/python-project/bandit-producer.yaml +++ b/examples/pipelines/python-project/bandit-producer.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Producer metadata: name: python-project-bandit-producer diff --git a/examples/pipelines/python-project/elasticsearch-consumer.yaml b/examples/pipelines/python-project/elasticsearch-consumer.yaml index cbfadd4..aa5b5a9 100644 --- a/examples/pipelines/python-project/elasticsearch-consumer.yaml +++ b/examples/pipelines/python-project/elasticsearch-consumer.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Consumer metadata: name: python-project-elasticsearch-consumer diff --git a/examples/pipelines/python-project/enricher.yaml b/examples/pipelines/python-project/enricher.yaml index e8efdcb..0053ae7 100644 --- a/examples/pipelines/python-project/enricher.yaml +++ b/examples/pipelines/python-project/enricher.yaml @@ -1,12 +1,13 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Enricher metadata: name: python-project-enricher labels: {} spec: - inputs: {resources: []} - outputs: {resources: []} + resources: {inputs: [], outputs: []} + # inputs: {resources: []} + # outputs: {resources: []} steps: # run enricher - name: run-enricher diff --git a/examples/pipelines/python-project/git-source.yaml b/examples/pipelines/python-project/git-source.yaml index ae93dbf..98e68b1 100644 --- a/examples/pipelines/python-project/git-source.yaml +++ b/examples/pipelines/python-project/git-source.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1alpha1 +apiVersion: tekton.dev/v1beta1 kind: Task metadata: name: python-project-git-source diff --git a/examples/pipelines/python-project/pipeline-run.yaml b/examples/pipelines/python-project/pipeline-run.yaml index 4290dea..e268164 100644 --- a/examples/pipelines/python-project/pipeline-run.yaml +++ b/examples/pipelines/python-project/pipeline-run.yaml @@ -1,6 +1,6 @@ --- # git+https config -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: PipelineResource metadata: name: "{{.RunID}}-git-github-httpie-httpie" @@ -14,7 +14,7 @@ spec: value: https://github.com/httpie/httpie.git --- # pipelinerun -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: PipelineRun metadata: name: "python-project-{{.RunID}}" diff --git a/examples/pipelines/python-project/pipeline.yaml b/examples/pipelines/python-project/pipeline.yaml index 4ad00ce..c9982e1 100644 --- a/examples/pipelines/python-project/pipeline.yaml +++ b/examples/pipelines/python-project/pipeline.yaml @@ -1,5 +1,5 @@ --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Pipeline metadata: name: python-project diff --git a/examples/use-secrets.task.yaml b/examples/use-secrets.task.yaml index c1a68ab..d46c990 100644 --- a/examples/use-secrets.task.yaml +++ b/examples/use-secrets.task.yaml @@ -17,7 +17,7 @@ metadata: secrets: - name: my-secret-consumer --- -apiVersion: dracon/v1alpha1 +apiVersion: dracon/v1beta1 kind: Consumer metadata: name: secrets-consumer @@ -38,7 +38,7 @@ spec: ] --- # pipelinerun -apiVersion: tekton.dev/v1alpha1 +apiVersion: tekton.dev/v1beta1 kind: PipelineRun metadata: name: "demo-secrets-{{.RunID}}" diff --git a/pkg/kubernetes/BUILD b/pkg/kubernetes/BUILD index 5d337ec..12d63b5 100644 --- a/pkg/kubernetes/BUILD +++ b/pkg/kubernetes/BUILD @@ -2,6 +2,7 @@ go_library( name = "kubernetes", srcs = [ "apply.go", + "run.go", ], visibility = ["PUBLIC"], deps = [ diff --git a/pkg/kubernetes/apply.go b/pkg/kubernetes/apply.go index 6df9842..c2b9c54 100644 --- a/pkg/kubernetes/apply.go +++ b/pkg/kubernetes/apply.go @@ -15,7 +15,7 @@ type KubectlOpts struct { // Apply config using kubectl func Apply(resources string, opts *KubectlOpts) error { - shCmd := GetCmd(opts) + shCmd := GetCmd(opts, "apply") cmd := exec.Command(shCmd[0], shCmd[1:]...) stdin, err := cmd.StdinPipe() if err != nil { @@ -38,8 +38,8 @@ func Apply(resources string, opts *KubectlOpts) error { } // GetCmd returns the kubectl command -func GetCmd(opts *KubectlOpts) []string { - cmd := []string{"kubectl", "apply", "-f", "-"} +func GetCmd(opts *KubectlOpts, arg string) []string { + cmd := []string{"kubectl", arg, "-f", "-"} if opts.Context != "" { cmd = append(cmd, fmt.Sprintf(`--context=%s`, opts.Context)) diff --git a/pkg/kubernetes/run.go b/pkg/kubernetes/run.go new file mode 100644 index 0000000..98f7d3e --- /dev/null +++ b/pkg/kubernetes/run.go @@ -0,0 +1,32 @@ +package kubernetes + +import ( + "fmt" + "io" + "log" + "os/exec" +) + +// Create config using kubectl +func Create(resources string, opts *KubectlOpts) error { + shCmd := GetCmd(opts, "create") + cmd := exec.Command(shCmd[0], shCmd[1:]...) + stdin, err := cmd.StdinPipe() + if err != nil { + return fmt.Errorf("could not create stdin pipe: %w", err) + } + go func() { + defer stdin.Close() + _, err := io.WriteString(stdin, resources) + if err != nil { + log.Fatal(err) + } + }() + + output, err := cmd.CombinedOutput() + if err != nil || !cmd.ProcessState.Success() { + return fmt.Errorf("%s\n%s:%w", resources, output, err) + } + log.Printf("%s\n", output) + return nil +} diff --git a/pkg/template/pipeline-resource.go b/pkg/template/pipeline-resource.go index 51abbed..3d4c30a 100644 --- a/pkg/template/pipeline-resource.go +++ b/pkg/template/pipeline-resource.go @@ -19,7 +19,7 @@ func GeneratePipelineResourceDocs() (ResourceDocs, error) { for _, t := range resources { buf := bytes.Buffer{} pR := pipelineResource{ - APIVersion: "tekton.dev/v1alpha1", + APIVersion: "tekton.dev/v1beta1", Kind: "PipelineResource", Metadata: pipelineResourceMetadata{ Name: fmt.Sprintf("%s-%s", TemplateVars.RunID, t.Name), diff --git a/resources/patches/BUILD b/resources/patches/BUILD index 5342288..72eafca 100644 --- a/resources/patches/BUILD +++ b/resources/patches/BUILD @@ -1,5 +1,11 @@ +# filegroup( +# name = "patches", +# srcs = glob(["*.yaml"]), +# visibility = ["//pkg/template/..."], +# ) + filegroup( name = "patches", - srcs = glob(["*.yaml"]), + srcs = ["blank.yaml"], visibility = ["//pkg/template/..."], ) diff --git a/resources/patches/blank.yaml b/resources/patches/blank.yaml new file mode 100644 index 0000000..e69de29 diff --git a/resources/patches/patch-dracon-enriched.Consumer.yaml b/resources/patches/patch-dracon-enriched.Consumer.yaml index ab9cf1c..9af8ae2 100644 --- a/resources/patches/patch-dracon-enriched.Consumer.yaml +++ b/resources/patches/patch-dracon-enriched.Consumer.yaml @@ -1,12 +1,12 @@ --- # Enriched input - op: add - path: /spec/inputs/resources/- + path: /spec/resources/inputs/- value: {name: enricher, type: storage} - op: replace path: /apiVersion - value: tekton.dev/v1alpha1 + value: tekton.dev/v1beta1 - op: replace path: /kind value: Task @@ -17,7 +17,7 @@ {{range .PipelineParams}} - op: add - path: /spec/inputs/params/- + path: /spec/params/- value: {name: {{.Name}}, type: {{.Type}}} {{end}} diff --git a/resources/patches/patch-dracon.Enricher.yaml b/resources/patches/patch-dracon.Enricher.yaml index b090bb5..9825901 100644 --- a/resources/patches/patch-dracon.Enricher.yaml +++ b/resources/patches/patch-dracon.Enricher.yaml @@ -1,7 +1,7 @@ --- - op: replace path: /apiVersion - value: tekton.dev/v1alpha1 + value: tekton.dev/v1beta1 - op: replace path: /kind value: Task @@ -11,7 +11,7 @@ value: dracon - op: add - path: /spec/outputs/resources/- + path: /spec/resources/outputs/- value: {name: enricher, type: storage} - op: add @@ -26,6 +26,6 @@ {{ $producers := .PipelineTaskProducers }} {{range $p := $producers}} - op: add - path: /spec/inputs/resources/- + path: /spec/resources/inputs/- value: {name: {{$p.Name}}-producer, type: storage} {{end}} diff --git a/resources/patches/patch-dracon.Pipeline.yaml b/resources/patches/patch-dracon.Pipeline.yaml index 9abf382..0bb430c 100644 --- a/resources/patches/patch-dracon.Pipeline.yaml +++ b/resources/patches/patch-dracon.Pipeline.yaml @@ -1,12 +1,17 @@ --- - op: replace path: /apiVersion - value: tekton.dev/v1alpha1 + value: tekton.dev/v1beta1 - op: add path: /metadata/labels/project value: dracon +# Add workspace +- op: add + path: /spec/workspaces + value: [{name: "dracon"}] + {{ $pipelineParams := .PipelineParams}} {{ $enrichers := .PipelineTaskEnrichers }} {{ $producers := .PipelineTaskProducers }} @@ -18,60 +23,28 @@ value: {name: {{$pp.Name}}, description: "{{$pp.Description}}", type: {{$pp.Type}}} {{end}} -# Source -- op: add - path: /spec/resources/- - value: {name: source, type: storage} -- op: add - path: /spec/tasks/0/resources/outputs/- - value: {name: source, resource: source} - - -# Producers -{{range $p := $producers}} -- op: add - path: /spec/resources/- - value: {name: {{$p.Name}}-producer, type: storage} - -- op: add - path: /spec/tasks/{{$p.Index}}/resources/inputs/- - value: {name: source, resource: source} - -- op: add - path: /spec/tasks/{{$p.Index}}/resources/outputs/- - value: {name: producer, resource: {{$p.Name}}-producer} -{{range $pp := $pipelineParams}} -- op: add - path: /spec/tasks/{{$p.Index}}/params/- - value: {name: {{$pp.Name}}, value: $(params.{{$pp.Name}})} -{{end}} -{{end}} - # Enrichers {{range $e := $enrichers}} - op: add - path: /spec/resources/- - value: {name: enricher, type: storage} -- op: add - path: /spec/tasks/{{$e.Index}}/resources/outputs/- - value: {name: enricher, resource: enricher} + path: /spec/tasks/{{$e.Index}}/workspaces + value: [{name: "dracon", workspace: "dracon"}] +{{end}} {{range $p := $producers}} - op: add - path: /spec/tasks/{{$e.Index}}/resources/inputs/- - value: {name: {{$p.Name}}-producer, resource: {{$p.Name}}-producer} -{{end}} + path: /spec/tasks/{{$p.Index}}/workspaces + value: [{name: "dracon", workspace: "dracon"}] {{end}} # Consumers {{range $c := $consumers}} - op: add - path: /spec/tasks/{{$c.Index}}/resources/inputs/- - value: {name: enricher, resource: enricher} + path: /spec/tasks/{{$c.Index}}/workspaces + value: [{name: "dracon", workspace: "dracon"}] -{{range $pp := $pipelineParams}} -- op: add - path: /spec/tasks/{{$c.Index}}/params/- - value: {name: {{$pp.Name}}, value: $(params.{{$pp.Name}})} -{{end}} +# {{range $pp := $pipelineParams}} +# - op: add +# path: /spec/tasks/{{$c.Index}}/params/- +# value: {name: {{$pp.Name}}, value: $(params.{{$pp.Name}})} +# {{end}} {{end}} diff --git a/resources/patches/patch-dracon.PipelineResource.yaml b/resources/patches/patch-dracon.PipelineResource.yaml index 9d97044..e1a44e4 100644 --- a/resources/patches/patch-dracon.PipelineResource.yaml +++ b/resources/patches/patch-dracon.PipelineResource.yaml @@ -1,7 +1,7 @@ --- - op: replace path: /apiVersion - value: tekton.dev/v1alpha1 + value: tekton.dev/v1beta1 - op: add path: /metadata/labels/project diff --git a/resources/patches/patch-dracon.PipelineRun.yaml b/resources/patches/patch-dracon.PipelineRun.yaml index d1f4132..4e12b38 100644 --- a/resources/patches/patch-dracon.PipelineRun.yaml +++ b/resources/patches/patch-dracon.PipelineRun.yaml @@ -1,7 +1,7 @@ --- - op: replace path: /apiVersion - value: tekton.dev/v1alpha1 + value: tekton.dev/v1beta1 - op: add path: /metadata/labels/project diff --git a/resources/patches/patch-dracon.Producer.yaml b/resources/patches/patch-dracon.Producer.yaml index d50d6f1..a3d357d 100644 --- a/resources/patches/patch-dracon.Producer.yaml +++ b/resources/patches/patch-dracon.Producer.yaml @@ -1,7 +1,7 @@ --- - op: replace path: /apiVersion - value: tekton.dev/v1alpha1 + value: tekton.dev/v1beta1 - op: replace path: /kind value: Task @@ -11,11 +11,11 @@ value: dracon - op: add - path: /spec/inputs/resources/- + path: /spec/resources/inputs/- value: {name: source, type: storage} - op: add - path: /spec/outputs/resources/- + path: /spec/resources/outputs/- value: {name: producer, type: storage} - op: add @@ -43,6 +43,6 @@ path: /spec/steps/*/env/- value: {name: {{.Name}}, value: $(inputs.params.{{.Name}})} - op: add - path: /spec/inputs/params/- + path: /spec/params/- value: {name: {{.Name}}, type: "string"} {{end}} diff --git a/scripts/development/kind/setup.sh b/scripts/development/kind/setup.sh index 7b2e1fc..b8b3d67 100644 --- a/scripts/development/kind/setup.sh +++ b/scripts/development/kind/setup.sh @@ -11,9 +11,9 @@ kubernetes_version="${FLAGS_kubernetes_version//./_}" declare -A KUBERNETES_VERSIONS KUBERNETES_VERSIONS=( - ["1_16"]="kindest/node:v1.16.15@sha256:c10a63a5bda231c0a379bf91aebf8ad3c79146daca59db816fb963f731852a99" ["1_17"]="kindest/node:v1.17.17@sha256:7b6369d27eee99c7a85c48ffd60e11412dc3f373658bc59b7f4d530b7056823e" ["1_18"]="kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4" + ["1_19"]="kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4" ) kubernetes_context="kind-${FLAGS_kind_cluster}" diff --git a/source/git/git.sh b/source/git/git.sh index f5b48c3..b1fb977 100755 --- a/source/git/git.sh +++ b/source/git/git.sh @@ -1,13 +1,9 @@ #!/bin/sh -xe -git_src="/workspace/git-source" -out_src="/workspace/output/source/source.tgz" - +git_src="/workspace/source" cd "${git_src}" addr=$(git remote -v | cut -f1 -d" " | cut -f2 | head -n1 | cut -f2 -d"@") rev=$(git rev-parse HEAD) echo "${addr}?ref=${rev}" > .source.dracon - -tar -C "${git_src}/" -czf "${out_src}" . diff --git a/third_party/k8s/BUILD b/third_party/k8s/BUILD index ff31d56..ba07b03 100644 --- a/third_party/k8s/BUILD +++ b/third_party/k8s/BUILD @@ -33,3 +33,17 @@ remote_file( url = f"https://github.com/tektoncd/dashboard/releases/download/v{TEKTONCD_DASHBOARD_VERSION}/tekton-dashboard-release-readonly.yaml", visibility = ["//scripts/development/..."], ) + +remote_file( + name = "nfs-external-provisioner", + hashes = ["938edf9e16fdc91585a944b8f50182638fd5b653ac1311bfac73be768e2a730c"], + url = f"https://raw.githubusercontent.com/kubernetes-sigs/nfs-ganesha-server-and-external-provisioner/master/deploy/kubernetes/deployment.yaml", + visibility = ["//scripts/development/..."], +) + +remote_file( + name = "tektoncd_catalog_git-clone", + url = "https://raw.githubusercontent.com/tektoncd/catalog/main/task/git-clone/0.3/git-clone.yaml", + hashes = ["b84503226b69f807d6970f3b80cf275498229e682a8ed3a685f21276ad7b5760"], + visibility = ["//examples/pipelines/..."], +)