diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..e6dc42a --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,2 @@ +[bumpversion] +current_version = 2.23.0 diff --git a/.github/lint-config.yaml b/.github/lint-config.yaml new file mode 100644 index 0000000..21095dd --- /dev/null +++ b/.github/lint-config.yaml @@ -0,0 +1 @@ +target-branch: "main" diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml new file mode 100644 index 0000000..58cd9c4 --- /dev/null +++ b/.github/workflows/helm-lint.yaml @@ -0,0 +1,14 @@ +name: Helm lint + +on: + push: + +jobs: + helm-lint: + runs-on: ubuntu-22.04 + steps: + - name: Lint Helm chart + uses: bakdata/ci-templates/actions/helm-lint@1.46.3 + with: + lint-config-path: ".github/lint-config.yaml" + ref: ${{ github.ref_name }} diff --git a/.github/workflows/helm-publish.yaml b/.github/workflows/helm-publish.yaml new file mode 100644 index 0000000..d273d28 --- /dev/null +++ b/.github/workflows/helm-publish.yaml @@ -0,0 +1,37 @@ +name: Publish Helm Charts + +on: + push: + +jobs: + get-version: + name: Get version from bumpversion + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get-version.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Get current version from bumpversion + id: get-version + run: | + version=$(sed -nE 's/.*current_version = (.*)/\1/p' < .bumpversion.cfg) + if ! [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then + version="${version}-SNAPSHOT" + fi + echo "version=$version" >> $GITHUB_OUTPUT + shell: bash + + call-workflow-passing-data: + name: Publish Helm chart + uses: bakdata/ci-templates/.github/workflows/helm-multi-release.yaml@1.46.4 + needs: get-version + with: + charts-path: "./charts" + subdirs: "['rclone-copy']" + gh-pages-branch: gh-pages + version: ${{ needs.get-version.outputs.version }} + secrets: + github-username: ${{ secrets.GH_USERNAME }} + github-email: ${{ secrets.GH_EMAIL }} + github-token: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..dcaf8d9 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,25 @@ +name: Release + +on: + workflow_dispatch: + inputs: + release-type: + description: "The scope of the release (major, minor or patch)." + type: choice + required: true + default: patch + options: + - patch + - minor + - major + +jobs: + bump-version-release: + name: Bumpversion Release + uses: bakdata/ci-templates/.github/workflows/bump-version-release.yaml@1.46.4 + with: + release-type: "${{ inputs.release-type }}" + secrets: + github-email: "${{ secrets.GH_EMAIL }}" + github-username: "${{ secrets.GH_USERNAME }}" + github-token: "${{ secrets.GH_TOKEN }}" diff --git a/charts/.helmignore b/charts/.helmignore new file mode 100644 index 0000000..a1006ff --- /dev/null +++ b/charts/.helmignore @@ -0,0 +1,2 @@ +*.tgz +.* diff --git a/charts/rclone-copy/Chart.yaml b/charts/rclone-copy/Chart.yaml new file mode 100644 index 0000000..1ef2c03 --- /dev/null +++ b/charts/rclone-copy/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +name: rclone-copy +description: A helm chart for rclone copy on Kubernetes. +version: 1.0.0 +maintainers: + - name: bakdata + email: opensource@bakdata.com + url: bakdata.com diff --git a/charts/rclone-copy/README.md b/charts/rclone-copy/README.md new file mode 100644 index 0000000..6aaecb0 --- /dev/null +++ b/charts/rclone-copy/README.md @@ -0,0 +1,16 @@ +# rclone Helm for Kubernetes + +## Usage: +- Create `rclone.conf` (either via `rclone configure` or from `rclone.conf.template`) +- Adapt `values.yaml` to contain your values. +- Install helm with: `helm install . -f values.yaml -n "mySource-mySoruceType-myDestType"` in this directory. + +## Creating rclone.conf from scratch +- Download `rclone` (on Mac e.g. `brew install rclone`) +- Run the following commands: +```bash +cd my-repository +rclone config --config "rclone.conf" +``` +- Select `n` for "New remote" +- Now continue to follow the outlined steps for the storage provider you chose diff --git a/charts/rclone-copy/templates/_helpers.tpl b/charts/rclone-copy/templates/_helpers.tpl new file mode 100644 index 0000000..4028b9c --- /dev/null +++ b/charts/rclone-copy/templates/_helpers.tpl @@ -0,0 +1,29 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "rclone-copy.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "rclone-copy.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Values.nameOverride -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "rclone-copy.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/rclone-copy/templates/configmap.yaml b/charts/rclone-copy/templates/configmap.yaml new file mode 100644 index 0000000..6255f67 --- /dev/null +++ b/charts/rclone-copy/templates/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rclone-config-{{ .Release.Name }} +data: + rclone.conf: | +{{ .Values.rcloneConf | indent 4 }} + include-pattern.conf: | +{{ .Values.includePattern | indent 4 }} diff --git a/charts/rclone-copy/templates/rclone-cron.yaml b/charts/rclone-copy/templates/rclone-cron.yaml new file mode 100644 index 0000000..7ddf601 --- /dev/null +++ b/charts/rclone-copy/templates/rclone-cron.yaml @@ -0,0 +1,88 @@ +{{- $root := . -}} +{{- if .Capabilities.APIVersions.Has "batch/v1/CronJob" }} +apiVersion: batch/v1 +{{- else }} +apiVersion: batch/v1beta1 +{{- end }} +kind: CronJob +metadata: + name: {{ template "rclone-copy.fullname" . }} +{{- if .Values.annotations }} + annotations: + {{- range $key, $value := .Values.annotations }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} +{{- end }} + labels: + app: {{ template "rclone-copy.name" . }} + chart: {{ template "rclone-copy.chart" . }} + release: {{ .Release.Name }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value }} + {{- end }} +spec: + suspend: {{ .Values.suspend }} + schedule: "{{ .Values.schedule }}" + successfulJobsHistoryLimit: {{ .Values.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.failedJobsHistoryLimit }} + concurrencyPolicy: Replace + jobTemplate: + spec: + template: + metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- range $key, $value := .Values.podAnnotations }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} + {{- end }} + labels: + app: {{ template "rclone-copy.name" . }} + release: {{ .Release.Name }} + {{- range $key, $value := .Values.podLabels }} + {{ $key }}: {{ $value }} + {{- end }} + spec: + {{- if .Values.serviceAccountName }} + serviceAccountName: {{ .Values.serviceAccountName }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 12 }} + {{- end }} + containers: + - name: rclone-container + image: rclone/rclone:{{ .Values.imageRelease }} + + command: + - /bin/sh + args: + - -c + # copy as workaround for rclone.conf read only (see https://github.com/rclone/rclone/issues/3655) + - >- + cp /root/.config/rclone/rclone_ro.conf /root/.config/rclone/rclone.conf && + rclone copy -v {{ .Values.arguments | join " " }} --include-from /root/include-pattern.conf "{{ .Values.sync.source.name }}:{{ .Values.sync.source.path }}" "{{ .Values.sync.dest.name }}:{{ .Values.sync.dest.path }}" + + volumeMounts: + - name: config + # This is the default path where the rclone implementation assumes the config is located + mountPath: "/root/.config/rclone/rclone_ro.conf" + subPath: "rclone.conf" + - name: config + mountPath: "/root/include-pattern.conf" + subPath: "include-pattern.conf" + + resources: +{{ toYaml .Values.resources | indent 14 }} + + restartPolicy: {{ .Values.restartPolicy }} + volumes: + - name: config + configMap: + name: rclone-config-{{ .Release.Name }} + backoffLimit: {{ .Values.backoffLimit }} + diff --git a/charts/rclone-copy/values.yaml b/charts/rclone-copy/values.yaml new file mode 100644 index 0000000..ace2fc0 --- /dev/null +++ b/charts/rclone-copy/values.yaml @@ -0,0 +1,78 @@ +nameOverride: bakdata-rclone-copy + +# This is the cofiguration for the specific rclone-cronjob that you want to run. +# `sync` contains the config for the source and destination of the rclone copy job. +# It can be interpreted as: +# rclone copy {source.name}:{source.path} {dest.name}:{dest.path} +sync: + source: + # This is the name for the source remote from the rclone.conf file. + # If the value specified here is not present in the config file, the job will fail. + # See rclone.conf.template for an example. + name: my-sftp + + # This is the path to the source directory. + path: /tmp/sync-dir + + dest: + # This is the name for the destination remote from the rclone.conf file. + # If the value specified here is not present in the config file, the job will fail. + # See rclone.conf.template for an example. + name: my-s3 + + # This is the path to the target directory. + path: my-s3-bucket + +# Additional arguments to pass to rclone. +# The expected value is an array with comma seperated flag and value passed as a string. +# Example: ["--transfers", '"1"'] +arguments: [] + +# Pattern of files to copy. Standard UNIX file glob patterns are used. Default: "*" +# See templates/include-pattern-config.yaml for more details on usage in ConfigMap. +includePattern: "*" + +# Cron schedule for this connect job. Default is at 12:00 every day. +schedule: "0 12 * * *" + +suspend: false + +# Content of the rclone.conf file. +rcloneConf: "" + +# Release version for the openbridge/ob_bulkstash docker image. +# You probably don't need to change this. +imageRelease: 1.53.1 + +restartPolicy: OnFailure + +# serviceAccountName: foo + +tolerations: [] +# - key: "foo" +# operator: "Exists" +# effect: "NoSchedule" +# - key: "bar" +# operator: "Exists" +# effect: "NoSchedule" + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +resources: + requests: + cpu: 200m + memory: 300Mi + limits: + memory: 2G + cpu: 500m + +successfulJobsHistoryLimit: 1 +failedJobsHistoryLimit: 1 +backoffLimit: 6 + +podAnnotations: {} + +podLabels: {}