diff --git a/.ci/download_image.sh b/.ci/download_image.sh new file mode 100755 index 000000000..14b419031 --- /dev/null +++ b/.ci/download_image.sh @@ -0,0 +1,66 @@ +#!/bin/bash +components=( + zookeeper + zookeeper.customTools.backup + zookeeper.customTools.restore + bookie + presto + autorecovery + broker + proxy + pulsar_detector + functions + function_worker + toolset + toolset.kafka + prometheus + alert_manager + grafana + streamnative_console + node_exporter + nginx_ingress_controller + vault + vault_init + bank_vaults + custom_metric_server_prometheus + custom_metric_server + pulsar_metadata + configmapReload + external_dns + functionnmesh + pulsar_operator + bookkeeper_operator + zookeeper_operator + cert_manager_controller + cert_manager_cainjector + cert_manager_webhook + vault_operator) +sn_platform_tag=$1 +regexp="^sn-platform-([0-9]{1,3})\.([0-9]{0,3})\.([0-9]{0,3})$" +mkdir -p $sn_platform_tag +cd $sn_platform_tag +# if [[ $sn_platform_tag =~ $regexp ]]; then +# curl https://raw.githubusercontent.com/streamnative/charts/${sn_platform_tag}/charts/sn-platform/values.yaml -o values.yaml +curl https://raw.githubusercontent.com/streamnative/charts/fixed/support-upload-image-to-aws/charts/sn-platform/values.yaml -o values.yaml +image_list="" +for i in ${components[@]}; do + repository=$(cat values.yaml | yq .images.$i.repository) + tag=$(cat values.yaml | yq .images.$i.tag) + echo "Downloading docker image: $repository:$tag" + image_list=$image_list" $repository:$tag" + docker pull $repository:$tag +done + +uniq_image_list=($(echo ${image_list[*]} | sed 's/ /\n/g'|sort| uniq)) +echo ${uniq_image_list} +for j in ${uniq_image_list[@]}; do + image=$(echo ${j} | sed 's/:/-/g;s/\//-/g'); + docker save -o ${image}.tar ${j} + echo https://downloads-streamnative-cloud.oss-cn-beijing.aliyuncs.com/sn-products/$sn_platform_tag/${image}.tar +done + # test + # docker save -o $sn_platform_tag.tar jimmidyson/configmap-reload:v0.3.0 +# else +# echo "Please use the correct version number format => sn-platform-x.x.x" +# echo "Currently only sn-platform images are supported for upload" +# fi \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e99761b38..5d57091a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,3 +68,35 @@ jobs: run: | .ci/release.sh ${{ github.event.inputs.chart }} release-${{ github.event.inputs.chart }}-${{ github.event.inputs.release }} + # Upload all images to aws s3 and ali oss + - name: Setup environments + run: | + sudo apt update + sudo snap install yq + + - name: Download and package all image for sn platform + run: | + .ci/download_image.sh ${{ github.event.inputs.chart }}-${{ github.event.inputs.release }} + + - uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read --follow-symlinks --delete + env: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'us-east-1' # optional: defaults to us-east-1 + SOURCE_DIR: ${{ github.event.inputs.chart }}-${{ github.event.inputs.release }} + DEST_DIR: ${{ github.event.inputs.chart }}-${{ github.event.inputs.release }} + + - name: Install Alibaba Cloud OSSUTIL + run: wget http://gosspublic.alicdn.com/ossutil/1.7.10/ossutil64 && chmod +x ossutil64 + - name: Configure Alibaba Cloud OSSUTIL + run: ./ossutil64 config -i ${ALI_ACCESS_KEY_ID} -k ${ALI_SECRET_ACCESS_KEY} -e ${ALI_OSS_ENDPOINT} -c .ossutilconfig + env: + ALI_OSS_ENDPOINT: ${{ secrets.ALI_OSS_ENDPOINT }} + ALI_ACCESS_KEY_ID: ${{ secrets.ALI_ACCESS_KEY_ID }} + ALI_SECRET_ACCESS_KEY: ${{ secrets.ALI_SECRET_ACCESS_KEY }} + - name: Upload the file to the chosen OSS bucket + run: ./ossutil64 --config-file .ossutilconfig cp ${{ github.workspace }}/${{ github.event.inputs.chart }}-${{ github.event.inputs.release }} oss://${{ secrets.ALI_OSS_BUCKET }}/sn-products/${{ github.event.inputs.chart }}-${{ github.event.inputs.release }} -r -f + diff --git a/.github/workflows/sn_platform_image.yml b/.github/workflows/sn_platform_image.yml new file mode 100644 index 000000000..ca2594340 --- /dev/null +++ b/.github/workflows/sn_platform_image.yml @@ -0,0 +1,49 @@ +# curl -X POST https://api.github.com/repos/streamnative/sn-platform/dispatches -d '{"event_type":"upload-image-to-s3-and-oss","tag":"sn-platform-1.2.38"}' +name: Trigger Upload Image to aws s3 and oss + +on: + pull_request: + branches: + - '*' + # repository_dispatch: + # types: + # - upload-image-to-s3-and-oss + +jobs: + deploy: + name: Upload sn platform image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Setup environments + run: | + sudo apt update + sudo snap install yq + + - name: Download and package all image for sn platform + run: | # .ci/download_image.sh sn-platform-1.4.0-rc3 + .ci/download_image.sh sn-platform-1.4.0-rc3 + + - uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read --follow-symlinks --delete + env: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'us-east-1' # optional: defaults to us-east-1 + SOURCE_DIR: sn-platform-1.4.0-rc3 + DEST_DIR: sn-platform-1.4.0-rc3 + + - name: Install Alibaba Cloud OSSUTIL + run: wget http://gosspublic.alicdn.com/ossutil/1.7.10/ossutil64 && chmod +x ossutil64 + - name: Configure Alibaba Cloud OSSUTIL + run: ./ossutil64 config -i ${ALI_ACCESS_KEY_ID} -k ${ALI_SECRET_ACCESS_KEY} -e ${ALI_OSS_ENDPOINT} -c .ossutilconfig + env: + ALI_OSS_ENDPOINT: ${{ secrets.ALI_OSS_ENDPOINT }} + ALI_ACCESS_KEY_ID: ${{ secrets.ALI_ACCESS_KEY_ID }} + ALI_SECRET_ACCESS_KEY: ${{ secrets.ALI_SECRET_ACCESS_KEY }} + - name: Upload the file to the chosen OSS bucket + run: ./ossutil64 --config-file .ossutilconfig cp ${{ github.workspace }}/sn-platform-1.4.0-rc3 oss://${{ secrets.ALI_OSS_BUCKET }}/sn-products/sn-platform-1.4.0-rc3 -r -f \ No newline at end of file diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index a780563ec..88fbd1be6 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -134,22 +134,22 @@ monitoring: images: zookeeper: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent customTools: backup: repository: "streamnative/pulsar-metadata-tool" - tag: "2.9.2.9" + tag: "2.9.2.12.0" restore: repository: "streamnative/pulsar-metadata-tool" - tag: "2.9.2.9" + tag: "2.9.2.12.0" bookie: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent presto: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent # NOTE: allow overriding the presto worker image # presto_worker: @@ -158,32 +158,32 @@ images: # pullPolicy: IfNotPresent autorecovery: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent broker: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent proxy: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent pulsar_detector: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent functions: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent function_worker: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent # NOTE: allow overriding the toolset image toolset: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent kafka: repository: confluentinc/cp-kafka @@ -221,6 +221,9 @@ images: vault_init: repository: streamnative/pulsar_vault_init tag: "v1.0.2" + bank_vaults: + repository: ghcr.io/banzaicloud/bank-vaults + tag: "1.15.2" custom_metric_server_prometheus: repository: prom/prometheus tag: "v2.19.2" @@ -231,7 +234,7 @@ images: pullPolicy: IfNotPresent pulsar_metadata: repository: streamnative/sn-platform - tag: "2.9.2.9" + tag: "2.9.2.12.0" pullPolicy: IfNotPresent configmapReload: repository: jimmidyson/configmap-reload @@ -250,7 +253,39 @@ images: repository: gcr.io/google_containers/pause tag: "0.3.0" pullPolicy: IfNotPresent - + # The list of images that put the operator is here, so that we can export it to aws s3 and ali oss + functionnmesh: + repository: streamnative/function-mesh + tag: v0.1.6 + pullPolicy: IfNotPresent + pulsar_operator: + repository: docker.cloudsmith.io/streamnative/operators/pulsar-operator + tag: v0.9.4 + pullPolicy: IfNotPresent + bookkeeper_operator: + repository: docker.cloudsmith.io/streamnative/operators/pulsar-operator + tag: v0.9.4 + pullPolicy: IfNotPresent + zookeeper_operator: + repository: docker.cloudsmith.io/streamnative/operators/pulsar-operator + tag: v0.9.4 + pullPolicy: IfNotPresent + cert_manager_controller: + repository: quay.io/jetstack/cert-manager-controller + tag: v1.3.1 + pullPolicy: IfNotPresent + cert_manager_cainjector: + repository: quay.io/jetstack/cert-manager-cainjector + tag: v1.3.1 + pullPolicy: IfNotPresent + cert_manager_webhook: + repository: quay.io/jetstack/cert-manager-webhook + tag: v1.3.1 + pullPolicy: IfNotPresent + vault_operator: + repository: banzaicloud/vault-operator + tag: "1.3.0" + pullPolicy: IfNotPresent ## TLS ## templates/tls-certs.yaml ##