Merge experiment-trigger-jenkins

Signed-off-by: Thanawan Atchariyachanvanit <[email protected]>
opensearch-project · Aug 9, 2023 · 732181c · 732181c
1 parent 440d4bf
commit 732181c
Show file tree

Hide file tree

Showing 5 changed files with 262 additions and 126 deletions.
diff --git a/.github/workflows/model_listing_uploader.yml b/.github/workflows/model_listing_uploader.yml
@@ -4,8 +4,10 @@ on:
     branches: # TO-BE-CHANGED
      - workflow-for-review 
      - experiment-description
+     - experiment-trigger-jenkins
     paths: 
      - utils/model_uploader/model_listing/pretrained_model_listing.json
+  workflow_dispatch: # TODO: Remove this
 
 jobs:
   upload-model-listing:
@@ -34,22 +36,18 @@ jobs:
     - name: Update pre_trained_models.json in S3
       run: aws s3 cp ${{ env.repo_model_listing_path }} s3://${{ secrets.PERSONAL_MODEL_BUCKET }}/${{ env.bucket_model_listing_file_path }}
 
-# # TODO: Trigger Jenkins Workflow
-#   trigger-model-release-workflow:
-#     runs-on: 'ubuntu-latest'
-#     steps:
-#       - name: Trigger Jenkins Workflow with Generic Webhook
-#         run: |
-#           JENKINS_URL="https://build.ci.opensearch.org"
-#           JENKINS_TRIGGER_TOKEN=${{ secrets.JENKINS_ML_MODELS_RELEASE_GENERIC_WEBHOOK_TOKEN }}
-
-#           JENKINS_PARAMS="{\"BASE_DOWNLOAD_PATH\":\"ml-models/model_listing\"}"
-
-#           # TODO: Set up JENKINS_TRIGGER_TOKEN
-#           JENKINS_REQ=`curl -s -XPOST \
-#              -H "Authorization: Bearer $JENKINS_TRIGGER_TOKEN" \
-#              -H "Content-Type: application/json" \
-#              "$JENKINS_URL/generic-webhook-trigger/invoke" \
-#              --data "$(echo $JENKINS_PARAMS)"`
-
-#           echo $JENKINS_REQ
+  trigger-ml-models-release-workflow:
+    needs: upload-model-listing
+    runs-on: 'ubuntu-latest'
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+      - name: Trigger Jenkins Workflow with Generic Webhook
+        run: |
+          # TODO: Set up JENKINS_TRIGGER_TOKEN & Remove ${{ secrets.JENKINS_URL }}
+          JENKINS_URL=${{ secrets.JENKINS_URL }} # "https://build.ci.opensearch.org"
+          JENKINS_TRIGGER_TOKEN=${{ secrets.JENKINS_ML_MODELS_RELEASE_GENERIC_WEBHOOK_TOKEN }}
+          JENKINS_PARAMS="{\"BASE_DOWNLOAD_PATH\":\"ml-models/model_listing\"}"
+          sh utils/model_uploader/trigger_ml_models_release.sh $JENKINS_TRIGGER_TOKEN $JENKINS_PARAMS $JENKINS_URL
diff --git a/.github/workflows/model_uploader.yml b/.github/workflows/model_uploader.yml
@@ -39,7 +39,7 @@ on:
         - "MAX"
         - "MEAN_SQRT_LEN"
       model_description:
-        description: "(Optional) Description (Specify here if you want to overwrite the default model description)(Does not accept double quotes)"
+        description: "(Optional) Description (Specify here if you want to overwrite the default model description)"
         required: false
         type: string
 
@@ -67,20 +67,20 @@ jobs:
         model_description="${{ github.event.inputs.model_description }}"
         
         workflow_info="
-        ========= Workflow Details ==========
+        ============= Workflow Details ==============
         - Workflow Name: ${{ github.workflow }}
         - Workflow Run ID: ${{ github.run_id }}
         - Workflow Initiator: @${{ github.actor }}
           
-        ========= Model Information =========
+        ========== Model Input Information ==========
         - Model ID: ${{ github.event.inputs.model_id }}
         - Model Version: ${{ github.event.inputs.model_version }}
         - Tracing Format: ${{ github.event.inputs.tracing_format }}
         - Embedding Dimension: ${embedding_dimension:-Default}
         - Pooling Mode: ${pooling_mode:-Default}
         - Model Description: ${model_description:-Default}
         
-        ========= Test Information ==========
+        ======== Workflow Output Information =========
         - Embedding Verification: Passed"
 
         echo "workflow_info<<EOF" >> $GITHUB_OUTPUT
@@ -93,7 +93,7 @@ jobs:
         echo "verified=:white_check_mark: — It is verified that this model is licensed under Apache 2.0" >> $GITHUB_OUTPUT
         echo "unverified=- [ ]  :warning: The license cannot be verified. Please confirm by yourself that the model is licensed under Apache 2.0  :warning:" >> $GITHUB_OUTPUT
     outputs:
-      model_folder: ${{ steps.init_folders.outputs.sentence_transformer_folder }}
+      model_folder: ${{ steps.init_folders.outputs.model_folder }}
       sentence_transformer_folder: ${{ steps.init_folders.outputs.sentence_transformer_folder }}
       workflow_info: ${{ steps.init_workflow_info.outputs.workflow_info }}
       verified_license_line: ${{ steps.init_license_line.outputs.verified }}
@@ -129,7 +129,7 @@ jobs:
           aws s3api head-object --bucket ${{ secrets.PERSONAL_MODEL_BUCKET }} --key $TORCH_FILE_PATH > /dev/null 2>&1 || TORCH_MODEL_NOT_EXIST=true
           if [[ -z $TORCH_MODEL_NOT_EXIST ]]
           then
-            echo "${{ github.event.inputs.model_id }} already exists on model hub for TORCH_SCRIPT format and ${{ github.event.inputs.model_version }} version."
+            echo "TORCH_SCRIPT Model already exists on model hub."
             exit 1
           fi
       - name: Check if ONNX Model Exists
@@ -141,7 +141,7 @@ jobs:
           aws s3api head-object --bucket ${{ secrets.PERSONAL_MODEL_BUCKET }} --key $ONNX_FILE_PATH > /dev/null 2>&1 || ONNX_MODEL_NOT_EXIST=true
           if [[ -z $ONNX_MODEL_NOT_EXIST ]]
           then
-            echo "${{ github.event.inputs.model_id }} already exists on model hub for ONNX format and ${{ github.event.inputs.model_version }} version."
+            echo "TORCH_SCRIPT Model already exists on model hub."
             exit 1
           fi
   
@@ -173,21 +173,23 @@ jobs:
           echo "MODEL_DESCRIPTION=${{ github.event.inputs.model_description }}" >> $GITHUB_ENV     
       - name: Autotracing ${{ matrix.cluster }} secured=${{ matrix.secured }} version=${{matrix.entry.opensearch_version}}
         run: "./.ci/run-tests ${{ matrix.cluster }} ${{ matrix.secured }} ${{ matrix.entry.opensearch_version }} trace"
-      - name: License verification
+      - name: License Verification
         id: license_verification
-        continue-on-error: true
         run: |
           apache_verified=$(<trace_output/apache_verified.txt)
           if [[ $apache_verified == "True" ]]
           then
-            echo "license_line=${{ needs.init-workflow-var.outputs.verified_license_line }}" >> $GITHUB_OUTPUT   
+            echo "license_line=${{ needs.init-workflow-var.outputs.verified_license_line }}" >> $GITHUB_OUTPUT
+            echo "license_info=Automatically Verified" >> $GITHUB_OUTPUT
           else
-            echo "license_line=${{ needs.init-workflow-var.outputs.unverified_license_line }}" >> $GITHUB_OUTPUT   
+            echo "license_line=${{ needs.init-workflow-var.outputs.unverified_license_line }}" >> $GITHUB_OUTPUT 
+            echo "license_info=Manually Verified" >> $GITHUB_OUTPUT
           fi
-      - name: Handle if license_verification fails
-        if: steps.license_verification.outcome == 'failure'
+      - name: Model Description Info
+        id: model_description_info
         run: |
-          echo "license_line=${{ needs.init-workflow-var.outputs.unverified_license_line }}" >> $GITHUB_OUTPUT
+          model_description_info=$(<trace_output/description.txt)
+          echo "model_description_info=$model_description_info" >> $GITHUB_OUTPUT
       - name: Upload Artifact
         uses: actions/upload-artifact@v3
         with: 
@@ -213,6 +215,8 @@ jobs:
           echo "${dryrun_output@E}"
     outputs:
       license_line: ${{ steps.license_verification.outputs.license_line }}
+      license_info: ${{ steps.license_verification.outputs.license_info }}
+      model_description_info: ${{ steps.model_description.outputs.model_description_info }}
       dryrun_output: ${{ steps.dryrun_model_uploading.outputs.dryrun_output }}
 
   # Step 5: Ask for manual approval from the CODEOWNERS
@@ -236,6 +240,7 @@ jobs:
           ${{ needs.model-auto-tracing.outputs.license_line }}
           
           ${{ needs.init-workflow-var.outputs.workflow_info }}
+          ${{ needs.model-auto-tracing.outputs.model_description_info }}
           
           ===== Dry Run of Model Uploading =====
           ${{ needs.model-auto-tracing.outputs.dryrun_output }}"
@@ -320,7 +325,9 @@ jobs:
           - [ ] CHANGELOG.md has been updated by the workflow or by you if the workflow fails to do so. 
           - [ ] Merge conflicts have been resolved.
           
-          ${{ needs.init-workflow-var.outputs.workflow_info }}"
+          ${{ needs.init-workflow-var.outputs.workflow_info }}
+          ${{ needs.model-auto-tracing.outputs.license_info }}
+          ${{ needs.model-auto-tracing.outputs.model_description }}"
           
           echo "pr_body<<EOF" >> $GITHUB_OUTPUT
           echo "${pr_body@E}" >> $GITHUB_OUTPUT
@@ -384,27 +391,27 @@ jobs:
             Warning:exclamation:: The workflow failed to update CHANGELOG.md. Please add the following line manually.
             ${{ steps.create_changelog_line.outputs.changelog_line }}
             
-  # # Step 8: Trigger Jenkins ml-models workflow
-  # trigger-model-release-workflow:
-  #   needs: [init-workflow-var, history-update]
-  #   runs-on: 'ubuntu-latest'
-  #   steps:
-  #     - name: Trigger Jenkins Workflow with Generic Webhook
-  #       run: |
-  #         JENKINS_URL="https://build.ci.opensearch.org"
-  #         JENKINS_TRIGGER_TOKEN=${{ secrets.JENKINS_ML_MODELS_RELEASE_GENERIC_WEBHOOK_TOKEN }}
-
-  #         BASE_DOWNLOAD_PATH=${{ needs.init-workflow-var.outputs.model_folder }}
-  #         VERSION=${{ github.event.inputs.model_version }}
-  #         MODEL_FORMAT=${{ github.event.inputs.tracing_format }}
-  #         JENKINS_PARAMS="{\"BASE_DOWNLOAD_PATH\":\"$BASE_DOWNLOAD_PATH\", \"VERSION\":\"$VERSION\", \"TRACING_FORMAT\": \"$TRACING_FORMAT\"}"
-
-  #         # TODO: Set up JENKINS_TRIGGER_TOKEN
-  #         JENKINS_REQ=`curl -s -XPOST \
-  #            -H "Authorization: Bearer $JENKINS_TRIGGER_TOKEN" \
-  #            -H "Content-Type: application/json" \
-  #            "$JENKINS_URL/generic-webhook-trigger/invoke" \
-  #            --data "$(echo $JENKINS_PARAMS)"`
+  # Step 8: Trigger Jenkins ml-models workflow
+  trigger-ml-models-release-workflow:
+    needs: [init-workflow-var, history-update]
+    runs-on: 'ubuntu-latest'
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+      - name: Trigger Jenkins Workflow with Generic Webhook
+        run: |
+          # TODO: Set up JENKINS_TRIGGER_TOKEN & Remove ${{ secrets.JENKINS_URL }}
+          # sh utils/model_uploader/trigger_ml_models_release.sh ${{ secrets.JENKINS_ML_MODELS_RELEASE_GENERIC_WEBHOOK_TOKEN }} \
+          #   ${{ needs.init-workflow-var.outputs.model_folder }} ${{ github.event.inputs.model_version }} ${{ github.event.inputs.tracing_format }} \
+          #   ${{ secrets.JENKINS_URL }} 
 
-  #         echo $JENKINS_REQ
-
+          JENKINS_URL=${{ secrets.JENKINS_URL }} # "https://build.ci.opensearch.org"
+          JENKINS_TRIGGER_TOKEN=${{ secrets.JENKINS_ML_MODELS_RELEASE_GENERIC_WEBHOOK_TOKEN }}
+          
+          BASE_DOWNLOAD_PATH=${{ needs.init-workflow-var.outputs.model_folder }}
+          VERSION=${{ github.event.inputs.model_version }}
+          FORMAT=${{ github.event.inputs.tracing_format }}
+          JENKINS_PARAMS="{\"BASE_DOWNLOAD_PATH\":\"$BASE_DOWNLOAD_PATH\", \"VERSION\":\"$VERSION\", \"FORMAT\":\"$FORMAT\"}"
+          sh utils/model_uploader/trigger_ml_models_release.sh $JENKINS_TRIGGER_TOKEN "$JENKINS_PARAMS" $JENKINS_URL