forked from opensearch-project/opensearch-py-ml
-
Notifications
You must be signed in to change notification settings - Fork 0
140 lines (137 loc) · 6.3 KB
/
update_model_listing.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
name: Update Pretrained Model Listing
on:
workflow_dispatch:
jobs:
update-model-listing:
runs-on: 'ubuntu-latest'
permissions:
id-token: write
contents: write
pull-requests: write
environment: opensearch-py-ml-cicd-env
env:
bucket_model_listing_file_path: ml-models/model_listing/pre_trained_models.json
repo_model_listing_path: ./utils/model_uploader/model_listing/pretrained_model_listing.json
path_prefixes: "ml-models/huggingface/ ml-models/abc/"
steps:
# DEMO
# - name: Fail if branch is not main
# if: github.ref == 'refs/heads/main'
# run: |
# echo "This workflow should only be triggered on 'main' branch"
# exit 1
- name: Checkout Main Branch
uses: actions/checkout@v3
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v2
with:
aws-region: ${{ secrets.PERSONAL_MODEL_UPLOADER_AWS_REGION }}
role-to-assume: ${{ secrets.PERSONAL_MODEL_UPLOADER_ROLE }}
role-session-name: update-model-listing
- name: List Models
run: |
path_prefixes="${{ env.path_prefixes }}"
for prefix in $path_prefixes
do
aws s3api list-objects --bucket ${{ secrets.PERSONAL_MODEL_BUCKET }} --prefix $prefix --query "Contents[].{Key: Key}" --output text | grep "/config.json$" >> config_paths.txt
done
echo $(cat config_paths.txt)
- name: Download config files
run: |
mkdir config_folder
path_prefixes="${{ env.path_prefixes }}"
for prefix in $path_prefixes
do
echo $prefix
aws s3 cp s3://${{ secrets.PERSONAL_MODEL_BUCKET }}/$prefix config_folder --recursive --exclude "*" --include "*/config.json"
done
echo $(ls config_folder)
- name: Set Up Python
uses: actions/setup-python@v2
with:
python-version: '3.x'
- name: Update pre_trained_models.json
run: |
python utils/model_uploader/update_pretrained_model_listing.py "config_paths.txt" "config_folder"
- name: Create PR Body
id: create_pr_body
run: |
update_time=$(TZ='America/Los_Angeles' date "+%Y-%m-%d %T")
echo "update_time=$update_time" >> $GITHUB_OUTPUT
pr_body="
- [ ] This PR made commit to only these two files: pretrained_model_listing.json and CHANGELOG.md.
- [ ] CHANGELOG.md has been updated by the workflow or by you if the workflow fails to do so.
- [ ] Merge conflicts have been resolved.
========= Workflow Details ==========
- Workflow Name: ${{ github.workflow }}
- Workflow Run ID: ${{ github.run_id }}
- Workflow Initiator: @${{ github.actor }}
- File Update Time: $update_time"
echo "pr_body<<EOF" >> $GITHUB_OUTPUT
echo "${pr_body@E}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
echo "${pr_body@E}"
- name: Create a Branch & Raise a PR
uses: peter-evans/create-pull-request@v5
id: create_pr
with:
committer: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
commit-message: 'GitHub Actions Workflow: Update Pretrained Model Listing'
signoff: true
title: 'Update Pretrained Model Listing - ${{ steps.create_pr_body.outputs.update_time }}'
body: ${{ steps.create_pr_body.outputs.pr_body }}
labels: ModelListingUploading
branch: model-listing-uploader/${{ github.run_id }}
delete-branch: true
add-paths: ${{ env.repo_model_listing_path }}
- name: Checkout PR Branch
id: checkout_pr_branch
continue-on-error: true
uses: actions/checkout@v3
with:
ref: model-listing-uploader/${{ github.run_id }}
- name: Create a line for updating CHANGELOG.md
id: create_changelog_line
if: steps.checkout_pr_branch.outcome == 'success'
continue-on-error: true
run: |
pr_ref="([#${{ steps.create_pr.outputs.pull-request-number }}](${{ steps.create_pr.outputs.pull-request-url }}))"
changelog_line="Update pretrained_model_listing.json (${{ steps.create_pr_body.outputs.update_time }}) by @${{ github.actor }} $pr_ref"
echo "changelog_line=$changelog_line" >> $GITHUB_OUTPUT
- name: Warning Comment on PR if create_changelog_line fails
if: steps.checkout_pr_branch.outcome == 'success' && steps.create_changelog_line.outcome == 'failure'
uses: thollander/actions-comment-pull-request@v2
with:
pr_number: ${{ steps.create_pr.outputs.pull-request-number }}
message: "Warning:exclamation:: The workflow failed to update CHANGELOG.md. Please update CHANGELOG.md manually."
- name: Update CHANGELOG.md
if: steps.checkout_pr_branch.outcome == 'success' && steps.create_changelog_line.outcome == 'success'
id: update_changelog
continue-on-error: true
run: |
python -m pip install mdutils
python utils/model_uploader/update_changelog_md.py "${{ steps.create_changelog_line.outputs.changelog_line }}"
- name: Commit Updates
if: steps.checkout_pr_branch.outcome == 'success' && steps.create_changelog_line.outcome == 'success' && steps.update_changelog.outcome == 'success'
uses: stefanzweifel/git-auto-commit-action@v4
id: commit
with:
branch: model-listing-uploader/${{ github.run_id }}
commit_user_email: "github-actions[bot]@users.noreply.github.com"
commit_message: 'GitHub Actions Workflow: Update CHANGELOG.md - ${{ env.model_info }}'
commit_options: '--signoff'
file_pattern: CHANGELOG.md
- name: Warning Comment on PR if update_changelog fails
if: steps.checkout_pr_branch.outcome == 'success' && steps.create_changelog_line.outcome == 'success' && steps.update_changelog.outcome == 'failure'
uses: thollander/actions-comment-pull-request@v2
with:
pr_number: ${{ steps.create_pr.outputs.pull-request-number }}
message: |
Warning:exclamation:: The workflow failed to update CHANGELOG.md. Please add the following line manually.
>>>
${{ steps.create_changelog_line.outputs.changelog_line }}
- name: No Change in Model Listing
if: steps.checkout_pr_branch.outcome == 'failure'
run: |
echo "There is no change in model listing."
echo "Exiting the workflow"