This repository has been archived by the owner on Aug 14, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 97
93 lines (89 loc) · 4.01 KB
/
run-eval-pf-pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
name: Test and Evaulate Prompts with Promptflow
on:
workflow_dispatch:
push:
branches: [ main ]
env:
GROUP: ${{secrets.GROUP}}
WORKSPACE: ${{secrets.WORKSPACE}}
SUBSCRIPTION_ID: ${{secrets.SUBSCRIPTION_ID}}
RUN_NAME: web_classification_variant_1_20230816_215600_605116
EVAL_RUN_NAME: classification_accuracy_eval_default_20230821_111809_077086
jobs:
login-and-run-and-evalpf:
runs-on: ubuntu-latest
steps:
- name: Check out repo
uses: actions/checkout@v2
- name: Install az ml extension
run: az extension add -n ml -y
- name: Azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.11.4'
- name: List current directory
run: ls
- name: Install promptflow
run: pip install -r promptflow/web-classification/requirements.txt
- name: Run promptflow
run: |
pfazure run create -f promptflow/web-classification/run.yml --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > promptflow/llmops-helper/run_info.txt
cat promptflow/llmops-helper/run_info.txt
- name: Set run name
run: |
echo "RUN_NAME=$(python promptflow/llmops-helper/parse_run_output.py run_info.txt)" >> "$GITHUB_ENV"
- name: Show the current run name
run: echo "Run name is:" ${{env.RUN_NAME}}
- name: Show promptflow results
run: pfazure run show-details --name ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
- name: Run promptflow evaluations
run: pfazure run create -f promptflow/web-classification/run_evaluation.yml --run ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > promptflow/llmops-helper/eval_info.txt
- name: Get eval run name
run: echo "EVAL_RUN_NAME=$(python promptflow/llmops-helper/parse_run_output.py eval_info.txt)" >> "$GITHUB_ENV"
- name: Show promptflow details
run: pfazure run show-details --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
- name: Show promptflow metrics
run: pfazure run show-metrics --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} > promptflow/llmops-helper/eval_result.json
assert-and-register-model:
needs: login-and-run-and-evalpf
runs-on: ubuntu-latest
steps:
- name: Check out repo
uses: actions/checkout@v2
- name: Install az ml extension
run: az extension add -n ml -y
- name: Azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.11.4'
- name: Set default subscription
run: |
az account set -s ${{env.SUBSCRIPTION}}
- name: List current directory
run: ls
- name: Install promptflow
run: pip install -r promptflow/web-classification/requirements.txt
- name: Get assert eval results
id: jobMetricAssert
run: |
export ASSERT=$(python promptflow/llmops-helper/assert.py result.json 0.6) # NOTE <file>.json is the file name and decimal is the threshold for the assertion
echo "::debug::Assert has returned the following value: $ASSERT"
# assert.py will return True or False, but bash expects lowercase.
if ${ASSERT,,} ; then
echo "::debug::Prompt flow run met the quality bar and can be deployed."
echo "::set-output name=result::true"
else
echo "::warning::Prompt flow run didn't meet quality bar."
echo "::set-output name=result::false"
fi
- name: Register promptflow model
if: ${{ steps.jobMetricAssert.outputs.result == 'true' }}
run: az ml model create --file promptflow/deployment/model.yaml -g ${{env.GROUP}} -w ${{env.WORKSPACE}}