Skip to content

Commit 598826e

Browse files
ntnyarpechenin
ntny
and
arpechenin
authored
fix component retry test (#11836)
Signed-off-by: ntny <ntny1986@gmail.com> Signed-off-by: arpechenin <arpechenin@avito.ru> Co-authored-by: arpechenin <arpechenin@avito.ru>
1 parent a4f0723 commit 598826e

2 files changed

Lines changed: 114 additions & 101 deletions

File tree

‎backend/src/v2/compiler/argocompiler/argo_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ func Test_argo_compiler(t *testing.T) {
4141
argoYAMLPath string // path of expected output argo workflow YAML
4242
envVars map[string]string
4343
}{
44+
{
45+
jobPath: "../testdata/hello_world_with_retry.json",
46+
platformSpecPath: "",
47+
argoYAMLPath: "testdata/hello_world_with_retry.yaml",
48+
},
4449
{
4550
jobPath: "../testdata/hello_world.json",
4651
platformSpecPath: "",

‎backend/src/v2/compiler/argocompiler/testdata/hello_world_with_retry.yaml‎

Lines changed: 109 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
apiVersion: argoproj.io/v1alpha1
22
kind: Workflow
33
metadata:
4-
annotations:
5-
pipelines.kubeflow.org/components-comp-hello-world: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}'
6-
pipelines.kubeflow.org/components-root: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"retryPolicy":{"backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}'
7-
pipelines.kubeflow.org/implementations-comp-hello-world:
8-
'{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf
9-
\"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def
10-
hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser
11-
= argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\",
12-
dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args
13-
= vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.7"}'
144
creationTimestamp: null
155
generateName: hello-world-
166
spec:
17-
arguments: {}
7+
arguments:
8+
parameters:
9+
- name: components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7
10+
value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}'
11+
- name: implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7
12+
value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf
13+
\"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def
14+
hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser
15+
= argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\",
16+
dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args
17+
= vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.9"}'
18+
- name: components-root
19+
value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"retryPolicy":{"backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}'
1820
entrypoint: entrypoint
1921
podMetadata:
2022
annotations:
@@ -30,34 +32,38 @@ spec:
3032
- --pipeline_name
3133
- namespace/n1/pipeline/hello-world
3234
- --run_id
33-
- "{{workflow.uid}}"
35+
- '{{workflow.uid}}'
36+
- --run_name
37+
- '{{workflow.name}}'
38+
- --run_display_name
39+
- ""
3440
- --dag_execution_id
35-
- "{{inputs.parameters.parent-dag-id}}"
41+
- '{{inputs.parameters.parent-dag-id}}'
3642
- --component
37-
- "{{inputs.parameters.component}}"
43+
- '{{inputs.parameters.component}}'
3844
- --task
39-
- "{{inputs.parameters.task}}"
45+
- '{{inputs.parameters.task}}'
4046
- --container
41-
- "{{inputs.parameters.container}}"
47+
- '{{inputs.parameters.container}}'
4248
- --iteration_index
43-
- "{{inputs.parameters.iteration-index}}"
49+
- '{{inputs.parameters.iteration-index}}'
4450
- --cached_decision_path
45-
- "{{outputs.parameters.cached-decision.path}}"
51+
- '{{outputs.parameters.cached-decision.path}}'
4652
- --pod_spec_patch_path
47-
- "{{outputs.parameters.pod-spec-patch.path}}"
53+
- '{{outputs.parameters.pod-spec-patch.path}}'
4854
- --condition_path
49-
- "{{outputs.parameters.condition.path}}"
55+
- '{{outputs.parameters.condition.path}}'
5056
- --kubernetes_config
51-
- "{{inputs.parameters.kubernetes-config}}"
57+
- '{{inputs.parameters.kubernetes-config}}'
5258
- --http_proxy
53-
- ''
59+
- ""
5460
- --https_proxy
55-
- ''
61+
- ""
5662
- --no_proxy
57-
- ''
63+
- ""
5864
command:
5965
- driver
60-
image: gcr.io/ml-pipeline/kfp-driver
66+
image: ghcr.io/kubeflow/kfp-driver
6167
name: ""
6268
resources:
6369
limits:
@@ -98,10 +104,10 @@ spec:
98104
- arguments:
99105
parameters:
100106
- name: pod-spec-patch
101-
value: "{{inputs.parameters.pod-spec-patch}}"
107+
value: '{{inputs.parameters.pod-spec-patch}}'
102108
name: executor
103109
template: hello-world-system-container-impl
104-
when: "{{inputs.parameters.cached-decision}} != true"
110+
when: '{{inputs.parameters.cached-decision}} != true'
105111
inputs:
106112
parameters:
107113
- name: pod-spec-patch
@@ -110,57 +116,47 @@ spec:
110116
metadata: {}
111117
name: hello-world-system-container-executor
112118
outputs: {}
113-
- containerSet:
114-
containers:
115-
- command:
116-
- should-be-overridden-during-runtime
117-
env:
118-
- name: KFP_POD_NAME
119-
valueFrom:
120-
fieldRef:
121-
fieldPath: metadata.name
122-
- name: KFP_POD_UID
123-
valueFrom:
124-
fieldRef:
125-
fieldPath: metadata.uid
126-
envFrom:
127-
- configMapRef:
128-
name: metadata-grpc-configmap
129-
optional: true
130-
image: gcr.io/ml-pipeline/should-be-overridden-during-runtime
131-
name: main
132-
resources: {}
133-
volumeMounts:
134-
- mountPath: /kfp-launcher
135-
name: kfp-launcher
136-
- command:
137-
- watcher
138-
env:
139-
- name: KFP_POD_NAME
140-
valueFrom:
141-
fieldRef:
142-
fieldPath: metadata.name
143-
- name: KFP_POD_UID
144-
valueFrom:
145-
fieldRef:
146-
fieldPath: metadata.uid
147-
envFrom:
148-
- configMapRef:
149-
name: metadata-grpc-configmap
150-
optional: true
151-
name: watcher
152-
resources:
153-
limits:
154-
cpu: 500m
155-
memory: 128Mi
156-
requests:
157-
cpu: 100m
119+
- container:
120+
command:
121+
- should-be-overridden-during-runtime
122+
env:
123+
- name: KFP_POD_NAME
124+
valueFrom:
125+
fieldRef:
126+
fieldPath: metadata.name
127+
- name: KFP_POD_UID
128+
valueFrom:
129+
fieldRef:
130+
fieldPath: metadata.uid
131+
envFrom:
132+
- configMapRef:
133+
name: metadata-grpc-configmap
134+
optional: true
135+
image: gcr.io/ml-pipeline/should-be-overridden-during-runtime
136+
name: ""
137+
resources: {}
138+
volumeMounts:
139+
- mountPath: /kfp-launcher
140+
name: kfp-launcher
141+
- mountPath: /gcs
142+
name: gcs-scratch
143+
- mountPath: /s3
144+
name: s3-scratch
145+
- mountPath: /minio
146+
name: minio-scratch
147+
- mountPath: /.local
148+
name: dot-local-scratch
149+
- mountPath: /.cache
150+
name: dot-cache-scratch
151+
- mountPath: /.config
152+
name: dot-config-scratch
158153
initContainers:
159-
- command:
160-
- launcher-v2
154+
- args:
161155
- --copy
162156
- /kfp-launcher/launch
163-
image: gcr.io/ml-pipeline/kfp-launcher
157+
command:
158+
- launcher-v2
159+
image: ghcr.io/kubeflow/kfp-launcher
164160
name: kfp-launcher
165161
resources:
166162
limits:
@@ -177,7 +173,7 @@ spec:
177173
metadata: {}
178174
name: hello-world-system-container-impl
179175
outputs: {}
180-
podSpecPatch: "{{inputs.parameters.pod-spec-patch}}"
176+
podSpecPatch: '{{inputs.parameters.pod-spec-patch}}'
181177
retryStrategy:
182178
backoff:
183179
duration: "0"
@@ -187,27 +183,39 @@ spec:
187183
volumes:
188184
- emptyDir: {}
189185
name: kfp-launcher
186+
- emptyDir: {}
187+
name: gcs-scratch
188+
- emptyDir: {}
189+
name: s3-scratch
190+
- emptyDir: {}
191+
name: minio-scratch
192+
- emptyDir: {}
193+
name: dot-local-scratch
194+
- emptyDir: {}
195+
name: dot-cache-scratch
196+
- emptyDir: {}
197+
name: dot-config-scratch
190198
- dag:
191199
tasks:
192200
- arguments:
193201
parameters:
194202
- name: component
195-
value: "{{workflow.annotations.pipelines.kubeflow.org/components-comp-hello-world}}"
203+
value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}'
196204
- name: task
197205
value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"retryPolicy":{"backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"hello-world"}}'
198206
- name: container
199-
value: "{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-hello-world}}"
207+
value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}'
200208
- name: parent-dag-id
201-
value: "{{inputs.parameters.parent-dag-id}}"
209+
value: '{{inputs.parameters.parent-dag-id}}'
202210
name: hello-world-driver
203211
template: system-container-driver
204212
- arguments:
205213
parameters:
206214
- name: pod-spec-patch
207-
value: "{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}"
215+
value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}'
208216
- default: "false"
209217
name: cached-decision
210-
value: "{{tasks.hello-world-driver.outputs.parameters.cached-decision}}"
218+
value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}'
211219
depends: hello-world-driver.Succeeded
212220
name: hello-world
213221
template: hello-world-system-container-executor
@@ -220,38 +228,40 @@ spec:
220228
- container:
221229
args:
222230
- --type
223-
- "{{inputs.parameters.driver-type}}"
231+
- '{{inputs.parameters.driver-type}}'
224232
- --pipeline_name
225233
- namespace/n1/pipeline/hello-world
226234
- --run_id
227-
- "{{workflow.uid}}"
235+
- '{{workflow.uid}}'
236+
- --run_name
237+
- '{{workflow.name}}'
238+
- --run_display_name
239+
- ""
228240
- --dag_execution_id
229-
- "{{inputs.parameters.parent-dag-id}}"
241+
- '{{inputs.parameters.parent-dag-id}}'
230242
- --component
231-
- "{{inputs.parameters.component}}"
243+
- '{{inputs.parameters.component}}'
232244
- --task
233-
- "{{inputs.parameters.task}}"
245+
- '{{inputs.parameters.task}}'
234246
- --runtime_config
235-
- "{{inputs.parameters.runtime-config}}"
247+
- '{{inputs.parameters.runtime-config}}'
236248
- --iteration_index
237-
- "{{inputs.parameters.iteration-index}}"
249+
- '{{inputs.parameters.iteration-index}}'
238250
- --execution_id_path
239-
- "{{outputs.parameters.execution-id.path}}"
251+
- '{{outputs.parameters.execution-id.path}}'
240252
- --iteration_count_path
241-
- "{{outputs.parameters.iteration-count.path}}"
253+
- '{{outputs.parameters.iteration-count.path}}'
242254
- --condition_path
243-
- "{{outputs.parameters.condition.path}}"
244-
- --run_metadata
245-
- "{{inputs.parameters.run-metadata}}"
255+
- '{{outputs.parameters.condition.path}}'
246256
- --http_proxy
247-
- ''
257+
- ""
248258
- --https_proxy
249-
- ''
259+
- ""
250260
- --no_proxy
251-
- ''
261+
- ""
252262
command:
253263
- driver
254-
image: gcr.io/ml-pipeline/kfp-driver
264+
image: ghcr.io/kubeflow/kfp-driver
255265
name: ""
256266
resources:
257267
limits:
@@ -273,8 +283,6 @@ spec:
273283
name: iteration-index
274284
- default: DAG
275285
name: driver-type
276-
- default: ""
277-
name: run-metadata
278286
metadata: {}
279287
name: system-dag-driver
280288
outputs:
@@ -295,7 +303,7 @@ spec:
295303
- arguments:
296304
parameters:
297305
- name: component
298-
value: "{{workflow.annotations.pipelines.kubeflow.org/components-root}}"
306+
value: '{{workflow.parameters.components-root}}'
299307
- name: runtime-config
300308
value: '{"parameters":{"text":{"stringValue":"hi there"}}}'
301309
- name: driver-type
@@ -305,7 +313,7 @@ spec:
305313
- arguments:
306314
parameters:
307315
- name: parent-dag-id
308-
value: "{{tasks.root-driver.outputs.parameters.execution-id}}"
316+
value: '{{tasks.root-driver.outputs.parameters.execution-id}}'
309317
- name: condition
310318
value: ""
311319
depends: root-driver.Succeeded

0 commit comments

Comments
 (0)