diff --git a/src/ingest-pipeline/airflow/dags/bulk_update_entities.py b/src/ingest-pipeline/airflow/dags/bulk_update_entities.py index 17d0e787..ca952893 100644 --- a/src/ingest-pipeline/airflow/dags/bulk_update_entities.py +++ b/src/ingest-pipeline/airflow/dags/bulk_update_entities.py @@ -76,7 +76,7 @@ def check_uuids(**kwargs): print(f"{uuid} is neither a dataset nor an upload and will be skipped.") print(repr(e)) - kwargs["ti"].xcom_push(key="uuids", value=filtered_uuids) + kwargs["dag_run"].conf["uuids"] = filtered_uuids check_uuids_t = PythonOperator( task_id="check_uuids", @@ -101,18 +101,17 @@ def update_uuids(**kwargs): uuids = kwargs["dag_run"].conf["uuids"] metadata = kwargs["dag_run"].conf["metadata"] - error_dict = {} - for uuid in uuids: endpoint = f"entities/{uuid}" try: response = http_hook.run(endpoint, headers=headers, data=json.dumps(metadata)) print("response: ") pprint(response.json()) - time.sleep(10) except: print(f"ERROR: UUID {uuid} could not be updated.") + time.sleep(10) + update_uuids_t = PythonOperator( task_id="update_uuids", python_callable=update_uuids,