Skip to content

Commit

Permalink
General: Fixes to address Sunset's comments
Browse files Browse the repository at this point in the history
  • Loading branch information
Juan Puerto committed May 7, 2024
1 parent 6a56f29 commit 55c1403
Showing 1 changed file with 3 additions and 4 deletions.
7 changes: 3 additions & 4 deletions src/ingest-pipeline/airflow/dags/bulk_update_entities.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def check_uuids(**kwargs):
print(f"{uuid} is neither a dataset nor an upload and will be skipped.")
print(repr(e))

kwargs["ti"].xcom_push(key="uuids", value=filtered_uuids)
kwargs["dag_run"].conf["uuids"] = filtered_uuids

check_uuids_t = PythonOperator(
task_id="check_uuids",
Expand All @@ -101,18 +101,17 @@ def update_uuids(**kwargs):
uuids = kwargs["dag_run"].conf["uuids"]
metadata = kwargs["dag_run"].conf["metadata"]

error_dict = {}

for uuid in uuids:
endpoint = f"entities/{uuid}"
try:
response = http_hook.run(endpoint, headers=headers, data=json.dumps(metadata))
print("response: ")
pprint(response.json())
time.sleep(10)
except:
print(f"ERROR: UUID {uuid} could not be updated.")

time.sleep(10)

update_uuids_t = PythonOperator(
task_id="update_uuids",
python_callable=update_uuids,
Expand Down

0 comments on commit 55c1403

Please sign in to comment.