adding small enhancement for handling exceptions on HttpHook

adding env variables requirement for future use
hubmapconsortium · Feb 7, 2023 · 96e3be7 · 96e3be7
1 parent 1ddcc17
commit 96e3be7
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 0 deletions.
diff --git a/src/ingest-pipeline/airflow/dags/trigger_file_reindex.py b/src/ingest-pipeline/airflow/dags/trigger_file_reindex.py
@@ -7,6 +7,7 @@
 from airflow.operators.python import PythonOperator
 from airflow.providers.http.hooks.http import HttpHook
 from airflow.configuration import conf as airflow_conf
+from airflow.exceptions import AirflowException
 
 import utils
 from utils import (
@@ -60,6 +61,10 @@ def launch_file_reindex(**kwargs):
             response.raise_for_status()
         except http.client.HTTPException as he:
             print(f'Error {he}')
+        except AirflowException as ae:
+            print(f'Error {ae}')
+        except Exception as e:
+            print(f'Broad Exception {e}')
 
     t_launch_file_reindex = PythonOperator(
         task_id='launch_file_reindex',

diff --git a/src/ingest-pipeline/airflow/plugins/hubmap_api/endpoint.py b/src/ingest-pipeline/airflow/plugins/hubmap_api/endpoint.py
@@ -35,6 +35,7 @@
 NEEDED_ENV_VARS = [
     'AIRFLOW_CONN_INGEST_API_CONNECTION',
     'AIRFLOW_CONN_UUID_API_CONNECTION',
+    'AIRFLOW_CONN_FILES_API_CONNECTION',
     'AIRFLOW_CONN_SEARCH_API_CONNECTION',
     'AIRFLOW_CONN_ENTITY_API_CONNECTION'
     ]

diff --git a/src/ingest-pipeline/misc/tools/airflow_wrapper.sh b/src/ingest-pipeline/misc/tools/airflow_wrapper.sh
@@ -53,6 +53,7 @@ source source_platform_file.sh
 envvars=( CONFIG HOME \
 	  CONN_INGEST_API_CONNECTION \
 	  CONN_UUID_API_CONNECTION \
+	  CONN_FILES_API_CONNECTION \
 	  CONN_CELLS_API_CONNECTION \
 	  CONN_SEARCH_API_CONNECTION \
 	  CONN_ENTITY_API_CONNECTION \