Metamist ETL integration example. (#694)

* Metamist ETL integration example.

* Added audience_list to metamist ETL driver.

* Updating etl_caller_example.py.

* Added metamist-etl-custom-audience-list to read from config.

* Change as per PR review.

metamist_infrastructure/driver.py

@@ -550,7 +550,10 @@ def _setup_etl_pubsub(self):
     @cached_property
     def etl_extract_function(self):
         """etl_extract_function"""
-        return self._etl_function('extract', self.etl_extract_service_account)
+        return self._etl_function(
+            'extract',
+            self.etl_extract_service_account,
+        )
 
     @cached_property
     def etl_load_function(self):
@@ -646,6 +649,16 @@ def _etl_function(
             opts=pulumi.ResourceOptions(replace_on_changes=['*']),
         )
 
+        # prepare custom audience_list
+        custom_audience_list = None
+        if (
+            self.config.metamist.etl.custom_audience_list
+            and self.config.metamist.etl.custom_audience_list.get(f_name)
+        ):
+            custom_audience_list = json.dumps(
+                self.config.metamist.etl.custom_audience_list.get(f_name)
+            )
+
         fxn = gcp.cloudfunctionsv2.Function(
             f'metamist-etl-{f_name}',
             name=f'metamist-etl-{f_name}',
@@ -694,6 +707,11 @@ def _etl_function(
                     'SM_ENVIRONMENT': self.config.metamist.etl.environment,
                     'CONFIGURATION_SECRET': self.etl_configuration_secret_version.id,
                 },  # type: ignore
+                annotations=(
+                    {'run.googleapis.com/custom-audiences': custom_audience_list}
+                    if custom_audience_list
+                    else None
+                ),
                 ingress_settings='ALLOW_ALL',
                 all_traffic_on_latest_revision=True,
                 service_account_email=sa.email,

scripts/etl_caller_example.py

@@ -6,6 +6,7 @@
 
     pip install requests google-auth requests urllib3
 """
+
 import os
 
 import google.auth.transport.requests
@@ -14,7 +15,7 @@
 from requests.adapters import HTTPAdapter
 from urllib3 import Retry
 
-URL = 'https://metamist-etl-mnrpw3mdza-ts.a.run.app'
+URL = 'https://metamist-extract.popgen.rocks'
 TYPE = 'NAME_OF_EXTERNAL_PARTY/v1'
 
 
@@ -60,4 +61,10 @@ def make_request(body: dict | list):
 
 
 if __name__ == '__main__':
+    # simple payload which would be sent to the server
     print(make_request({'test-value': 'test'}))
+
+    # This will raise a 400 with an error message 'Potentially detected PII, the following keys were found in the body'
+    # it won't update anything in the database
+    payload = {'first_name': 'This is a test to demonstrate PII detection'}
+    make_request(payload)