Merge branch 'datacommonsorg:master' into brazil_rural

Author: SudhishaK

import-automation/executor/Dockerfile

@@ -33,8 +33,8 @@ libasound2 \
 fonts-liberation \
 xdg-utils \
 chromium \
-chromium-driver
-
+chromium-driver \
+p7zip-full
 # Install the Google Cloud CLI
 RUN apt-get update && \
     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \

import-automation/executor/app/configs.py

@@ -122,7 +122,7 @@ class ExecutorConfig:
     # Cloud workflow id.
     cloud_workflow_id: str = 'import-automation-workflow'
     # Maximum time a user script can run for in seconds.
-    user_script_timeout: float = 36000
+    user_script_timeout: float = 86400
     # Arguments for the user script
     user_script_args: List[str] = ()
     # Environment variables for the user script
@@ -165,17 +165,7 @@ class ExecutorConfig:
 
     def get_data_refresh_config(self):
         """Returns the config used for Cloud Scheduler data refresh jobs."""
-        fields = set([
-            'github_repo_name', 'github_repo_owner_username',
-            'github_auth_username', 'github_auth_access_token',
-            'dashboard_oauth_client_id', 'importer_oauth_client_id',
-            'email_account', 'email_token', 'gcs_project_id',
-            'storage_prod_bucket_name', 'user_script_args', 'user_script_env',
-            'user_script_timeout'
-        ])
-        return {
-            k: v for k, v in dataclasses.asdict(self).items() if k in fields
-        }
+        return {k: v for k, v in dataclasses.asdict(self).items()}
 
 
 def _setup_logging():

import-automation/executor/app/executor/cloud_batch.py

@@ -138,7 +138,7 @@ def get_gce_instance(required_cpu: float,
 
 
 def create_job_request(import_name: str, import_config: dict, import_spec: dict,
-                       default_resources: dict) -> str:
+                       default_resources: dict, timeout: int) -> str:
     resources = import_spec.get('resource_limits', default_resources)
     machine_type = get_gce_instance(resources['cpu'], resources['memory'])
 
@@ -154,7 +154,8 @@ def create_job_request(import_name: str, import_config: dict, import_spec: dict,
         "jobName": job_name,
         "importName": import_name,
         "importConfig": import_config_string,
-        "resources": resources
+        "resources": resources,
+        "timeout": timeout
     }
     argument_string = json.dumps(argument_payload)
     final_payload = {