Merge branch 'master' into pep_monthly_cloud_job_fix

Author: dhhhanushree

scripts/us_bts/latch/manifest.json

@@ -36,10 +36,10 @@
             ],
             "cron_schedule": "0 1 6 * *",
             "resource_limits": {
-                "cpu": 8,
-                "memory": 64,
-                "disk": 100
+                "cpu": 16,
+                "memory": 128,
+                "disk": 300
             }
         }
     ]
-}
+}

scripts/us_census/pep/us_pep_sex/process.py

@@ -1092,7 +1092,11 @@ def process(self):
                 value_vars=['Count_Person_Male', 'Count_Person_Female'],
                 var_name="SV",
                 value_name="Observation")
-
+            subset_cols = ['Year', 'geo_ID', 'Measurement_Method', 'SV']
+            # 2. Drop duplicates based on those columns, keeping the first occurrence
+            final_df.drop_duplicates(subset=subset_cols,
+                                     keep='first',
+                                     inplace=True)
             final_df.to_csv(self._cleaned_csv_file_path, index=False)
             sv_list = ['Count_Person_Female', 'Count_Person_Male']
             self._generate_mcf(sv_list)
@@ -1117,7 +1121,7 @@ def fetch_skip_urls_from_gcs(GCS_BUCKET_NAME: str,
 
 def is_valid_url(url):
     try:
-        response = requests.get(url, timeout=10)
+        response = requests.get(url, timeout=20)
         if response.status_code != 200:
             return False
         content_type = response.headers.get("Content-Type", "")