resolved core comments

Author: HarishC727

statvar_imports/us_urban_school/ap_ib_gt_enrollment/README.md

@@ -78,23 +78,18 @@ The scripts perform the following processing steps:
 
 ## Processing the downloaded data
 
-After downloading the data, you can process it by running the `run_process.sh` script in each of the data directories.
-For example, to process the Advanced Placement data, run the following command:
+To process the data, you can run the `run_process.sh` script in each of the data directories. This script first downloads the relevant data and then processes it.
 
-```bash
-bash statvar_imports/us_urban_school/ap_ib_gt_enrollment/advanced_placements/run_process.sh
-```
-
-You can also download and process the data in one step by using the `--download` flag:
+For example, to process the Advanced Placement data, change into the directory and then run the script:
 
 ```bash
-bash statvar_imports/us_urban_school/ap_ib_gt_enrollment/advanced_placements/run_process.sh --download
+cd statvar_imports/us_urban_school/ap_ib_gt_enrollment/advanced_placements/
+bash run_process.sh
 ```
 
 The processing script will:
 
-1.  Create an `output_files` directory.
-
-2.  Process the downloaded data for each year.
-
-3.  Generate statistical variables using the `stat_var_processor.py` script.
+1.  **Download the data** using `download_ap_ib_gt.py` and `download_2015_16.py`.
+2.  Create an `output_files` directory.
+3.  Process the downloaded data for each year.
+4.  Generate statistical variables using the `stat_var_processor.py` script.

statvar_imports/us_urban_school/ap_ib_gt_enrollment/config/ap_enrollment_pvmap.csv

@@ -1,9 +1,6 @@
 key,,,,,,,,,,
 YEAR,observationDate,{Number},populationType,Student,assessmentType,AdvancedPlacement,enrollmentStatus,EnrolledInEducationOrTraining,,
 ncesid,observationAbout,ncesId/{Data},,,,,,,,
-SCH_APENR_IND,,,,,,,,,,
-SCH_APCOURSES,,,,,,,,,,
-SCH_APSEL,,,,,,,,,,
 SCH_APENR_HI_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,HispanicOrLatino,,
 SCH_APENR_HI_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,race,HispanicOrLatino,,
 SCH_APENR_AM_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,AmericanIndianOrAlaskaNative,,
@@ -24,7 +21,6 @@ SCH_APENR_LEP_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,languag
 SCH_APENR_LEP_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,languageFluencyLevel,LimitedEnglishProficient,,
 SCH_APENR_IDEA_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,disabilityStatus,WithDisability,,
 SCH_APENR_IDEA_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,disabilityStatus,WithDisability,,
-SCH_APMATHENR_IND,,,,,,,,,,
 SCH_APMATHENR_HI_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,HispanicOrLatino,schoolSubject,Mathematics
 SCH_APMATHENR_HI_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,race,HispanicOrLatino,schoolSubject,Mathematics
 SCH_APMATHENR_AM_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,AmericanIndianOrAlaskaNative,schoolSubject,Mathematics
@@ -45,7 +41,6 @@ SCH_APMATHENR_LEP_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,lan
 SCH_APMATHENR_LEP_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,languageFluencyLevel,LimitedEnglishProficient,schoolSubject,Mathematics
 SCH_APMATHENR_IDEA_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,disabilityStatus,WithDisability,schoolSubject,Mathematics
 SCH_APMATHENR_IDEA_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,disabilityStatus,WithDisability,schoolSubject,Mathematics
-SCH_APSCIENR_IND,,,,,,,,,,
 SCH_APSCIENR_HI_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,HispanicOrLatino,schoolSubject,Science
 SCH_APSCIENR_HI_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,race,HispanicOrLatino,schoolSubject,Science
 SCH_APSCIENR_AM_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,AmericanIndianOrAlaskaNative,schoolSubject,Science
@@ -66,7 +61,6 @@ SCH_APSCIENR_LEP_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,lang
 SCH_APSCIENR_LEP_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,languageFluencyLevel,LimitedEnglishProficient,schoolSubject,Science
 SCH_APSCIENR_IDEA_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,disabilityStatus,WithDisability,schoolSubject,Science
 SCH_APSCIENR_IDEA_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,disabilityStatus,WithDisability,schoolSubject,Science
-SCH_APCOMPENR_IND,,,,,,,,,,
 SCH_APCOMPENR_HI_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,HispanicOrLatino,schoolSubject,Computer
 SCH_APCOMPENR_HI_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,race,HispanicOrLatino,schoolSubject,Computer
 SCH_APCOMPENR_AM_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,AmericanIndianOrAlaskaNative,schoolSubject,Computer
@@ -87,7 +81,6 @@ SCH_APCOMPENR_LEP_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,lan
 SCH_APCOMPENR_LEP_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,languageFluencyLevel,LimitedEnglishProficient,schoolSubject,Computer
 SCH_APCOMPENR_IDEA_M,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,disabilityStatus,WithDisability,schoolSubject,Computer
 SCH_APCOMPENR_IDEA_F,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,disabilityStatus,WithDisability,schoolSubject,Computer
-,,,,,,,,,,
 M_AME_7_MATH_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,AmericanIndianOrAlaskaNative,schoolSubject,Mathematics
 M_ASI_7_MATH_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,Asian,schoolSubject,Mathematics
 M_HIS_7_MATH_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,HispanicOrLatino,schoolSubject,Mathematics
@@ -108,7 +101,6 @@ F_2_OR_MORE_7_MATH_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female
 F_TOT_7_MATH_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,,,schoolSubject,Mathematics
 F_DIS_7_MATH_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,disabilityStatus,WithDisability,schoolSubject,Mathematics
 F_LEP_7_MATH_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,languageFluencyLevel,LimitedEnglishProficient,schoolSubject,Mathematics
-,,,,,,,,,,
 M_AME_7_SCIENCE_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,AmericanIndianOrAlaskaNative,schoolSubject,Science
 M_ASI_7_SCIENCE_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,Asian,schoolSubject,Science
 M_HIS_7_SCIENCE_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,HispanicOrLatino,schoolSubject,Science
@@ -129,7 +121,6 @@ F_2_OR_MORE_7_SCIENCE_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Fem
 F_TOT_7_SCIENCE_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,,,schoolSubject,Science
 F_DIS_7_SCIENCE_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,disabilityStatus,WithDisability,schoolSubject,Science
 F_LEP_7_SCIENCE_AP,value,{Number},#Filter,Allow=int(value) >= 0,gender,Female,languageFluencyLevel,LimitedEnglishProficient,schoolSubject,Science
-,,,,,,,,,,
 M_AME_7_MATH_ENR,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,AmericanIndianOrAlaskaNative,schoolSubject,Mathematics
 M_ASI_7_MATH_ENR,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,Asian,schoolSubject,Mathematics
 M_HIS_7_MATH_ENR,value,{Number},#Filter,Allow=int(value) >= 0,gender,Male,race,HispanicOrLatino,schoolSubject,Mathematics

statvar_imports/us_urban_school/ap_ib_gt_enrollment/download_2015_16.py

@@ -27,9 +27,6 @@
 
 FLAGS = flags.FLAGS
 
-
-
-
 def download_and_extract_2015_16(data_type, output_dir):
     """
     Downloads the ZIP archive for 2015-16 and extracts the target file.

statvar_imports/us_urban_school/ap_ib_gt_enrollment/download_ap_ib_gt.py

@@ -96,44 +96,26 @@
 
 }
 
-
-
 HARDCODED_CONFIGS = {
-
     "2021-22": {"final_year": 2022},
-
     "2020-21": {"final_year": 2021},
-
     "2017-18": {"final_year": 2018},
-
     "2015-16": {"final_year": 2016},
-
     "2013-14": {"final_year": 2014},
-
     "2011-12": {"final_year": 2012},
-
     "2009-10": {"final_year": 2010}
 
 }
 
-
-
 SCRIPT_DIR = Path(os.path.dirname(os.path.abspath(__file__)))
 
-
-
 def add_year_column(file_path, year):
 
     """
-
     Reads the file (CSV or XLSX), adds the 'YEAR' and 'ncesid' columns,
-
     and saves it back, using 'latin1' encoding for robustness.
-
     If the file is empty or contains no data rows, it is deleted.
-
     """
-
     logging.info(f"-> Starting post-processing for {file_path.name}")
 
     df = pd.DataFrame()
@@ -164,8 +146,6 @@ def add_year_column(file_path, year):
 
             return
 
-
-
         if df.empty:
 
             logging.warning(f"Warning: DataFrame is empty after reading {file_path.name}. Deleting empty file.")
@@ -174,14 +154,10 @@ def add_year_column(file_path, year):
 
             return
 
-
-
         # Add YEAR column
 
         df['YEAR'] = year
 
-
-
         # Create NCESID column
 
         if all(col in df.columns for col in ["LEAID", "SCHID"]):
@@ -196,8 +172,6 @@ def add_year_column(file_path, year):
 
             logging.warning(f"-> Warning: Missing LEAID or SCHID columns for NCESID creation in {file_path.name}")
 
-
-
         # Reorder columns to place 'YEAR', 'ncesid', and 'JJ' (if present) at the beginning
 
         cols = df.columns.tolist()
@@ -208,26 +182,18 @@ def add_year_column(file_path, year):
 
             desired_first_cols.append('JJ')
 
-
-
         # Remove desired_first_cols from their original positions
 
         for col in desired_first_cols:
-
             if col in cols:
-
                 cols.remove(col)
 
-
-
         # Construct the new column order
 
         new_column_order = desired_first_cols + cols
 
         df = df[new_column_order]
 
-
-
         if file_path.suffix == '.xlsx':
 
             df.to_excel(file_path, index=False)
@@ -236,12 +202,8 @@ def add_year_column(file_path, year):
 
             df.to_csv(file_path, index=False)
 
-            
-
         logging.info(f"-> Successfully added 'YEAR' and 'ncesid' columns to {file_path.name}")
 
-
-
     except Exception as e:
 
         # If an error occurs during processing but after the file was created, attempt to delete it.
@@ -254,88 +216,41 @@ def add_year_column(file_path, year):
 
         raise RuntimeError(f"Failed to add year/ncesid column to {file_path.name}: {e}")
 
-
-
 def generate_future_configs(start_year):
 
     """
-
     Generates configuration dictionaries for future biennial CRDC years.
-
     """
-
     current_calendar_year = date.today().year
 
     generated_configs = {}
 
-    
-
     for year in range(start_year, current_calendar_year + 1):
-
         end_year = year + 1
-
         year_range_key = f"{year}-{end_year % 100:02d}"
 
-        
-
         if year_range_key in HARDCODED_CONFIGS:
-
             continue
 
-
-
         generated_configs[year_range_key] = {
 
             "final_year": end_year
-
         }
 
-        
-
     return generated_configs
 
-
-
 @retry(tries=3, delay=5, backoff=2)
 
-
-
 def download_url_with_retry(zip_url):
-
-
-
     """
-
-
-
     Handles downloading the ZIP content with retries and status checks.
-
-
-
     """
-
-
-
     head_response = requests.head(zip_url, allow_redirects=True, timeout=10)
-
-
-
     head_response.raise_for_status()
-
-
-
     response = requests.get(zip_url, stream=True, timeout=180)
-
-
-
     response.raise_for_status()
-
-
-
     return response
 
-
-
 def get_file_keywords(data_type, config_key):
 
     """
@@ -360,44 +275,26 @@ def get_file_keywords(data_type, config_key):
 
     return keywords, specific_constraint, output_name_fragment, pvmap_config
 
-
-
-
-
 def get_req_cols_from_config(config_path):
 
     """
-
     Reads a CSV config file to extract a list of required column names.
-
     It takes the first column, filters out specific values ('YEAR', 'ncesid', ''),
-
     and adds 'LEAID' and 'SCHID'.
-
     """
 
     try:
 
         df = pd.read_csv(config_path, header=None, usecols=[0], on_bad_lines='skip')
-
         # Get first column as a list
-
         req_cols = df[0].tolist()
-
         # Remove header 'key'
-
         if 'key' in req_cols:
-
             req_cols.remove('key')
-
         # Filter out values
-
         req_cols = [col for col in req_cols if col not in ['YEAR', 'ncesid', ''] and pd.notna(col)]
-
         # Add necessary columns for ncesid creation
-
         req_cols.extend(['LEAID', 'SCHID'])
-
         return req_cols
 
     except Exception as e:
@@ -406,8 +303,6 @@ def get_req_cols_from_config(config_path):
 
         return None
 
-
-
 def download_and_extract(config_key, config_data, data_type, output_dir):
     """
     Downloads the ZIP archive and extracts the target file using keyword search,
@@ -525,10 +420,6 @@ def download_and_extract(config_key, config_data, data_type, output_dir):
     except Exception as e:
         raise RuntimeError(f"An unhandled error occurred during extraction/processing for {config_key}: {e}")
 
-
-
-
-
 def main(_):
     """
     Main function to run the download process for AP, IB, and GT data.
@@ -565,8 +456,6 @@ def main(_):
 
     logging.info(f"\n--- All downloads complete ---")
 
-
-
 if __name__ == '__main__':
 
     app.run(main)