Ejscreen semi-automatic

scripts/us_epa/ejscreen/README.md

@@ -18,11 +18,23 @@ which are a small subset of the available EJSCREEN variables.
 
 ## Usage
 
+To add a new year of data, you must update the config.json file.
+
+1. Update Core Configuration
+First, add the new year to the YEARS array. Next, specify the exact column names for that year in the CSV_COLUMNS_BY_YEAR object. You also need to provide the filename for the uncompressed CSV in FILENAMES and the name of the compressed file in ZIP_FILENAMES. Finally, if the new year's data is located in a different web directory, update the URL_SUFFIX accordingly.
+
+2. Handle Column Renaming
+If the column names in the new year's file do not match the standardized schema (e.g., ID is used instead of ID_New), you must add the year to the RENAME_COLUMNS_YEARS array. Then, create a mapping in the RENAME_COLUMNS_BY_YEAR object, which instructs the script on how to rename the raw column headers to the standardized names. This step is crucial for maintaining data consistency across all years.
+
+
 To generate `ejscreen_airpollutants.csv` and `ejscreen.tmcf` run the following:  
 
-    `python3 ejscreen.py`
+#Downloading and Processing Data
+To perform "download and process", run the below command: python3 ejscreen.py Running this command generates input_fles and csv, mcf, tmcf files
+
+If you want to perform "only process", run the below command: python3 ejscreen.py --mode=process
 
-As of July, 2021 this includes data through the end of 2020.
+If you want to perform "only download", run the below command: python3 ejscreen.py --mode=download
 
 ### Unit Tests
 

scripts/us_epa/ejscreen/config.json

@@ -0,0 +1,100 @@
+{
+   "#comment": "To add a new year (e.g., 2025), extend the YEARS list and provide corresponding values in each of the subsequent sections (CSV_COLUMNS_BY_YEAR, ZIP_FILENAMES, FILENAMES, URL_SUFFIX, and RENAME_COLUMNS_YEARS if applicable).",
+  "YEARS": [ "2015", "2016", "2017", "2018", "2019", "2020", "2021", "2022", "2023", "2024"],
+  "NORM_CSV_COLUMNS": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+  "NORM_CSV_COLUMNS1": ["ID", "DSLPM", "OZONE", "PM25"],
+  "CSV_COLUMNS_BY_YEAR": {
+    "2015": ["FIPS", "dpm", "cancer", "resp", "o3", "pm"],
+    "2016": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+    "2017": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+    "2018": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+    "2019": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+    "2020": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+    "2021": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+    "2022": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+    "2023": ["ID", "DSLPM", "CANCER", "RESP", "OZONE", "PM25"],
+    "2024": ["ID", "DSLPM", "OZONE", "PM25"]
+  },
+  "ZIP_FILENAMES": {
+    "2015": "EJSCREEN_20150505.csv",
+    "2016": "EJSCREEN_V3_USPR_090216_CSV",
+    "2017": null,
+    "2018": "EJSCREEN_2018_USPR_csv",
+    "2019": "EJSCREEN_2019_USPR.csv",
+    "2020": "EJSCREEN_2020_USPR.csv",
+    "2021": "EJSCREEN_2021_USPR.csv",
+    "2022": "EJSCREEN_2022_with_AS_CNMI_GU_VI.csv",
+    "2023": "EJSCREEN_2023_BG_with_AS_CNMI_GU_VI.csv",
+    "2024": "EJScreen_2024_Tract_with_AS_CNMI_GU_VI.csv"
+  },
+  "FILENAMES": {
+    "2015": "EJSCREEN_20150505",
+    "2016": "EJSCREEN_Full_V3_USPR_TSDFupdate",
+    "2017": "EJSCREEN_2017_USPR_Public",
+    "2018": "EJSCREEN_Full_USPR_2018",
+    "2019": "EJSCREEN_2019_USPR",
+    "2020": "EJSCREEN_2020_USPR",
+    "2021": "EJSCREEN_2021_USPR",
+    "2022": "EJSCREEN_2022_Full_with_AS_CNMI_GU_VI",
+    "2023": "EJSCREEN_2023_BG_with_AS_CNMI_GU_VI",
+    "2024": "EJScreen_2024_Tract_with_AS_CNMI_GU_VI"
+  },
+  "TEMPLATE_MCF": [
+    {
+      "Node": "E:ejscreen_airpollutants->E0",
+      "typeOf": "dcs:StatVarObservation",
+      "variableMeasured": "dcs:Mean_Concentration_AirPollutant_DieselPM",
+      "observationDate": "C:ejscreen_airpollutants->year",
+      "observationAbout": "C:ejscreen_airpollutants->FIPS",
+      "observationPeriod": "dcs:P1Y",
+      "value": "C:ejscreen_airpollutants->DSLPM",
+      "unit": "dcs:MicrogramsPerCubicMeter"
+    },
+    {
+      "Node": "E:ejscreen_airpollutants->E1",
+      "typeOf": "dcs:StatVarObservation",
+      "variableMeasured": "dcs:AirPollutant_Cancer_Risk",
+      "observationDate": "C:ejscreen_airpollutants->year",
+      "observationAbout": "C:ejscreen_airpollutants->FIPS",
+      "observationPeriod": "dcs:P1Y",
+      "value": "C:ejscreen_airpollutants->CANCER",
+      "unit": "dcs:PerMillionPerson"
+    },
+    {
+      "Node": "E:ejscreen_airpollutants->E2",
+      "typeOf": "dcs:StatVarObservation",
+      "variableMeasured": "dcs:AirPollutant_Respiratory_Hazard",
+      "observationDate": "C:ejscreen_airpollutants->year",
+      "observationAbout": "C:ejscreen_airpollutants->FIPS",
+      "observationPeriod": "dcs:P1Y",
+      "value": "C:ejscreen_airpollutants->RESP"
+    },
+    {
+      "Node": "E:ejscreen_airpollutants->E3",
+      "typeOf": "dcs:StatVarObservation",
+      "variableMeasured": "dcs:Mean_Concentration_AirPollutant_Ozone",
+      "observationDate": "C:ejscreen_airpollutants->year",
+      "observationAbout": "C:ejscreen_airpollutants->FIPS",
+      "observationPeriod": "dcs:P1Y",
+      "value": "C:ejscreen_airpollutants->OZONE",
+      "unit": "dcs:PartsPerBillion"
+    },
+    {
+      "Node": "E:ejscreen_airpollutants->E4",
+      "typeOf": "dcs:StatVarObservation",
+      "variableMeasured": "dcs:Mean_Concentration_AirPollutant_PM2.5",
+      "observationDate": "C:ejscreen_airpollutants->year",
+      "observationAbout": "C:ejscreen_airpollutants->FIPS",
+      "observationPeriod": "dcs:P1Y",
+      "value": "C:ejscreen_airpollutants->PM25",
+      "unit": "dcs:MicrogramsPerCubicMeter"
+    }
+  ]
+  ,
+  "BASE_URL": "https://gaftp.epa.gov/EJSCREEN",
+  "URL_SUFFIX": {
+    "2023": "2.22_September_UseMe",
+    "2024": "2.32_August_UseMe"
+  },
+  "RENAME_COLUMNS_YEARS": ["2024"]
+}