Skip to content

Commit dcd96c4

Browse files
shapateriyashapateriya
committed
Addressed the core comments
1 parent f749961 commit dcd96c4

3 files changed

Lines changed: 48 additions & 80 deletions

File tree

‎statvar_imports/crdc/instructional_wifi_devices/common_pvmap.csv‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
key,,,,,,,,
22
COMBOKEY,#Format,observationAbout=nces/{Data:0>12},statType,measuredValue,,,,
33
YEAR,observationDate,{Data},,,,,,
4-
SCH_INTERNET_FIBER,#ignore,ignore,value,{Number},,,,
5-
SCH_INTERNET_WIFI,#ignore,ignore,value,{Number},,,,
6-
SCH_INTERNET_SCHDEV,#ignore,ignore,value,{Number},,,,
7-
SCH_INTERNET_STUDDEV,#ignore,ignore,value,{Number},,,,
4+
SCH_INTERNET_FIBER,#ignore,ignore
5+
SCH_INTERNET_WIFI,#ignore,ignore
6+
SCH_INTERNET_SCHDEV,#ignore,ignore
7+
SCH_INTERNET_STUDDEV,#ignore,ignore
88
SCH_INTERNET_WIFIENDEV,populationType,School,measuredProperty,count,internetSubscriptionType,WifiEnabledDevice,value,{Number}
99
,,,,,,,,
1010
JJ:Yes,educationalInstitutionType,JuvenileJusticeFacility,,,,,,

‎statvar_imports/crdc/instructional_wifi_devices/download.py‎

Lines changed: 43 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -7,37 +7,17 @@
77
from pathlib import Path
88
import time
99
import datetime
10-
import logging # Import the logging module
10+
import logging
1111
import sys
12-
# --- NEW: Simplified Logging Configuration ---
13-
# Configure logging to output ONLY to the console (StreamHandler)
14-
# and set the log format.
15-
logging.basicConfig(
16-
level=logging.INFO,
17-
format='%(asctime)s - %(levelname)s - %(message)s',
18-
handlers=[
19-
logging.StreamHandler()
20-
]
21-
)
2212

23-
24-
OUTPUT_DIR = Path("input_files")
25-
26-
# Create the output directory if it doesn't exist
27-
try:
28-
OUTPUT_DIR.mkdir(exist_ok=True)
29-
logging.info(f"Output directory created/verified: {OUTPUT_DIR.resolve()}")
30-
except Exception as e:
31-
logging.fatal(f"FATAL: Could not create output directory {OUTPUT_DIR.resolve()}. Exiting. Error: {e}")
32-
sys.exit(1)
33-
34-
35-
# --- Configuration ---
13+
# --- 1. Constants (Configuration Only) ---
14+
# It is okay to keep static configuration at the top level.
3615
BASE_URL = "https://civilrightsdata.ed.gov/assets/ocr/docs/"
37-
# The file names we are looking for inside the zip files
3816
TARGET_CSV_NAME = "Internet Access and Devices"
17+
OUTPUT_DIR = Path("input_files")
18+
19+
# --- 2. Function Definitions ---
3920

40-
# --- Dynamic Year String Generation ---
4121
def generate_year_strings(start_year=2020):
4222
"""
4323
Generates academic year strings (e.g., '2020-21') from the start_year
@@ -47,110 +27,99 @@ def generate_year_strings(start_year=2020):
4727
current_year = current_date.year
4828
current_month = current_date.month
4929

50-
# Determine the start year of the most recent academic cycle.
5130
if current_month >= 8:
5231
max_start_year = current_year
5332
else:
5433
max_start_year = current_year - 1
5534

5635
year_strings = []
57-
# Loop from the earliest year to the latest possible start year
5836
for y in range(start_year, max_start_year + 1):
5937
end_year_two_digits = str(y + 1)[2:]
6038
year_string = f"{y}-{end_year_two_digits}"
6139
year_strings.append(year_string)
6240

63-
# Reverse the list so the script attempts to download the newest data first.
6441
return list(reversed(year_strings))
6542

66-
YEAR_STRINGS = generate_year_strings()
67-
6843
def get_full_year(year_string):
69-
"""
70-
Derives the full year (e.g., 2024 from '2023-24') for the YEAR column.
71-
"""
44+
"""Derives the full year (e.g., 2024 from '2023-24') for the YEAR column."""
7245
try:
73-
# Extract the second two-digit part ('24', '22', or '21')
7446
two_digit_year = year_string.split('-')[1]
7547
return int(f"20{two_digit_year}")
7648
except IndexError:
7749
logging.error(f"Could not parse year from string '{year_string}'. Using 0.")
7850
return 0
7951

8052
def process_crdc_data(year_string):
81-
"""
82-
Downloads, extracts, finds, processes, and saves the target CSV for a given year.
83-
"""
53+
"""Downloads, extracts, finds, processes, and saves the target CSV."""
8454
zip_filename = f"{year_string}-crdc-data.zip"
8555
full_url = f"{BASE_URL}{zip_filename}"
8656
target_year = get_full_year(year_string)
8757

8858
logging.info(f"\n--- Starting processing for academic year {year_string} (Column YEAR={target_year}) ---")
89-
logging.info(f"Attempting to download from: {full_url}")
9059

91-
# 1. Download the ZIP file
9260
try:
9361
response = requests.get(full_url, stream=True)
94-
response.raise_for_status() # Raise an exception for bad status codes
95-
logging.info(f"Successfully downloaded ZIP file content for {year_string}.")
62+
response.raise_for_status()
9663
except requests.exceptions.RequestException as e:
97-
# 404 errors or other connection issues are handled here
98-
logging.error(f"Error downloading {zip_filename}. File might not be released yet or connection failed: {e}")
64+
logging.error(f"Error downloading {zip_filename}: {e}")
9965
return
10066

101-
# Use a temporary directory for extraction
10267
with tempfile.TemporaryDirectory() as temp_dir:
10368
temp_path = Path(temp_dir)
104-
logging.info(f"Extracting contents to temporary directory: {temp_dir}")
105-
106-
# Unzip the file from memory
10769
try:
108-
# We use io.BytesIO to treat the response content as a file in memory
10970
with zipfile.ZipFile(io.BytesIO(response.content)) as zf:
110-
# Extract all contents into the temporary directory
11171
zf.extractall(temp_path)
112-
logging.info("Extraction complete.")
11372
except zipfile.BadZipFile:
11473
logging.error(f"The downloaded file for {year_string} is not a valid ZIP file.")
11574
return
11675

117-
# 2. Recursively search for the target CSV file
11876
found_csv_path = None
119-
# Use Path.rglob() for recursive search
12077
for file_path in temp_path.rglob('*.csv'):
121-
# Check if the file name contains the target phrase (case-insensitive)
12278
if TARGET_CSV_NAME.lower() in file_path.stem.lower():
12379
found_csv_path = file_path
12480
break
12581

12682
if not found_csv_path:
127-
logging.error(f"Could not find a CSV file matching '{TARGET_CSV_NAME}' for year {year_string}.")
83+
logging.error(f"Could not find a CSV matching '{TARGET_CSV_NAME}' for year {year_string}.")
12884
return
12985

130-
logging.info(f"Found target CSV: {found_csv_path.name} in subdirectory {found_csv_path.parent.name}")
131-
132-
# 3. Load, process, and save the CSV
13386
try:
134-
# Use 'latin-1' encoding as CRDC files often contain special characters
13587
df = pd.read_csv(found_csv_path, encoding='latin-1', low_memory=False)
136-
logging.info(f"Loaded DataFrame with {len(df)} rows and {len(df.columns)} columns.")
137-
138-
# 4. Append the YEAR column
13988
df['YEAR'] = target_year
140-
141-
# 5. Save the modified DataFrame to the output folder
14289
output_filename = OUTPUT_DIR / f"{TARGET_CSV_NAME.replace(' ', '_')}_{target_year}.csv"
14390
df.to_csv(output_filename, index=False, encoding='utf-8')
144-
145-
logging.info(f"Successfully processed and saved data to: {output_filename.resolve()}")
146-
91+
logging.info(f"Successfully saved data to: {output_filename.resolve()}")
14792
except Exception as e:
148-
# Catch exceptions during file operations or Pandas processing
14993
logging.error(f"An error occurred during CSV processing for {year_string}: {e}")
15094

151-
# --- Main execution loop ---
152-
for year in YEAR_STRINGS:
153-
process_crdc_data(year)
154-
# Be polite to the server by adding a small delay between large downloads
155-
time.sleep(2)
156-
logging.info("\nAll tasks complete.")
95+
# --- 3. Main Execution Block ---
96+
97+
def main():
98+
# A. Configure Logging (Moved inside main)
99+
logging.basicConfig(
100+
level=logging.INFO,
101+
format='%(asctime)s - %(levelname)s - %(message)s',
102+
handlers=[logging.StreamHandler()]
103+
)
104+
105+
# B. Directory Creation Logic (Moved inside main)
106+
try:
107+
OUTPUT_DIR.mkdir(exist_ok=True)
108+
logging.info(f"Output directory verified: {OUTPUT_DIR.resolve()}")
109+
except Exception as e:
110+
logging.fatal(f"FATAL: Could not create output directory {OUTPUT_DIR.resolve()}. Error: {e}")
111+
sys.exit(1)
112+
113+
# C. Dynamic Data Generation (Moved inside main)
114+
year_strings = generate_year_strings()
115+
116+
# D. Processing Loop
117+
for year in year_strings:
118+
process_crdc_data(year)
119+
# Small delay to be polite to the server
120+
time.sleep(2)
121+
122+
logging.info("\nAll tasks complete.")
123+
124+
if __name__ == "__main__":
125+
main()

‎statvar_imports/crdc/instructional_wifi_devices/manifest.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@
88
"provenance_url": "https://civilrightsdata.ed.gov/data",
99
"provenance_description": "The Civil Rights Data Collection (CRDC) has released data concerning the total number of devices (computers, laptops, tablets, etc.) that can connect via Wi-Fi and are used by students for instructional purposes",
1010
"scripts": [
11-
"download.py",
12-
"../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=common_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --config_file=common_metadata.csv --output_path=output/instructional_wifi_devices"
11+
"download.py", "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=common_pvmap.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --config_file=common_metadata.csv --output_path=output/instructional_wifi_devices"
1312
],
1413
"import_inputs": [
1514
{

0 commit comments

Comments
 (0)