Skip to content

Commit

Permalink
Merge pull request #195 from UrbanSystemsLab/josiahkp
Browse files Browse the repository at this point in the history 
Fix build_wrf_label_matrix_http to return a dict instead of a set
  • Loading branch information
@Katsutoshii
Katsutoshii authored Nov 14, 2024
2 parents 2b8c06a + c78f09f commit 4e1f3b2
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 21 deletions.
2 changes: 1 addition & 1 deletion usl_pipeline/cloud_functions/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,7 +444,7 @@ def build_wrf_label_matrix_http(request: flask.Request) -> flask.Response:
request_json["name"],
cloud_storage.LABEL_CHUNKS_BUCKET,
)
return flask.jsonify({"message", "Label matrix built."})
return flask.jsonify({"message": "Label matrix built."})


def _build_wrf_label_matrix(
Expand Down
26 changes: 6 additions & 20 deletions usl_pipeline/cloud_functions/notebooks/test_cloud_functions.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,31 +25,17 @@
"outputs": [],
"source": [
"main._build_wrf_label_matrix(\n",
" bucket_name='test-climateiq-atmospheric-simulation-output',\n",
" chunk_name='Test_NYC_Heat/Test_Config_Group/wrfout.d03.2010-06-25_00:00:00',\n",
" output_bucket='test-climateiq-atmoml-labels')"
" bucket_name=\"test-climateiq-atmospheric-simulation-output\",\n",
" chunk_name=\"Test_NYC_Heat/Test_Config_Group/wrfout.d03.2010-06-25_00:00:00\",\n",
" output_bucket=\"climateiq-study-area-label-chunks\"\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "all the input array dimensions except for the concatenation axis must match exactly, but along dimension 0, the array at index 0 has size 200 and the array at index 5 has size 1",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmain\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_feature_matrix\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mbucket_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtest-climateiq-atmospheric-simulation-input\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunk_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mTest_NYC_Heat/Test_Config_Group/met_em.d03.2010-06-25_00:00:00.nc\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_bucket\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtest-climateiq-atmoml-features\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\n\u001b[1;32m 5\u001b[0m \u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/climateiq-terraform/climateiq-cnn/usl_pipeline/cloud_functions/main.py:678\u001b[0m, in \u001b[0;36m_build_feature_matrix\u001b[0;34m(bucket_name, chunk_path, output_bucket)\u001b[0m\n\u001b[1;32m 676\u001b[0m \u001b[38;5;66;03m# Heat (WRF) - treat one WPS outout file as one chunk\u001b[39;00m\n\u001b[1;32m 677\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m re\u001b[38;5;241m.\u001b[39msearch(file_names\u001b[38;5;241m.\u001b[39mWPS_DOMAIN3_NC_REGEX, chunk_path):\n\u001b[0;32m--> 678\u001b[0m feature_matrices, metadata \u001b[38;5;241m=\u001b[39m \u001b[43m_build_wps_feature_matrices\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchunk\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 679\u001b[0m \u001b[38;5;66;03m# Write a separate file for each variable type\u001b[39;00m\n\u001b[1;32m 680\u001b[0m \u001b[38;5;66;03m# (spatial, spatiotemporal, lu_index).\u001b[39;00m\n\u001b[1;32m 681\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m var_type, feature_matrix \u001b[38;5;129;01min\u001b[39;00m feature_matrices\u001b[38;5;241m.\u001b[39mitems():\n",
"File \u001b[0;32m~/climateiq-terraform/climateiq-cnn/usl_pipeline/cloud_functions/main.py:1001\u001b[0m, in \u001b[0;36m_build_wps_feature_matrices\u001b[0;34m(fd)\u001b[0m\n\u001b[1;32m 995\u001b[0m feature \u001b[38;5;241m=\u001b[39m _process_wps_feature(\n\u001b[1;32m 996\u001b[0m feature\u001b[38;5;241m=\u001b[39mds\u001b[38;5;241m.\u001b[39mdata_vars[var\u001b[38;5;241m.\u001b[39mname],\n\u001b[1;32m 997\u001b[0m var_config\u001b[38;5;241m=\u001b[39mwps_data\u001b[38;5;241m.\u001b[39mVAR_CONFIGS[var],\n\u001b[1;32m 998\u001b[0m )\n\u001b[1;32m 999\u001b[0m features_components[var\u001b[38;5;241m.\u001b[39mvalue] \u001b[38;5;241m=\u001b[39m feature\n\u001b[0;32m-> 1001\u001b[0m matrices[var_type] \u001b[38;5;241m=\u001b[39m \u001b[43mnumpy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdstack\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures_components\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1003\u001b[0m \u001b[38;5;66;03m# Get snapshot time\u001b[39;00m\n\u001b[1;32m 1004\u001b[0m snapshot_time \u001b[38;5;241m=\u001b[39m ds\u001b[38;5;241m.\u001b[39mTimes\u001b[38;5;241m.\u001b[39mvalues[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mstr\u001b[39m)\n",
"File \u001b[0;32m~/miniconda3/envs/climateiq/lib/python3.11/site-packages/numpy/lib/shape_base.py:715\u001b[0m, in \u001b[0;36mdstack\u001b[0;34m(tup)\u001b[0m\n\u001b[1;32m 713\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(arrs, \u001b[38;5;28mlist\u001b[39m):\n\u001b[1;32m 714\u001b[0m arrs \u001b[38;5;241m=\u001b[39m [arrs]\n\u001b[0;32m--> 715\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_nx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconcatenate\u001b[49m\u001b[43m(\u001b[49m\u001b[43marrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m\n",
"\u001b[0;31mValueError\u001b[0m: all the input array dimensions except for the concatenation axis must match exactly, but along dimension 0, the array at index 0 has size 200 and the array at index 5 has size 1"
]
}
],
"outputs": [],
"source": [
"main._build_feature_matrix(\n",
" bucket_name='test-climateiq-atmospheric-simulation-input',\n",
Expand Down

0 comments on commit 4e1f3b2

Please sign in to comment.