diff --git a/EN4_postprocessing/merge_mean_surface_crps.py b/EN4_postprocessing/merge_mean_surface_crps.py index d158961..071bdbf 100644 --- a/EN4_postprocessing/merge_mean_surface_crps.py +++ b/EN4_postprocessing/merge_mean_surface_crps.py @@ -40,7 +40,7 @@ def merge_by_time(season="all"): with ProgressBar(): save_path = config.dn_out + "profiles/" ds_index.to_netcdf(save_path + "%03s_CRPS_merged.nc"%(season)) - print(f'File written to {config.dn_out+"%03s_CRPS_merged.nc"%(season)}') + print(f'File written to {save_path+"%03s_CRPS_merged.nc"%(season)}') def regional_means(): """ mean over each region """ @@ -49,8 +49,8 @@ def regional_means(): fn_dom_nemo = "%s%s"%(config.dn_dom, config.grid_nc) fn_cfg_nemo = config.fn_cfg_nemo fn_cfg_prof = config.fn_cfg_prof - fn_analysis_crps = "%s%03s_CRPS_merged.nc"%(config.dn_out, season) - fn_out = "%s%03s_mask_means_crps_daily.nc"%(config.dn_out, season) + fn_analysis_crps = "%s%03s_CRPS_merged.nc"%(config.dn_out+"profiles/", season) + fn_out = "%s%03s_mask_means_crps_daily.nc"%(config.dn_out+"profiles/", season) # get the CRPS data as a profile object crps = coast.Profile(config=fn_cfg_prof) diff --git a/EN4_postprocessing/plot_surface_crps.py b/EN4_postprocessing/plot_surface_crps.py index 84e996e..94016df 100644 --- a/EN4_postprocessing/plot_surface_crps.py +++ b/EN4_postprocessing/plot_surface_crps.py @@ -27,10 +27,9 @@ # on each axis of the plot -# Assuming loading two configs: co7 and the P0.0. HARD WIRING. NOT IDEAL -fn_list = [config.dn_out+"All_mask_means_crps_daily.nc", - config.dn_out.replace(config.case,'co7')+"All_mask_means_crps_daily_co7.nc"]# - +# Comparison case defined in config. +fn_list = [config.dn_out+"profiles/"+All_mask_means_crps_daily.nc", + config.comp_case["proc_data"]+"profiles/+"All_mask_means_crps_daily.nc"]# #%% General Plot Settings ## Specify the specific regions, their labels and order of appearance to plot. Region indexing to match EN4_postprocessing mean_season.py region_ind = [ 1, 7, 3, 2, 9, 5, 4, 6, 8] # Region indices (in analysis) to plot @@ -101,7 +100,7 @@ var_name = "profile_mean_{0}_crps".format(var_str.lower()) # profile_mean_temperature_crps # Filename for the output - fn_out = "FIGS/regional_{0}_{1}_{2}.pdf".format(var_name, config.case, 'co7') + fn_out = "FIGS/regional_{0}_{1}_{2}.pdf".format(var_name, config.case, config.comp_case["case"]) # Create plot and flatten axis array f,a = plt.subplots(n_r, n_c, figsize = figsize, sharey = sharey) diff --git a/EN4_postprocessing/spice_extract_season.sbatch b/EN4_processing/spice_extract_season.sbatch similarity index 100% rename from EN4_postprocessing/spice_extract_season.sbatch rename to EN4_processing/spice_extract_season.sbatch diff --git a/EN4_processing/spice_ana_MOD_METEST.sh b/EN4_processing/spice_map_profiles.sh similarity index 100% rename from EN4_processing/spice_ana_MOD_METEST.sh rename to EN4_processing/spice_map_profiles.sh diff --git a/EN4_processing/spice_regional_masking.slurm b/EN4_processing/spice_regional_masking.slurm new file mode 100644 index 0000000..aa683bf --- /dev/null +++ b/EN4_processing/spice_regional_masking.slurm @@ -0,0 +1,11 @@ +#!/bin/bash +#SBATCH --mem=40000 +#SBATCH -o LOGS/%A_%a.out +#SBATCH -e LOGS/%A_%a.err +#SBATCH --time=180 +#SBATCH --ntasks=1 +source config.sh +source activate $CONDA_ENV + +echo "python regional_masking.py " +python regional_masking.py $1 > LOGS/regional_masking.log diff --git a/PythonEnvCfg/spice_config.sh b/PythonEnvCfg/spice_config.sh index c2a53cf..912ef3b 100644 --- a/PythonEnvCfg/spice_config.sh +++ b/PythonEnvCfg/spice_config.sh @@ -8,7 +8,7 @@ export STARTYEAR=2013 #1980 # 2004 export ENDYEAR=2013 #1980 # 2014 ## Process monthly data. Required in iter_sub_METEST.sh -export MOD="P1.5c" # Model reference name +export MOD="P2" # Model reference name export GRID="domain_cfg_sf12.nc" # contains the grid information for NEMO # options?: "domain_cfg_MEs_01-003_opt_v1.nc" # "GEG_SF12.nc" # options?: "GEG_SF12.nc" CO7_EXACT_CFG_FILE.nc @@ -42,3 +42,10 @@ export DN_DOM="/data/users/o.lambertbrown/Datasets/models/CO9/u-cu674/" #fn_dat = "/scratch/fred/COMPARE_VN36_VN_4.0_TIDE_SSH/%s/DAILY/%s%02d*T.nc*"%(exper,startyear,month) export DN_DAT="/data/users/o.lambertbrown/Datasets/models/CO9/u-cu674/" export DN_OUT="/data/users/o.lambertbrown/Datasets/models/CO9/u-cu674/analysis/" + +# Directories for comparison with another model/experiment. These are used when plotting/postprocessing. +export COMP_MOD="Cray-Ex" +export COMP_DAT="/data/scratch/o.lambertbrown/u-do888/" +export COMP_GRID="domain_cfg_sf12.nc" +export COMP_OUT="/data/users/o.lambertbrown/Datasets/models/CO9/u-do888/analysis/" + diff --git a/README.md b/README.md index c9dd7a7..78eacbf 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ python map_profiles.py $1 $2 > LOGS/OUT_$1_$2.log ``` using arguments: $1 $2 corresponding to the above. -This outputs, in `DN_OUT/profiles/`, files like: +This outputs, in `DN_OUT/profiles/` (be sure to make a DN_OUT/profiles directory), files like: ``` extracted_profiles_200401.nc interpolated_profiles_200401.nc @@ -105,7 +105,6 @@ profile_errors_200401.nc surface_data_200401.nc mid_data_200401.nc bottom_data_200401.nc -mask_means_daily_200401.nc ``` @@ -122,12 +121,12 @@ cd ../EN4_processing rm LOGS/OUT* LOGS/*.err LOGS/*.out -#sbatch -J 201407 --time=2:00:00 lotus_ana_MOD_METEST.sh 2014 7 -#sbatch -J 201010 --time=2:00:00 lotus_ana_MOD_METEST.sh 2010 10 -#sbatch -J 201011 --time=2:00:00 lotus_ana_MOD_METEST.sh 2010 11 -sbatch -J 201109 --time=3:00:00 lotus_ana_MOD_METEST.sh 2011 9 -#sbatch -J 201110 --time=2:00:00 lotus_ana_MOD_METEST.sh 2011 10 -sbatch -J 200905 --time=3:00:00 lotus_ana_MOD_METEST.sh 2009 5 +#sbatch -J 201407 --time=2:00:00 lotus_map_profiles.sh.sh 2014 7 +#sbatch -J 201010 --time=2:00:00 lotus_map_profiles.sh.sh 2010 10 +#sbatch -J 201011 --time=2:00:00 lotus_map_profiles.sh.sh 2010 11 +sbatch -J 201109 --time=3:00:00 lotus_map_profiles.sh.sh 2011 9 +#sbatch -J 201110 --time=2:00:00 lotus_map_profiles.sh.sh 2011 10 +sbatch -J 200905 --time=3:00:00 lotus_map_profiles.sh.sh 2009 5 ``` 2. `PythonEnvCfg/_config.sh` must both be edited for machine choices, conda environment, paths etc. @@ -138,43 +137,42 @@ Merge seasons (DJF, MAM, JJA, SON) from multiple years into single files. Execute with: ``` -iter_merge_season.sh +iter_extract_season.sh ``` which is just a simple concatenating loop over each season. -Each month invokes a machine specific sbatch scripts (e.g `spice_merge_season.sbatch`) where the model and season are +Each month invokes a machine specific sbatch scripts (e.g `spice_extract_season.sbatch`) where the model and season are passed onto a generic script -`python merge_season.py $1 $2 #1=Model, 2=month` +`python extract_season.py $1 $2 #1=Model, 2=month` -Outputs are written to DN_OUT by season string, sss: +Outputs are written to DN_OUT/profiles by season string, sss: ``` sss_PRO_INDEX.nc ## merging interpolated_profiles_*.nc (model profiles on ref levels) sss_PRO_DIFF.nc ## merging profile_errors_*.nc (diff between model & obs on ref levels) +sss_PRO_OBS.nc ## merging interpolated_obs_*.nc (obs on ref levels) ``` ### Create Means -Then call `iter_mean_season.sh` to compute the spatial means over subregions within the NWS domain. +Then call `regional_masking.sh` to compute the spatial means over subregions within the NWS domain. This launches machine specific script -`sbatch ${MACHINE,,}_mean_season.sbatch $MOD $month` +`sbatch ${MOD}_regional_mask ${MACHINE,,}_regional_masking.slurm` that in turn launches a machine independent script: ``` -python mean_season.py $1 $2 > LOGS/mean_season_$1_$2.log # 1=Model, 2=month +python regional_masking.py $1 > LOGS/regional_masking.log # 1=Model, 2=month ``` -to compute averages in each of the defined regions: +This reads in sss_PRO_INDEX.nc and ss_PRO_DIFF.nc to compute averages in each of the defined regions: ``` region_names = [ 'N. North Sea','S. North Sea','Eng. Channel','Outer Shelf', 'Irish Sea', 'Kattegat', 'Nor. Trench', 'FSC', 'Off-shelf'] ``` -Creating output: +Outputs are written to DN_OUT/profiles as: ``` -DJF_mask_means_daily.nc -MAM_mask_means_daily.nc -JJA_mask_means_daily.nc -SON_mask_means_daily.nc +profiles_by_region_and_season.nc ### Model profiles +profile_bias_by_region_and_season.nc ### Difference between model and obs ``` ### CRPS values @@ -187,16 +185,16 @@ Execute: `. ./iter_surface_crps.sh` This deploy monthly processes on ${MACHINE} (currently only tested on JASMIN's lotus) ``` -sbatch "${MACHINE,,}"_surface_crps.sh $MOD $start $month $end $GRID" +sbatch "${MACHINE,,}"_surface_crps.sh $start $month" ``` which in turn launches the python script ``` -python surface_crps.py $1 $2 $3 $4 $5 +python surface_crps.py $1 $2 ``` following the appropriate header commands for the batch scheduler. -Output files take the form: `surface_crps_data_p0_201101_2012.nc` +Output files are saved to DN_OUT/profiles and take the form: `surface_crps_data_p0_201101_2012.nc` Next merge and compute regional averages. E.g. merge_mean_surface_crps.py in EN4_postprocessing. @@ -247,6 +245,11 @@ which submits the following machine independent script ``` python merge_mean_surface_crps.py $1 > LOGS/merge_mean_surface_crps_$1.log ``` +This script will merge all CRPS data into one file and then average over each region. Outputs are written to DN_OUT/profiles: +``` +All_CRPS_merged.nc +All_mask_means_crps_daily.nc +``` Finally the plots can be made with ```