Skip to content

Commit 931ff1a

Browse files
trondkrtrondkr
committed
Added option to keep existing files when running extraction for many variables and years. Useful if you need to restart the extraction and want to keep existing files
1 parent 00c4ad2 commit 931ff1a

6 files changed

+368
-216
lines changed

‎.ipynb_checkpoints/ECMWF_convert_to_ROMS-checkpoint.py

+8-7
Original file line numberDiff line numberDiff line change
@@ -85,13 +85,13 @@ def change_reference_date(self, ds, config_ecmwf: ECMWF_query):
8585

8686
dates = num2date(era5_time, units=era5_time_units, calendar=era5_time_cal)
8787
logging.debug(
88-
"[ECMWF_convert_to_ROMS] Converted time: {} to {}".format(dates[0], dates[-1]))
88+
f"[ECMWF_convert_to_ROMS] Converted time: {dates[0]} to {dates[-1]}")
8989

9090
# Convert back to julian day and convert to days since 1948-01-01 as that is standard for ROMS
9191
# days_to_seconds = 86400.0
9292
times = netCDF4.date2num(dates, units=config_ecmwf.time_units) # * days_to_seconds
9393
logging.debug(
94-
"[ECMWF_convert_to_ROMS] Converted time: {} to {} units: {}".format(times[0], times[-1], config_ecmwf.time_units))
94+
f"[ECMWF_convert_to_ROMS] Converted time: {times[0]} to {times[-1]} units: {config_ecmwf.time_units}")
9595

9696
return times, config_ecmwf.time_units, era5_time_cal
9797

@@ -117,8 +117,9 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u
117117
latitude = ds.variables['latitude'][:]
118118
time, time_units, time_calendar = self.change_reference_date(ds, config_ecmwf)
119119

120-
netcdf_roms_filename = netcdf_file[0:-3] + '_roms.nc'
121-
if os.path.exists(netcdf_roms_filename): os.remove(netcdf_roms_filename)
120+
netcdf_roms_filename = f"{out_filename[0:-3]}_roms.nc"
121+
if os.path.exists(netcdf_roms_filename):
122+
os.remove(netcdf_roms_filename)
122123
logging.info(
123124
"[ECMWF_convert_to_ROMS] Writing final product to file {}".format(netcdf_roms_filename))
124125

@@ -128,7 +129,7 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u
128129
"Atmospheric data on original grid but converted to ROMS units and parameter names." \
129130
"Files created using the ECMWF_tools toolbox:" \
130131
"https://github.com/trondkr/ERA5-ROMS"
131-
f1.history = "Created {}".format(datetime.now())
132+
f1.history = f"Created {datetime.now()}"
132133
f1.link = "https://github.com/trondkr/"
133134
f1.Conventions = "CF-1.0"
134135
fill_val = 1.0e35
@@ -163,12 +164,12 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u
163164
vnc = f1.createVariable(metadata['roms_name'], 'd', (metadata['time_name'], 'lat', 'lon'), fill_value=fill_val)
164165
vnc.long_name = metadata["name"]
165166
vnc.standard_name = metadata["short_name"]
166-
vnc.coordinates = "lon lat {}".format(metadata['time_name'])
167+
vnc.coordinates = f"lon lat {metadata['time_name']}"
167168
vnc.units = var_units
168169
vnc.missing_value = fill_val
169170

170171
vnc[:, :, :] = data_array[:,::-1,:]
171172
logging.info(
172-
"[ECMWF_convert_to_ROMS] Finished writing to file {}".format(netcdf_roms_filename))
173+
f"[ECMWF_convert_to_ROMS] Finished writing to file {netcdf_roms_filename}")
173174
os.remove(netcdf_file)
174175
f1.close()

‎.ipynb_checkpoints/ECMWF_query-checkpoint.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ def __init__(self):
88
self.setup_logging()
99

1010
self.use_era5 = True
11-
self.start_year = 1991
12-
self.end_year = 2020
11+
self.start_year = 1981
12+
self.end_year = 1990
1313
self.project = "A20"
1414
self.area = self.get_area_for_project(self.project)
15-
15+
self.skip_existing_files=True
1616
self.resultsdir = "../oceanography/ERA5/{}/".format(self.project)
1717
self.debug = False
1818

19-
self.extract_data_every_6_hours = True
19+
self.extract_data_every_N_hours = True
2020
self.time_units = "days since 1948-01-01 00:00:00"
2121
self.optionals = True # optional variables to extract depending on ROMS version (Rutgers or Kate)
2222
self.ROMS_version = "Kate" # "Rutgers" or "Kate" - the sea-ice component of Kates ROMS version uses downward
@@ -44,7 +44,7 @@ def __init__(self):
4444
'total_cloud_cover',
4545
'total_precipitation',
4646
'specific_humidity']
47-
47+
4848
if self.ROMS_version == "Kate":
4949
self.parameters.append('mean_surface_downward_short_wave_radiation_flux')
5050
elif self.ROMS_version == "Rutgers":

‎.ipynb_checkpoints/ECMWF_tools-checkpoint.py

+171-96
Original file line numberDiff line numberDiff line change
@@ -12,100 +12,175 @@
1212
#
1313
# Login to your account here to get these : https://cds.climate.copernicus.eu
1414

15-
class ECMWF_tools:
1615

17-
def __init__(self):
18-
# https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=form
19-
# https://confluence.ecmwf.int/pages/viewpage.action?pageId=82870405#ERA5:datadocumentation-Table4
20-
# Check data availability: http://apps.ecmwf.int/datasets/
21-
22-
self.config_ecmwf = ECMWF_query.ECMWF_query()
23-
self.server = cdsapi.Client(debug=self.config_ecmwf.debug)
24-
25-
def create_requests(self):
26-
years = [self.config_ecmwf.start_year + y for y in
27-
range(self.config_ecmwf.end_year - self.config_ecmwf.start_year)]
28-
29-
if not os.path.exists(self.config_ecmwf.resultsdir):
30-
os.mkdir(self.config_ecmwf.resultsdir)
31-
for year in years:
32-
33-
print("=> Downloading for year {}".format(year))
34-
35-
for parameter in self.config_ecmwf.parameters:
36-
print("=> getting data for : {} ".format(parameter))
37-
metadata = self.config_ecmwf.get_parameter_metadata(parameter)
38-
39-
out_filename = "{}{}_{}_year_{}.nc".format(self.config_ecmwf.resultsdir,
40-
self.config_ecmwf.dataset,
41-
metadata["short_name"],
42-
year)
43-
if os.path.exists(out_filename):
44-
os.remove(out_filename)
45-
46-
self.submit_request(parameter, str(year), out_filename)
47-
48-
def submit_request(self, parameter, year, out_filename):
49-
50-
times = [
51-
'00:00', '01:00', '02:00',
52-
'03:00', '04:00', '05:00',
53-
'06:00', '07:00', '08:00',
54-
'09:00', '10:00', '11:00',
55-
'12:00', '13:00', '14:00',
56-
'15:00', '16:00', '17:00',
57-
'18:00', '19:00', '20:00',
58-
'21:00', '22:00', '23:00',
59-
]
60-
61-
if self.config_ecmwf.extract_data_every_6_hours is True:
62-
times = ['00:00', '06:00', '12:00', '18:00']
63-
64-
options = {
65-
'product_type': 'reanalysis',
66-
"year": year,
67-
"month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
68-
'day': [
69-
'01', '02', '03',
70-
'04', '05', '06',
71-
'07', '08', '09',
72-
'10', '11', '12',
73-
'13', '14', '15',
74-
'16', '17', '18',
75-
'19', '20', '21',
76-
'22', '23', '24',
77-
'25', '26', '27',
78-
'28', '29', '30',
79-
'31',
80-
],
81-
'time': times,
82-
"variable": [parameter],
83-
'format': "netcdf",
84-
"area": self.config_ecmwf.area,
85-
"verbose": self.config_ecmwf.debug,
86-
}
87-
88-
# Add more specific options for variables on pressure surfaces
89-
if parameter == "specific_humidity":
90-
self.config_ecmwf.reanalysis = "reanalysis-era5-pressure-levels"
91-
options["levtype"] = 'pl'
92-
options["pressure_level"] = '1000'
93-
else:
94-
self.config_ecmwf.reanalysis = 'reanalysis-era5-single-levels'
95-
96-
try:
97-
# Do the request
98-
self.server.retrieve(self.config_ecmwf.reanalysis, options, out_filename)
99-
except Exception as e:
100-
print(e)
101-
print("[!] -------------------------- PROBLEMS WITH {0}".format(out_filename))
102-
103-
metadata = self.config_ecmwf.get_parameter_metadata(parameter)
104-
105-
converter = ECMWF_convert_to_ROMS.ECMWF_convert_to_ROMS()
106-
converter.convert_to_ROMS_units_standards(out_filename, metadata, parameter, self.config_ecmwf)
107-
108-
109-
if __name__ == '__main__':
110-
tool = ECMWF_tools()
111-
tool.create_requests()
16+
class ECMWF_tools:
17+
def __init__(self):
18+
# https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=form
19+
# https://confluence.ecmwf.int/pages/viewpage.action?pageId=82870405#ERA5:datadocumentation-Table4
20+
# Check data availability: http://apps.ecmwf.int/datasets/
21+
22+
self.config_ecmwf = ECMWF_query.ECMWF_query()
23+
self.server = cdsapi.Client(debug=self.config_ecmwf.debug)
24+
25+
def create_requests(self):
26+
years = [
27+
self.config_ecmwf.start_year + y
28+
for y in range(self.config_ecmwf.end_year - self.config_ecmwf.start_year)
29+
]
30+
31+
if not os.path.exists(self.config_ecmwf.resultsdir):
32+
os.mkdir(self.config_ecmwf.resultsdir)
33+
for year in years:
34+
35+
print("=> Downloading for year {}".format(year))
36+
37+
for parameter in self.config_ecmwf.parameters:
38+
print("=> getting data for : {} ".format(parameter))
39+
metadata = self.config_ecmwf.get_parameter_metadata(parameter)
40+
41+
out_filename = "{}{}_{}_year_{}.nc".format(
42+
self.config_ecmwf.resultsdir,
43+
self.config_ecmwf.dataset,
44+
metadata["short_name"],
45+
year,
46+
)
47+
roms_outfile = f"{out_filename[0:-3]}_roms.nc"
48+
if os.path.exists(roms_outfile):
49+
if not self.config_ecmwf.skip_existing_files:
50+
os.remove(out_filename)
51+
self.submit_request(parameter, str(year), out_filename)
52+
else:
53+
print(f"Skipping existing file: {out_filename}")
54+
else:
55+
self.submit_request(parameter, str(year), out_filename)
56+
57+
def submit_request(self, parameter, year, out_filename):
58+
59+
times = [
60+
"00:00",
61+
"01:00",
62+
"02:00",
63+
"03:00",
64+
"04:00",
65+
"05:00",
66+
"06:00",
67+
"07:00",
68+
"08:00",
69+
"09:00",
70+
"10:00",
71+
"11:00",
72+
"12:00",
73+
"13:00",
74+
"14:00",
75+
"15:00",
76+
"16:00",
77+
"17:00",
78+
"18:00",
79+
"19:00",
80+
"20:00",
81+
"21:00",
82+
"22:00",
83+
"23:00",
84+
]
85+
86+
if self.config_ecmwf.extract_data_every_N_hours is True:
87+
times = times = [
88+
"00:00",
89+
"02:00",
90+
"04:00",
91+
"06:00",
92+
"08:00",
93+
"10:00",
94+
"12:00",
95+
"14:00",
96+
"16:00",
97+
"18:00",
98+
"20:00",
99+
"22:00",
100+
]
101+
102+
options = {
103+
"product_type": "reanalysis",
104+
"year": year,
105+
"month": [
106+
"01",
107+
"02",
108+
"03",
109+
"04",
110+
"05",
111+
"06",
112+
"07",
113+
"08",
114+
"09",
115+
"10",
116+
"11",
117+
"12",
118+
],
119+
"day": [
120+
"01",
121+
"02",
122+
"03",
123+
"04",
124+
"05",
125+
"06",
126+
"07",
127+
"08",
128+
"09",
129+
"10",
130+
"11",
131+
"12",
132+
"13",
133+
"14",
134+
"15",
135+
"16",
136+
"17",
137+
"18",
138+
"19",
139+
"20",
140+
"21",
141+
"22",
142+
"23",
143+
"24",
144+
"25",
145+
"26",
146+
"27",
147+
"28",
148+
"29",
149+
"30",
150+
"31",
151+
],
152+
"time": times,
153+
"variable": [parameter],
154+
"format": "netcdf",
155+
"area": self.config_ecmwf.area,
156+
"verbose": self.config_ecmwf.debug,
157+
}
158+
159+
# Add more specific options for variables on pressure surfaces
160+
if parameter == "specific_humidity":
161+
self.config_ecmwf.reanalysis = "reanalysis-era5-pressure-levels"
162+
options["levtype"] = "pl"
163+
options["pressure_level"] = "1000"
164+
else:
165+
self.config_ecmwf.reanalysis = "reanalysis-era5-single-levels"
166+
167+
try:
168+
# Do the request
169+
self.server.retrieve(self.config_ecmwf.reanalysis, options, out_filename)
170+
except Exception as e:
171+
print(e)
172+
print(
173+
"[!] -------------------------- PROBLEMS WITH {0}".format(out_filename)
174+
)
175+
176+
metadata = self.config_ecmwf.get_parameter_metadata(parameter)
177+
178+
converter = ECMWF_convert_to_ROMS.ECMWF_convert_to_ROMS()
179+
converter.convert_to_ROMS_units_standards(
180+
out_filename, metadata, parameter, self.config_ecmwf
181+
)
182+
183+
184+
if __name__ == "__main__":
185+
tool = ECMWF_tools()
186+
tool.create_requests()

0 commit comments

Comments
 (0)