From 8c961885b76dbc29cade8186058fd41e7bb0713c Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 25 Sep 2025 15:58:41 +0200 Subject: [PATCH 01/14] automated copernicus product selection --- src/virtualship/cli/_fetch.py | 153 +++++++++++++++++++++++++++++----- 1 file changed, 133 insertions(+), 20 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index ac039d76..79fa50b7 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TYPE_CHECKING +import numpy as np from pydantic import BaseModel from virtualship.errors import IncompleteDownloadError @@ -86,6 +87,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / SCHEDULE, download_folder / SCHEDULE) + # data download if ( ( {"XBT", "CTD", "CDT_BGC", "SHIP_UNDERWATER_ST"} @@ -96,6 +98,13 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ): print("Ship data will be downloaded. Please wait...") + phys_product_id = select_product_id( + physical=True, + scheduled_time=end_datetime, + username=username, + password=password, + ) + # Define all ship datasets to download, including bathymetry download_dict = { "Bathymetry": { @@ -104,17 +113,17 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None "output_filename": "bathymetry.nc", }, "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "dataset_id": phys_product_id, "variables": ["uo", "vo"], "output_filename": "ship_uv.nc", }, "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "dataset_id": phys_product_id, "variables": ["so"], "output_filename": "ship_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "dataset_id": phys_product_id, "variables": ["thetao"], "output_filename": "ship_t.nc", }, @@ -151,12 +160,12 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None print("Drifter data will be downloaded. Please wait...") drifter_download_dict = { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "dataset_id": phys_product_id, "variables": ["uo", "vo"], "output_filename": "drifter_uv.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "dataset_id": phys_product_id, "variables": ["thetao"], "output_filename": "drifter_t.nc", }, @@ -193,17 +202,17 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None print("Argo float data will be downloaded. Please wait...") argo_download_dict = { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "dataset_id": phys_product_id, "variables": ["uo", "vo"], "output_filename": "argo_float_uv.nc", }, "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "dataset_id": phys_product_id, "variables": ["so"], "output_filename": "argo_float_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "dataset_id": phys_product_id, "variables": ["thetao"], "output_filename": "argo_float_t.nc", }, @@ -239,44 +248,50 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None if InstrumentType.CTD_BGC in instruments_in_schedule: print("CTD_BGC data will be downloaded. Please wait...") + bgc_args = { + "physical": False, + "scheduled_time": end_datetime, + "username": username, + "password": password, + } + ctd_bgc_download_dict = { "o2data": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "dataset_id": select_product_id(**{**bgc_args, "variable": "o2"}), "variables": ["o2"], "output_filename": "ctd_bgc_o2.nc", }, "chlorodata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "dataset_id": select_product_id(**{**bgc_args, "variable": "chl"}), "variables": ["chl"], "output_filename": "ctd_bgc_chl.nc", }, "nitratedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "dataset_id": select_product_id(**{**bgc_args, "variable": "no3"}), "variables": ["no3"], "output_filename": "ctd_bgc_no3.nc", }, "phosphatedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "dataset_id": select_product_id(**{**bgc_args, "variable": "po4"}), "variables": ["po4"], "output_filename": "ctd_bgc_po4.nc", }, "phdata": { - "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "dataset_id": select_product_id( + **{**bgc_args, "variable": "ph"} + ), # this will be monthly resolution if reanalysis(_interim) period "variables": ["ph"], "output_filename": "ctd_bgc_ph.nc", }, "phytoplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "dataset_id": select_product_id( + **{**bgc_args, "variable": "phyc"} + ), # this will be monthly resolution if reanalysis(_interim) period, "variables": ["phyc"], "output_filename": "ctd_bgc_phyc.nc", }, - "zooplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", - "variables": ["zooc"], - "output_filename": "ctd_bgc_zooc.nc", - }, "primaryproductiondata": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "dataset_id": select_product_id(**{**bgc_args, "variable": "nppv"}), "variables": ["nppv"], "output_filename": "ctd_bgc_nppv.nc", }, @@ -411,3 +426,101 @@ def complete_download(download_path: Path) -> None: metadata = DownloadMetadata(download_complete=True, download_date=datetime.now()) metadata.to_yaml(download_metadata) return + + +def select_product_id( + physical: bool, + scheduled_time: datetime, + username: str, + password: str, + variable: str | None = None, # only needed for BGC datasets +) -> str: + """ + Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC. + + BGC is more complicated than physical products. Often (re)analysis period and variable dependent, hence more custom logic here. + """ + product_ids = { + "phys": { + "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", + "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", + }, + "bgc": { + "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", + "analysis": None, # will be set per variable + }, + } + + bgc_analysis_ids = { + "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + } + + # pH and phytoplankton variables are available as *monthly* products only in renalysis(_interim) period + monthly_bgc_reanalysis_ids = { + "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", + } + monthly_bgc_reanalysis_interim_ids = { + "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", + } + + key = "phys" if physical else "bgc" + selected_id = None + + for period, pid in product_ids[key].items(): + # for BGC analysis, set pid per variable + if key == "bgc" and period == "analysis": + if variable is None or variable not in bgc_analysis_ids: + continue + pid = bgc_analysis_ids[variable] + # for BGC reanalysis, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis" + and variable in monthly_bgc_reanalysis_ids + ): + monthly_pid = monthly_bgc_reanalysis_ids[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, + username=username, + password=password, + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(scheduled_time) <= time_end_monthly: + pid = monthly_pid + # for BGC reanalysis_interim, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis_interim" + and variable in monthly_bgc_reanalysis_interim_ids + ): + monthly_pid = monthly_bgc_reanalysis_interim_ids[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, username=username, password=password + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(scheduled_time) <= time_end_monthly: + pid = monthly_pid + if pid is None: + continue + ds = copernicusmarine.open_dataset(pid, username=username, password=password) + time_end = ds["time"][-1].values + if np.datetime64(scheduled_time) <= time_end: + selected_id = pid + break + + if selected_id is None: + raise ValueError( + "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." + ) + + return selected_id From 04b50b9262290afdf5387846a16fd184a74be6fb Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 25 Sep 2025 15:59:16 +0200 Subject: [PATCH 02/14] extend plan tool time range back to start of reanalysis period --- src/virtualship/cli/_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 85539e3f..c426dade 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -155,7 +155,7 @@ def compose(self) -> ComposeResult: (str(year), year) # TODO: change from hard coding? ...flexibility for different datasets... for year in range( - 2022, + 1993, datetime.datetime.now().year + 1, ) ], From 3ccdc676bf03a97e4dd7a304c3c6eb62401e88fd Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 25 Sep 2025 16:00:00 +0200 Subject: [PATCH 03/14] remove zooplankton variable from CTD_BGC --- docs/user-guide/tutorials/CTD_transects.ipynb | 10 ++-------- src/virtualship/expedition/input_data.py | 3 --- src/virtualship/instruments/ctd_bgc.py | 6 ------ tests/instruments/test_ctd_bgc.py | 12 ------------ 4 files changed, 2 insertions(+), 29 deletions(-) diff --git a/docs/user-guide/tutorials/CTD_transects.ipynb b/docs/user-guide/tutorials/CTD_transects.ipynb index 63d8ad00..90f1ef2c 100644 --- a/docs/user-guide/tutorials/CTD_transects.ipynb +++ b/docs/user-guide/tutorials/CTD_transects.ipynb @@ -9,7 +9,7 @@ "\n", "This notebook demonstrates a simple plotting exercise for CTD data across a transect, using the output of a VirtualShip expedition. There are example plots embedded at the end, but these will ultimately be replaced by your own versions as you work through the notebook.\n", "\n", - "We can plot physical (temperature, salinity) or biogeochemical data (oxygen, chlorophyll, primary production, phyto/zoo-plankton, nutrients, pH) as measured by the VirtualShip `CTD` and `CTD_BGC` instruments, respectively.\n", + "We can plot physical (temperature, salinity) or biogeochemical data (oxygen, chlorophyll, primary production, phytoplankton, nutrients, pH) as measured by the VirtualShip `CTD` and `CTD_BGC` instruments, respectively.\n", "\n", "The plot(s) we will produce are simple plots which follow the trajectory of the expedition as a function of distance from the first waypoint, and are intended to be a starting point for your analysis. \n", "\n", @@ -93,7 +93,6 @@ "- \"nitrate\"\n", "- \"phosphate\"\n", "- \"ph\"\n", - "- \"zooplankton\"\n", "- \"phytoplankton\"\n", "- \"primary_production\"\n", "- \"chlorophyll\"\n", @@ -126,7 +125,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "id": "b32d2730", "metadata": {}, "outputs": [], @@ -162,11 +161,6 @@ " \"label\": \"pH\",\n", " \"ds_name\": \"ph\",\n", " },\n", - " \"zooplankton\": {\n", - " \"cmap\": cmo.algae,\n", - " \"label\": r\"Total zooplankton (mmol m$^{-3}$)\",\n", - " \"ds_name\": \"zooc\",\n", - " },\n", " \"phytoplankton\": {\n", " \"cmap\": cmo.algae,\n", " \"label\": r\"Total phytoplankton (mmol m$^{-3}$)\",\n", diff --git a/src/virtualship/expedition/input_data.py b/src/virtualship/expedition/input_data.py index 921daeda..fa48e0a7 100644 --- a/src/virtualship/expedition/input_data.py +++ b/src/virtualship/expedition/input_data.py @@ -142,7 +142,6 @@ def _load_ctd_bgc_fieldset(cls, directory: Path) -> FieldSet: "po4": directory.joinpath("ctd_bgc_po4.nc"), "ph": directory.joinpath("ctd_bgc_ph.nc"), "phyc": directory.joinpath("ctd_bgc_phyc.nc"), - "zooc": directory.joinpath("ctd_bgc_zooc.nc"), "nppv": directory.joinpath("ctd_bgc_nppv.nc"), } variables = { @@ -154,7 +153,6 @@ def _load_ctd_bgc_fieldset(cls, directory: Path) -> FieldSet: "po4": "po4", "ph": "ph", "phyc": "phyc", - "zooc": "zooc", "nppv": "nppv", } dimensions = { @@ -173,7 +171,6 @@ def _load_ctd_bgc_fieldset(cls, directory: Path) -> FieldSet: fieldset.po4.interp_method = "linear_invdist_land_tracer" fieldset.ph.interp_method = "linear_invdist_land_tracer" fieldset.phyc.interp_method = "linear_invdist_land_tracer" - fieldset.zooc.interp_method = "linear_invdist_land_tracer" fieldset.nppv.interp_method = "linear_invdist_land_tracer" # make depth negative diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index fde92ca1..0a34f61b 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -27,7 +27,6 @@ class CTD_BGC: Variable("po4", dtype=np.float32, initial=np.nan), Variable("ph", dtype=np.float32, initial=np.nan), Variable("phyc", dtype=np.float32, initial=np.nan), - Variable("zooc", dtype=np.float32, initial=np.nan), Variable("nppv", dtype=np.float32, initial=np.nan), Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. Variable("max_depth", dtype=np.float32), @@ -61,10 +60,6 @@ def _sample_phytoplankton(particle, fieldset, time): particle.phyc = fieldset.phyc[time, particle.depth, particle.lat, particle.lon] -def _sample_zooplankton(particle, fieldset, time): - particle.zooc = fieldset.zooc[time, particle.depth, particle.lat, particle.lon] - - def _sample_primary_production(particle, fieldset, time): particle.nppv = fieldset.nppv[time, particle.depth, particle.lat, particle.lon] @@ -166,7 +161,6 @@ def simulate_ctd_bgc( _sample_phosphate, _sample_ph, _sample_phytoplankton, - _sample_zooplankton, _sample_primary_production, _ctd_bgc_cast, ], diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index 5347a2ce..c1213884 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -49,7 +49,6 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: "po4": 14, "ph": 8.1, "phyc": 15, - "zooc": 16, "nppv": 17, "lat": ctd_bgcs[0].spacetime.location.lat, "lon": ctd_bgcs[0].spacetime.location.lon, @@ -61,7 +60,6 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: "po4": 19, "ph": 8.0, "phyc": 20, - "zooc": 21, "nppv": 22, "lat": ctd_bgcs[0].spacetime.location.lat, "lon": ctd_bgcs[0].spacetime.location.lon, @@ -75,7 +73,6 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: "po4": 14, "ph": 8.1, "phyc": 15, - "zooc": 16, "nppv": 17, "lat": ctd_bgcs[1].spacetime.location.lat, "lon": ctd_bgcs[1].spacetime.location.lon, @@ -87,7 +84,6 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: "po4": 19, "ph": 8.0, "phyc": 20, - "zooc": 21, "nppv": 22, "lat": ctd_bgcs[1].spacetime.location.lat, "lon": ctd_bgcs[1].spacetime.location.lon, @@ -105,7 +101,6 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: po4 = np.zeros((2, 2, 2, 2)) ph = np.zeros((2, 2, 2, 2)) phyc = np.zeros((2, 2, 2, 2)) - zooc = np.zeros((2, 2, 2, 2)) nppv = np.zeros((2, 2, 2, 2)) # Fill fields for both CTDs at surface and maxdepth @@ -139,11 +134,6 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: phyc[:, 1, 1, 0] = ctd_bgc_exp[1]["surface"]["phyc"] phyc[:, 0, 1, 0] = ctd_bgc_exp[1]["maxdepth"]["phyc"] - zooc[:, 1, 0, 1] = ctd_bgc_exp[0]["surface"]["zooc"] - zooc[:, 0, 0, 1] = ctd_bgc_exp[0]["maxdepth"]["zooc"] - zooc[:, 1, 1, 0] = ctd_bgc_exp[1]["surface"]["zooc"] - zooc[:, 0, 1, 0] = ctd_bgc_exp[1]["maxdepth"]["zooc"] - nppv[:, 1, 0, 1] = ctd_bgc_exp[0]["surface"]["nppv"] nppv[:, 0, 0, 1] = ctd_bgc_exp[0]["maxdepth"]["nppv"] nppv[:, 1, 1, 0] = ctd_bgc_exp[1]["surface"]["nppv"] @@ -159,7 +149,6 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: "po4": po4, "ph": ph, "phyc": phyc, - "zooc": zooc, "nppv": nppv, }, { @@ -208,7 +197,6 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: "po4", "ph", "phyc", - "zooc", "nppv", "lat", "lon", From d540e8edbe96bd6d4da227dcf57f0dbe28a6eed7 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 25 Sep 2025 16:56:47 +0200 Subject: [PATCH 04/14] catch errors where schedule spans two non-overlapping products --- src/virtualship/cli/_fetch.py | 50 ++++++++++++++++++++++++++++++----- src/virtualship/errors.py | 6 +++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 79fa50b7..ad44383c 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -9,7 +9,7 @@ import numpy as np from pydantic import BaseModel -from virtualship.errors import IncompleteDownloadError +from virtualship.errors import CopernicusTimeRangeError, IncompleteDownloadError from virtualship.utils import ( _dump_yaml, _generic_load_yaml, @@ -100,7 +100,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None phys_product_id = select_product_id( physical=True, - scheduled_time=end_datetime, + schedule_start=start_datetime, + schedule_end=end_datetime, username=username, password=password, ) @@ -250,7 +251,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None bgc_args = { "physical": False, - "scheduled_time": end_datetime, + "schedule_start": start_datetime, + "schedule_end": end_datetime, "username": username, "password": password, } @@ -430,7 +432,8 @@ def complete_download(download_path: Path) -> None: def select_product_id( physical: bool, - scheduled_time: datetime, + schedule_start: datetime, + schedule_end: datetime, username: str, password: str, variable: str | None = None, # only needed for BGC datasets @@ -495,7 +498,7 @@ def select_product_id( password=password, ) time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(scheduled_time) <= time_end_monthly: + if np.datetime64(schedule_end) <= time_end_monthly: pid = monthly_pid # for BGC reanalysis_interim, check if requires monthly product if ( @@ -508,13 +511,13 @@ def select_product_id( monthly_pid, username=username, password=password ) time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(scheduled_time) <= time_end_monthly: + if np.datetime64(schedule_end) <= time_end_monthly: pid = monthly_pid if pid is None: continue ds = copernicusmarine.open_dataset(pid, username=username, password=password) time_end = ds["time"][-1].values - if np.datetime64(scheduled_time) <= time_end: + if np.datetime64(schedule_end) <= time_end: selected_id = pid break @@ -523,4 +526,37 @@ def select_product_id( "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." ) + # confirm full schedule (start and end) is in the selected product's time range + schedule_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password + ) + return selected_id + + +def schedule_in_product_timerange( + selected_id: str, + schedule_start: datetime, + schedule_end: datetime, + username: str, + password: str, +) -> None: + """Raise error if schedule_start and schedule_end are not both within a selected Copernicus product's time range.""" + ds_selected = copernicusmarine.open_dataset( + selected_id, username=username, password=password + ) + time_values = ds_selected["time"].values + time_min = np.min(time_values) + time_max = np.max(time_values) + + if not ( + np.datetime64(schedule_start) >= time_min + and np.datetime64(schedule_end) <= time_max + ): + raise CopernicusTimeRangeError( + f"\n\n⚠️ Schedule start ({schedule_start}) and end ({schedule_end}) span different product IDs from the Copernicus Marine Catalogue. ⚠️" + "\n\nUnfortunately, this rare situation is not currently supported in VirtualShip." + "\n\nSee the following link for more information and guidance on how to fix: [<<< INSERT LINK >>>]" # TODO: ADD LINK when written the documentation! + f"\n\nSelected product '{selected_id}' covers {time_min} to {time_max}." + "\n\nPlease use the 'virtualship plan` tool to re-adjust your schedule, or manually edit the schedule.yaml and ship_config.yaml files." + ) diff --git a/src/virtualship/errors.py b/src/virtualship/errors.py index cdd58349..bfcba0e5 100644 --- a/src/virtualship/errors.py +++ b/src/virtualship/errors.py @@ -38,3 +38,9 @@ class UnexpectedError(Exception): """Error raised when there is an unexpected problem.""" pass + + +class CopernicusTimeRangeError(Exception): + """Error raised when start and end times in a schedule span different Copernicus Marine products.""" + + pass From b1a0d0d62cc3fc978df757ee6715fa62fb42a0a3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 26 Sep 2025 09:45:33 +0200 Subject: [PATCH 05/14] change non-overlapping handling to return analysis product instead --- src/virtualship/cli/_fetch.py | 39 ++++++++++++++++++----------------- src/virtualship/errors.py | 2 +- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index ad44383c..4e816f91 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -9,7 +9,7 @@ import numpy as np from pydantic import BaseModel -from virtualship.errors import CopernicusTimeRangeError, IncompleteDownloadError +from virtualship.errors import CopernicusCatalogueError, IncompleteDownloadError from virtualship.utils import ( _dump_yaml, _generic_load_yaml, @@ -522,41 +522,42 @@ def select_product_id( break if selected_id is None: - raise ValueError( + raise CopernicusCatalogueError( "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." ) - # confirm full schedule (start and end) is in the selected product's time range - schedule_in_product_timerange( + # handle the rare situation where start time and end time span different products, which is possible for reanalysis and reanalysis_interim + # in this case, return the analysis product which spans far back enough + if start_end_in_product_timerange( selected_id, schedule_start, schedule_end, username, password - ) + ): + return selected_id - return selected_id + else: + return ( + product_ids["phys"]["analysis"] if physical else bgc_analysis_ids[variable] + ) -def schedule_in_product_timerange( +def start_end_in_product_timerange( selected_id: str, schedule_start: datetime, schedule_end: datetime, username: str, password: str, -) -> None: - """Raise error if schedule_start and schedule_end are not both within a selected Copernicus product's time range.""" +) -> bool: + """Check schedule_start and schedule_end are both within a selected Copernicus product's time range.""" ds_selected = copernicusmarine.open_dataset( selected_id, username=username, password=password ) time_values = ds_selected["time"].values - time_min = np.min(time_values) - time_max = np.max(time_values) + time_min, time_max = np.min(time_values), np.max(time_values) - if not ( + if ( np.datetime64(schedule_start) >= time_min and np.datetime64(schedule_end) <= time_max ): - raise CopernicusTimeRangeError( - f"\n\n⚠️ Schedule start ({schedule_start}) and end ({schedule_end}) span different product IDs from the Copernicus Marine Catalogue. ⚠️" - "\n\nUnfortunately, this rare situation is not currently supported in VirtualShip." - "\n\nSee the following link for more information and guidance on how to fix: [<<< INSERT LINK >>>]" # TODO: ADD LINK when written the documentation! - f"\n\nSelected product '{selected_id}' covers {time_min} to {time_max}." - "\n\nPlease use the 'virtualship plan` tool to re-adjust your schedule, or manually edit the schedule.yaml and ship_config.yaml files." - ) + return True + + else: + return False diff --git a/src/virtualship/errors.py b/src/virtualship/errors.py index bfcba0e5..f7a4e962 100644 --- a/src/virtualship/errors.py +++ b/src/virtualship/errors.py @@ -40,7 +40,7 @@ class UnexpectedError(Exception): pass -class CopernicusTimeRangeError(Exception): +class CopernicusCatalogueError(Exception): """Error raised when start and end times in a schedule span different Copernicus Marine products.""" pass From b42b829d61ae7b20e016209730ac0f7a99ebd77b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 26 Sep 2025 10:39:02 +0200 Subject: [PATCH 06/14] update copernicus catalogue error class --- src/virtualship/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/errors.py b/src/virtualship/errors.py index f7a4e962..6026936e 100644 --- a/src/virtualship/errors.py +++ b/src/virtualship/errors.py @@ -41,6 +41,6 @@ class UnexpectedError(Exception): class CopernicusCatalogueError(Exception): - """Error raised when start and end times in a schedule span different Copernicus Marine products.""" + """Error raised when a relevant product is not found in the Copernicus Catalogue.""" pass From e90293a9ec0420eb9d0d912ac0ecf85033ecf415 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 26 Sep 2025 14:22:06 +0200 Subject: [PATCH 07/14] update test_cli and test_fetch for new logic --- tests/cli/test_cli.py | 33 ++++++++++++++++++++++++++++----- tests/cli/test_fetch.py | 41 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 015c3267..7f7446f9 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -1,5 +1,6 @@ from pathlib import Path +import numpy as np import pytest from click.testing import CliRunner @@ -8,13 +9,35 @@ @pytest.fixture -def copernicus_subset_no_download(monkeypatch): - """Mock the download function.""" +def copernicus_no_download(monkeypatch): + """Mock the download and open_dataset functions.""" + # mock for copernicusmarine.subset def fake_download(output_filename, output_directory, **_): Path(output_directory).joinpath(output_filename).touch() + # mock for copernicusmarine.open_dataset + class DummyTime: + def __getitem__(self, idx): + return self + + @property + def values(self): + return np.datetime64("2023-02-01") + + class DummyDS(dict): + def __getitem__(self, key): + if key == "time": + return DummyTime() + raise KeyError(key) + + def fake_open_dataset(*args, **kwargs): + return DummyDS() + monkeypatch.setattr("virtualship.cli._fetch.copernicusmarine.subset", fake_download) + monkeypatch.setattr( + "virtualship.cli._fetch.copernicusmarine.open_dataset", fake_open_dataset + ) yield @@ -68,15 +91,15 @@ def test_init_existing_schedule(): [".", "--password", "test"], ], ) -@pytest.mark.usefixtures("copernicus_subset_no_download") +@pytest.mark.usefixtures("copernicus_no_download") def test_fetch_both_creds_via_cli(runner, fetch_args): result = runner.invoke(fetch, fetch_args) assert result.exit_code == 1 assert "Both username and password" in result.exc_info[1].args[0] -@pytest.mark.usefixtures("copernicus_subset_no_download") +@pytest.mark.usefixtures("copernicus_no_download") def test_fetch(runner): - """Test the fetch command, but mock the download.""" + """Test the fetch command, but mock the downloads (and metadata interrogation).""" result = runner.invoke(fetch, [".", "--username", "test", "--password", "test"]) assert result.exit_code == 0 diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py index 856b72f6..0e50f873 100644 --- a/tests/cli/test_fetch.py +++ b/tests/cli/test_fetch.py @@ -1,5 +1,6 @@ from pathlib import Path +import numpy as np import pytest from pydantic import BaseModel @@ -21,13 +22,35 @@ @pytest.fixture -def copernicus_subset_no_download(monkeypatch): - """Mock the download function.""" +def copernicus_no_download(monkeypatch): + """Mock the download and open_dataset functions.""" + # mock for copernicusmarine.subset def fake_download(output_filename, output_directory, **_): Path(output_directory).joinpath(output_filename).touch() + # mock for copernicusmarine.open_dataset + class DummyTime: + def __getitem__(self, idx): + return self + + @property + def values(self): + return np.datetime64("2023-02-01") + + class DummyDS(dict): + def __getitem__(self, key): + if key == "time": + return DummyTime() + raise KeyError(key) + + def fake_open_dataset(*args, **kwargs): + return DummyDS() + monkeypatch.setattr("virtualship.cli._fetch.copernicusmarine.subset", fake_download) + monkeypatch.setattr( + "virtualship.cli._fetch.copernicusmarine.open_dataset", fake_open_dataset + ) yield @@ -55,9 +78,9 @@ def ship_config(tmpdir): return ship_config -@pytest.mark.usefixtures("copernicus_subset_no_download") +@pytest.mark.usefixtures("copernicus_no_download") def test_fetch(schedule, ship_config, tmpdir): - """Test the fetch command, but mock the download.""" + """Test the fetch command, but mock the download and dataset metadata interrogation.""" _fetch(Path(tmpdir), "test", "test") @@ -89,6 +112,16 @@ def test_complete_download(tmp_path): assert_complete_download(tmp_path) +def test_select_product_id(): + ... + # TODO + + +def test_start_end_in_product_timerange(): + ... + # TODO + + def test_assert_complete_download_complete(tmp_path): # Setup DownloadMetadata(download_complete=True).to_yaml(tmp_path / DOWNLOAD_METADATA) From 1152c719d9d0ece1e28cbf223962d2d1c04dc048 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:31:15 +0200 Subject: [PATCH 08/14] static schedule period into reanalysis period --- src/virtualship/static/schedule.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/virtualship/static/schedule.yaml b/src/virtualship/static/schedule.yaml index 7cb39423..34c5c01a 100644 --- a/src/virtualship/static/schedule.yaml +++ b/src/virtualship/static/schedule.yaml @@ -7,8 +7,8 @@ space_time_region: minimum_depth: 0 maximum_depth: 2000 time_range: - start_time: 2023-01-01 00:00:00 - end_time: 2023-02-01 00:00:00 + start_time: 1998-01-01 00:00:00 + end_time: 1998-02-01 00:00:00 waypoints: - instrument: - CTD @@ -16,27 +16,27 @@ waypoints: location: latitude: 0 longitude: 0 - time: 2023-01-01 00:00:00 + time: 1998-01-01 00:00:00 - instrument: - DRIFTER - CTD location: latitude: 0.01 longitude: 0.01 - time: 2023-01-01 01:00:00 + time: 1998-01-01 01:00:00 - instrument: - ARGO_FLOAT location: latitude: 0.02 longitude: 0.02 - time: 2023-01-01 02:00:00 + time: 1998-01-01 02:00:00 - instrument: - XBT location: latitude: 0.03 longitude: 0.03 - time: 2023-01-01 03:00:00 + time: 1998-01-01 03:00:00 - location: latitude: 0.03 longitude: 0.03 - time: 2023-01-01 03:00:00 + time: 1998-01-01 03:00:00 From dd96cd7fc89b24c11119bf38d3b9808c62baf183 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:32:50 +0200 Subject: [PATCH 09/14] add tests for product selection and time range validation --- tests/cli/test_fetch.py | 59 +++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py index 0e50f873..611911ff 100644 --- a/tests/cli/test_fetch.py +++ b/tests/cli/test_fetch.py @@ -2,6 +2,7 @@ import numpy as np import pytest +import xarray as xr from pydantic import BaseModel from virtualship.cli._fetch import ( @@ -16,6 +17,8 @@ get_existing_download, hash_model, hash_to_filename, + select_product_id, + start_end_in_product_timerange, ) from virtualship.models import Schedule, ShipConfig from virtualship.utils import get_example_config, get_example_schedule @@ -23,29 +26,24 @@ @pytest.fixture def copernicus_no_download(monkeypatch): - """Mock the download and open_dataset functions.""" + """Mock the copernicusmarine `subset` and `open_dataset` functions, approximating the reanalysis products.""" # mock for copernicusmarine.subset def fake_download(output_filename, output_directory, **_): Path(output_directory).joinpath(output_filename).touch() - # mock for copernicusmarine.open_dataset - class DummyTime: - def __getitem__(self, idx): - return self - - @property - def values(self): - return np.datetime64("2023-02-01") - - class DummyDS(dict): - def __getitem__(self, key): - if key == "time": - return DummyTime() - raise KeyError(key) - def fake_open_dataset(*args, **kwargs): - return DummyDS() + return xr.Dataset( + coords={ + "time": ( + "time", + [ + np.datetime64("1993-01-01"), + np.datetime64("2022-01-01"), + ], # mock up rough renanalysis period + ) + } + ) monkeypatch.setattr("virtualship.cli._fetch.copernicusmarine.subset", fake_download) monkeypatch.setattr( @@ -112,14 +110,29 @@ def test_complete_download(tmp_path): assert_complete_download(tmp_path) -def test_select_product_id(): - ... - # TODO +@pytest.mark.usefixtures("copernicus_no_download") +def test_select_product_id(schedule): + """Should return the physical reanalysis product id via the timings prescribed in the static schedule.yaml file.""" + result = select_product_id( + physical=True, + schedule_start=schedule.space_time_region.time_range.start_time, + schedule_end=schedule.space_time_region.time_range.end_time, + username="test", + password="test", + ) + assert result == "cmems_mod_glo_phy_my_0.083deg_P1D-m" -def test_start_end_in_product_timerange(): - ... - # TODO +@pytest.mark.usefixtures("copernicus_no_download") +def test_start_end_in_product_timerange(schedule): + """Should return True for valid range ass determined by the static schedule.yaml file.""" + assert start_end_in_product_timerange( + selected_id="cmems_mod_glo_phy_my_0.083deg_P1D-m", + schedule_start=schedule.space_time_region.time_range.start_time, + schedule_end=schedule.space_time_region.time_range.end_time, + username="test", + password="test", + ) def test_assert_complete_download_complete(tmp_path): From 56e400cbe740d141b0b8df79f8c77a951cabb0ae Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:33:12 +0200 Subject: [PATCH 10/14] refactor tests to use xarray for mock dataset --- tests/cli/test_cli.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 7f7446f9..5148176b 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -2,6 +2,7 @@ import numpy as np import pytest +import xarray as xr from click.testing import CliRunner from virtualship.cli.commands import fetch, init @@ -10,29 +11,24 @@ @pytest.fixture def copernicus_no_download(monkeypatch): - """Mock the download and open_dataset functions.""" + """Mock the copernicusmarine `subset` and `open_dataset` functions, approximating the reanalysis products.""" # mock for copernicusmarine.subset def fake_download(output_filename, output_directory, **_): Path(output_directory).joinpath(output_filename).touch() - # mock for copernicusmarine.open_dataset - class DummyTime: - def __getitem__(self, idx): - return self - - @property - def values(self): - return np.datetime64("2023-02-01") - - class DummyDS(dict): - def __getitem__(self, key): - if key == "time": - return DummyTime() - raise KeyError(key) - def fake_open_dataset(*args, **kwargs): - return DummyDS() + return xr.Dataset( + coords={ + "time": ( + "time", + [ + np.datetime64("1993-01-01"), + np.datetime64("2022-01-01"), + ], # mock up rough renanalysis period + ) + } + ) monkeypatch.setattr("virtualship.cli._fetch.copernicusmarine.subset", fake_download) monkeypatch.setattr( From 0552d3d56362fb3b37aac9a9e12a727ccae35fef Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 29 Sep 2025 13:35:40 +0200 Subject: [PATCH 11/14] update documentation, describing the different coperncius products used --- docs/user-guide/additional_information.md | 81 +++++++++++++++++++++++ docs/user-guide/index.md | 1 + docs/user-guide/quickstart.md | 4 ++ 3 files changed, 86 insertions(+) create mode 100644 docs/user-guide/additional_information.md diff --git a/docs/user-guide/additional_information.md b/docs/user-guide/additional_information.md new file mode 100644 index 00000000..1812d86b --- /dev/null +++ b/docs/user-guide/additional_information.md @@ -0,0 +1,81 @@ +# Additional information + +This file contains additional technical information and guidance not covered in the [Quickstart](https://virtualship.readthedocs.io/en/latest/user-guide/quickstart.html) guide or in the [Tutorials](https://virtualship.readthedocs.io/en/latest/user-guide/tutorials/index.html). + +### Copernicus Marine products + +VirtualShip supports running experiments anywhere in the global ocean from 1993 through to the present day (and approximately two weeks into the future), using the suite of products available from the [Copernicus Marine Data Store](https://data.marine.copernicus.eu/products). + +The data sourcing task is handled by the `virtualship fetch` command. The three products relied on by `fetch` to source data for all [VirtualShip instruments](https://virtualship.readthedocs.io/en/latest/user-guide/assignments/Research_proposal_intro.html#Measurement-Options) (both physical and biogeochemical) are: + +1. **Reanalysis** (or "hindcast" for biogeochemistry). +2. **Renalysis interim** (or "hindcast interim" for biogeochemistry). +3. **Analysis & Forecast**. + +```{tip} +The Copernicus Marine Service describe the differences between the three products in greater detail [here](https://help.marine.copernicus.eu/en/articles/4872705-what-are-the-main-differences-between-nearrealtime-and-multiyear-products). +``` + +As a general rule of thumb the three different products span different periods across the historical period to present and are intended to allow for continuity across the previous ~ 30 years. + +```{note} +The ethos for automated dataset selection in `virtualship fetch` is to prioritise the Reanalysis/Hindcast products where possible (the 'work horse'), then _interim products where possible for continuity, and finally filling the very near-present (and near-future) temporal range with the Analysis & Forecast products. +``` + +```{warning} +In the rare situation where the start and end times of an expedition schedule span different products, which is possible in the case of the end time being in the **Reanalysis_interim** period and the start time in the **Reanalysis** period, the **Analysis & Forecast** product will be automatically selected, as this spans back enough in time for this niche case. +``` + +### Data availability + +The following tables summarise which Copernicus product is selected by `virtualship fetch` per combination of time period and variable (see legend below). + +For biogeochemical variables `ph` and `phyc`, monthly products are required for hindcast and hindcast interim periods. For all other variables, daily products are available. + +#### Physical products + +| Period | Product ID | Temporal Resolution | Typical Years Covered | Variables | +| :------------------ | :--------------------------------------- | :------------------ | :---------------------------------- | :------------------------- | +| Reanalysis | `cmems_mod_glo_phy_my_0.083deg_P1D-m` | Daily | ~30 years ago to ~5 years ago | `uo`, `vo`, `so`, `thetao` | +| Reanalysis Interim | `cmems_mod_glo_phy_myint_0.083deg_P1D-m` | Daily | ~5 years ago to ~2 months ago | `uo`, `vo`, `so`, `thetao` | +| Analysis & Forecast | `cmems_mod_glo_phy_anfc_0.083deg_P1D-m` | Daily | ~2 months ago to ~2 weeks in future | `uo`, `vo`, `so`, `thetao` | + +--- + +#### Biogeochemical products + +| Period | Product ID | Temporal Resolution | Typical Years Covered | Variables | Notes | +| :---------------------------- | :----------------------------------------- | :------------------ | :---------------------------------- | :-------------------------------- | :------------------------------------- | +| Hindcast | `cmems_mod_glo_bgc_my_0.25deg_P1D-m` | Daily | ~30 years ago to ~5 years ago | `o2`, `chl`, `no3`, `po4`, `nppv` | Most BGC variables except `ph`, `phyc` | +| Hindcast (monthly) | `cmems_mod_glo_bgc_my_0.25deg_P1M-m` | Monthly | ~30 years ago to ~5 years ago | `ph`, `phyc` | Only `ph`, `phyc` (monthly only) | +| Hindcast Interim | `cmems_mod_glo_bgc_myint_0.25deg_P1D-m` | Daily | ~5 years ago to ~2 months ago | `o2`, `chl`, `no3`, `po4`, `nppv` | Most BGC variables except `ph`, `phyc` | +| Hindcast Interim (monthly) | `cmems_mod_glo_bgc_myint_0.25deg_P1M-m` | Monthly | ~5 years ago to ~2 months ago | `ph`, `phyc` | Only `ph`, `phyc` (monthly only) | +| Analysis & Forecast (O2) | `cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m` | Daily | ~2 months ago to ~2 weeks in future | `o2` | | +| Analysis & Forecast (Chl) | `cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m` | Daily | ~2 months ago to ~2 weeks in future | `chl`, `phyc` | | +| Analysis & Forecast (NO3/PO4) | `cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m` | Daily | ~2 months ago to ~2 weeks in future | `no3`, `po4` | | +| Analysis & Forecast (PH) | `cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m` | Daily | ~2 months ago to ~2 weeks in future | `ph` | | +| Analysis & Forecast (NPPV) | `cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m` | Daily | ~2 months ago to ~2 weeks in future | `nppv` | | + +--- + +```{note} +* "Typical Years Covered" are approximate and subject to change with new Copernicus data releases. +* For the most up-to-date information, always consult the Copernicus Marine product documentation. +* Certain BGC variables (`ph`, `phyc`) are only available as monthly products in hindcast and hindcast interim periods. +``` + +##### CMEMS variables legend + +| Variable Code | Full Variable Name | Category | +| :------------ | :------------------------------------------------------------ | :------------- | +| **uo** | Eastward Sea Water Velocity | Physical | +| **vo** | Northward Sea Water Velocity | Physical | +| **so** | Sea Water Salinity | Physical | +| **thetao** | Sea Water Potential Temperature | Physical | +| **o2** | Mole Concentration of Dissolved Molecular Oxygen in Sea Water | Biogeochemical | +| **chl** | Mass Concentration of Chlorophyll a in Sea Water | Biogeochemical | +| **no3** | Mole Concentration of Nitrate in Sea Water | Biogeochemical | +| **po4** | Mole Concentration of Phosphate in Sea Water | Biogeochemical | +| **nppv** | Net Primary Production of Biomass | Biogeochemical | +| **ph** | Sea Water pH | Biogeochemical | +| **phyc** | Mole Concentration of Phytoplankton | Biogeochemical | diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 9f8767ba..90c459bd 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -6,4 +6,5 @@ quickstart tutorials/index assignments/index +additional_information ``` diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index 59a514c7..58dbbae2 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -80,6 +80,10 @@ For the underway ADCP, there is a choice of using the 38 kHz OceanObserver or th ### Waypoint datetimes +```{note} +VirtualShip supports simulating experiments in the years 1993 through to the present day (and up to two weeks in the future) by leveraging the suite of products available Copernicus Marine Data Store (see [Fetch the data](#fetch-the-data)). The data download is automated based on the time period selected in the schedule. Different periods will rely on different products from the Copernicus Marine catalogue (see [Additional information](https://virtualship.readthedocs.io/en/latest/user-guide/additional_information.html) for more details). +``` + You will need to enter dates and times for each of the sampling stations/waypoints selected in the MFP route planning stage. This can be done under _Schedule Editor_ > _Waypoints & Instrument Selection_ in the planning tool. Each waypoint has its own sub-panel for parameter inputs (click on it to expand the selection options). Here, the time for each waypoint can be inputted. There is also an option to adjust the latitude/longitude coordinates and you can add or remove waypoints. From cf75a9e8ce436693be76ea1070164bf5266d3074 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 29 Sep 2025 15:31:42 +0200 Subject: [PATCH 12/14] fix broken link --- docs/user-guide/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index 58dbbae2..6b7125c9 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -81,7 +81,7 @@ For the underway ADCP, there is a choice of using the 38 kHz OceanObserver or th ### Waypoint datetimes ```{note} -VirtualShip supports simulating experiments in the years 1993 through to the present day (and up to two weeks in the future) by leveraging the suite of products available Copernicus Marine Data Store (see [Fetch the data](#fetch-the-data)). The data download is automated based on the time period selected in the schedule. Different periods will rely on different products from the Copernicus Marine catalogue (see [Additional information](https://virtualship.readthedocs.io/en/latest/user-guide/additional_information.html) for more details). +VirtualShip supports simulating experiments in the years 1993 through to the present day (and up to two weeks in the future) by leveraging the suite of products available Copernicus Marine Data Store (see [Fetch the data](#fetch-the-data)). The data download is automated based on the time period selected in the schedule. Different periods will rely on different products from the Copernicus Marine catalogue (see [Additional information](additional_information.md)). ``` You will need to enter dates and times for each of the sampling stations/waypoints selected in the MFP route planning stage. This can be done under _Schedule Editor_ > _Waypoints & Instrument Selection_ in the planning tool. From 62f140a6c6cb4852b5fa7213db65f78b499fad58 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 13 Oct 2025 13:58:18 +0200 Subject: [PATCH 13/14] re-label copernicus product information as documentation, move away from user guide & tutorials --- docs/index.md | 2 +- .../copernicus_products.md} | 8 ++------ docs/user-guide/index.md | 9 ++++++++- docs/user-guide/quickstart.md | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) rename docs/user-guide/{additional_information.md => documentation/copernicus_products.md} (91%) diff --git a/docs/index.md b/docs/index.md index 0210f2f5..9a7e8e38 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,7 +5,7 @@ :hidden: Home -user-guide/index +User Guide & Documation api/index contributing/index VirtualShip Website diff --git a/docs/user-guide/additional_information.md b/docs/user-guide/documentation/copernicus_products.md similarity index 91% rename from docs/user-guide/additional_information.md rename to docs/user-guide/documentation/copernicus_products.md index 1812d86b..78361984 100644 --- a/docs/user-guide/additional_information.md +++ b/docs/user-guide/documentation/copernicus_products.md @@ -1,8 +1,4 @@ -# Additional information - -This file contains additional technical information and guidance not covered in the [Quickstart](https://virtualship.readthedocs.io/en/latest/user-guide/quickstart.html) guide or in the [Tutorials](https://virtualship.readthedocs.io/en/latest/user-guide/tutorials/index.html). - -### Copernicus Marine products +# Copernicus Marine products VirtualShip supports running experiments anywhere in the global ocean from 1993 through to the present day (and approximately two weeks into the future), using the suite of products available from the [Copernicus Marine Data Store](https://data.marine.copernicus.eu/products). @@ -23,7 +19,7 @@ The ethos for automated dataset selection in `virtualship fetch` is to prioritis ``` ```{warning} -In the rare situation where the start and end times of an expedition schedule span different products, which is possible in the case of the end time being in the **Reanalysis_interim** period and the start time in the **Reanalysis** period, the **Analysis & Forecast** product will be automatically selected, as this spans back enough in time for this niche case. +In the rare situation where the start and end times of an expedition schedule span different products *and* there is no overlap in the respective dataset timeseries, which is possible in the case of the end time being in the **Reanalysis_interim** period and the start time in the **Reanalysis** period, the **Analysis & Forecast** product will be automatically selected, as this spans back enough in time for this niche case. ``` ### Data availability diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 90c459bd..6d950781 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -6,5 +6,12 @@ quickstart tutorials/index assignments/index -additional_information +``` + +# Documentation + +```{toctree} +:maxdepth: 1 + +documentation/copernicus_products.md ``` diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index 6b7125c9..2f1d4f63 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -81,7 +81,7 @@ For the underway ADCP, there is a choice of using the 38 kHz OceanObserver or th ### Waypoint datetimes ```{note} -VirtualShip supports simulating experiments in the years 1993 through to the present day (and up to two weeks in the future) by leveraging the suite of products available Copernicus Marine Data Store (see [Fetch the data](#fetch-the-data)). The data download is automated based on the time period selected in the schedule. Different periods will rely on different products from the Copernicus Marine catalogue (see [Additional information](additional_information.md)). +VirtualShip supports simulating experiments in the years 1993 through to the present day (and up to two weeks in the future) by leveraging the suite of products available Copernicus Marine Data Store (see [Fetch the data](#fetch-the-data)). The data download is automated based on the time period selected in the schedule. Different periods will rely on different products from the Copernicus Marine catalogue (see [Documentation](documentation/copernicus_products.md)). ``` You will need to enter dates and times for each of the sampling stations/waypoints selected in the MFP route planning stage. This can be done under _Schedule Editor_ > _Waypoints & Instrument Selection_ in the planning tool. From 5f77bc32bf306c46835e4dffabb63cc8d47d278e Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 13 Oct 2025 14:00:40 +0200 Subject: [PATCH 14/14] fix typo in User Guide & Documentation link --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 9a7e8e38..dc16c1b9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,7 +5,7 @@ :hidden: Home -User Guide & Documation +User Guide & Documentation api/index contributing/index VirtualShip Website