Skip to content
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
70 commits
Select commit Hold shift + click to select a range
7ac5d4d
add bbox and time range
iuryt Nov 13, 2024
8c641cc
add AreaOfInterest class
iuryt Nov 13, 2024
6b6b874
export AreaOfInterest
iuryt Nov 13, 2024
6743f1f
add AreaOfInterest to Schedule class
iuryt Nov 13, 2024
40d4d0d
fetch function for downloading data based on the area of interest
iuryt Nov 13, 2024
3194056
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 13, 2024
aacbb38
import copernicusmarine
iuryt Nov 13, 2024
5988c90
import datetime
iuryt Nov 13, 2024
fe77ea1
fix conflict
iuryt Nov 13, 2024
3d7697e
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 13, 2024
fafc74f
fix typo
iuryt Nov 13, 2024
ad97b93
fix conflict
iuryt Nov 13, 2024
46af584
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 13, 2024
e092658
Update src/virtualship/cli/commands.py
iuryt Nov 15, 2024
bb776a6
Update src/virtualship/cli/commands.py
iuryt Nov 15, 2024
870a2c2
patch `path` varname
VeckoTheGecko Nov 21, 2024
8df718a
update _get_ship_config and _get_schedule
VeckoTheGecko Nov 21, 2024
c691a47
Add credential config
VeckoTheGecko Nov 21, 2024
5fbf812
Add note to obtain credentials
VeckoTheGecko Nov 21, 2024
391f5ef
Add copernicusmarine dep
VeckoTheGecko Nov 21, 2024
17fef2b
Patch types
VeckoTheGecko Nov 21, 2024
672fa6a
Update user messages in `virtualship fetch`
VeckoTheGecko Nov 21, 2024
51451cf
Adding minimum and maximum depth to area_of_interest
erikvansebille Nov 22, 2024
bc88360
Setting None as default option for drifter endtime
erikvansebille Nov 22, 2024
f3a0dc9
Removing redundant import in text_drifter
erikvansebille Nov 22, 2024
3b5edc6
Pin copernicusmarine < 2
VeckoTheGecko Nov 22, 2024
fb1effd
Validate domains for area_of_interest
VeckoTheGecko Dec 17, 2024
97b4d8d
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 17, 2024
bb8bacd
Make depth optional in AreaOfInterest
VeckoTheGecko Dec 17, 2024
43f3329
Make AreaOfInterest optional in Schedule
VeckoTheGecko Dec 17, 2024
368bd64
Rename AreaOfInterest to SpaceTimeRegion
VeckoTheGecko Dec 17, 2024
123ad3d
Rename area_of_interest.py file
VeckoTheGecko Dec 17, 2024
471d43a
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 17, 2024
ef1e6bc
Add fetch utility functions and tests
VeckoTheGecko Dec 17, 2024
3bb4e6a
Patch attr name
VeckoTheGecko Dec 17, 2024
737e34f
pointing the right function to make it work
iuryt Dec 21, 2024
6b2eb20
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 21, 2024
60952d8
remove temporary code
iuryt Dec 21, 2024
1854258
fix typo
iuryt Dec 21, 2024
77d40ab
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 21, 2024
ac61cf1
ask for username and password if not provided
iuryt Dec 21, 2024
61ab177
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 21, 2024
34bea16
creates function that makes a sha256 from a dictionary
iuryt Dec 21, 2024
a125a11
create a data folder based on the hash from area_of_interest
iuryt Dec 21, 2024
61a9c32
merge with pre-commit ¬¬
iuryt Dec 21, 2024
a2a61c8
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 21, 2024
777f505
insert blank lines
iuryt Dec 21, 2024
f6908df
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 21, 2024
2a3c9ab
Update hash to use _fetch utils
VeckoTheGecko Jan 13, 2025
66a1bb9
Avoid mixing matching creds
VeckoTheGecko Jan 13, 2025
3012812
Download caching feature
VeckoTheGecko Jan 13, 2025
4160437
Update input data loading with new data folder structure
VeckoTheGecko Jan 13, 2025
fd060d7
Handle download_cleanup on wrong credentials
VeckoTheGecko Jan 13, 2025
d2a7bc8
Copy schedule to download folde
VeckoTheGecko Jan 13, 2025
18b86ec
Patch download command
VeckoTheGecko Jan 13, 2025
524c650
Add `input_data` param for testing
VeckoTheGecko Jan 13, 2025
ad995c3
Add tests
VeckoTheGecko Jan 13, 2025
afe6112
Add fetch test
VeckoTheGecko Jan 13, 2025
5669e41
Pin copernicusmarine >= 2
VeckoTheGecko Jan 13, 2025
bcfe01e
RTD explicit config key
VeckoTheGecko Jan 13, 2025
0ef7cf0
Delete download_data script
VeckoTheGecko Jan 13, 2025
b830393
Update pyproject.toml
VeckoTheGecko Jan 13, 2025
8d35d4b
Improve filename to hash conversion
VeckoTheGecko Jan 13, 2025
9ffd8de
Rename to assert_complete_download
VeckoTheGecko Jan 13, 2025
cfc50cc
Error message when area of interest isn't defined
VeckoTheGecko Jan 14, 2025
7132766
Add area of interest hash salting
VeckoTheGecko Jan 14, 2025
8698d13
Update 'area of interest' to 'space-time region' throughout
VeckoTheGecko Jan 14, 2025
04d7c13
Avoid circular import
VeckoTheGecko Jan 15, 2025
93576c8
virtualship help documentation
VeckoTheGecko Jan 15, 2025
a2290b6
Update help messages
VeckoTheGecko Jan 15, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 73 additions & 4 deletions src/virtualship/cli/commands.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from datetime import datetime
from pathlib import Path

import click
import copernicusmarine

from virtualship import utils
from virtualship.expedition.do_expedition import do_expedition
from virtualship.expedition.do_expedition import _get_schedule, do_expedition
from virtualship.utils import SCHEDULE, SHIP_CONFIG


Expand Down Expand Up @@ -45,9 +47,76 @@ def init(path):
"path",
type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True),
)
def fetch(path):
"""Entrypoint for the tool."""
raise NotImplementedError("Not implemented yet.")
def fetch(expedition_dir: str | Path) -> None:
"""Entrypoint for the tool to download data based on area of interest."""
if isinstance(expedition_dir, str):
expedition_dir = Path(expedition_dir)

# Load schedule
schedule = _get_schedule(expedition_dir)
if schedule is None:
print("Error: Schedule file not found.")
return

# Extract area_of_interest details from the schedule
spatial_range = schedule.area_of_interest.spatial_range
time_range = schedule.area_of_interest.time_range
start_datetime = datetime.strptime(time_range.start_time, "%Y-%m-%d %H:%M:%S")
end_datetime = datetime.strptime(time_range.end_time, "%Y-%m-%d %H:%M:%S")

# Prompt for user credentials
username = input("username: ")
password = input("password: ")

# Define all datasets to download, including bathymetry
download_dict = {
"Bathymetry": {
"dataset_id": "cmems_mod_glo_phy_my_0.083deg_static",
"variables": ["deptho"],
"output_filename": "bathymetry.nc",
"force_dataset_part": "bathy",
},
"UVdata": {
"dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i",
"variables": ["uo", "vo"],
"output_filename": "default_uv.nc",
},
"Sdata": {
"dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i",
"variables": ["so"],
"output_filename": "default_s.nc",
},
"Tdata": {
"dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i",
"variables": ["thetao"],
"output_filename": "default_t.nc",
},
}

# Iterate over all datasets and download each based on area_of_interest
for dataset in download_dict.values():
copernicusmarine.subset(
dataset_id=dataset["dataset_id"],
variables=dataset["variables"],
minimum_longitude=spatial_range.min_longitude,
maximum_longitude=spatial_range.max_longitude,
minimum_latitude=spatial_range.min_latitude,
maximum_latitude=spatial_range.max_latitude,
start_datetime=start_datetime,
end_datetime=end_datetime,
minimum_depth=0.49402499198913574,
maximum_depth=5727.9169921875,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we always need to download full-depth datasets? @ammedd, would there also be cases (when not using CTD?) that only upper/surface ocean data are needed? Or would that just overcomplicate the script?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can add this very easily to the area_of_interest . Do we have checks to see if the data covers the survey?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The are indeed (many) cases where the full-depth is not needed. It would be good to check the ship_config for the maximum depth needed.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this something that we should deal with in this PR? It would require some choices about depths for which datasets/instruments, which might be better to discuss in another issue/future PR.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No thanks for bringing it up, let's discuss in the meeting tomorrow and move it to a new PR.

output_filename=dataset["output_filename"],
output_directory=expedition_dir,
username=username,
password=password,
force_download=True,
force_dataset_part=dataset.get(
"force_dataset_part"
), # Only used if specified in dataset
)

click.echo("Data download based on area of interest completed.")


@click.command(help="Do the expedition.")
Expand Down
2 changes: 2 additions & 0 deletions src/virtualship/expedition/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Everything for simulating an expedition."""

from .area_of_interest import AreaOfInterest
from .do_expedition import do_expedition
from .instrument_type import InstrumentType
from .schedule import Schedule
Expand All @@ -23,6 +24,7 @@
"ShipConfig",
"ShipUnderwaterSTConfig",
"Waypoint",
"AreaOfInterest",
"do_expedition",
"instruments",
]
30 changes: 30 additions & 0 deletions src/virtualship/expedition/area_of_interest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
"""AreaOfInterest class."""

from dataclasses import dataclass
from datetime import datetime


@dataclass
class SpatialRange:
"""Defines the geographic boundaries for an area of interest."""

minimum_longitude: float
maximum_longitude: float
minimum_latitude: float
maximum_latitude: float


@dataclass
class TimeRange:
"""Defines the temporal boundaries for an area of interest."""

start_time: datetime
end_time: datetime


@dataclass
class AreaOfInterest:
"""An area of interest with spatial and temporal boundaries."""

spatial_range: SpatialRange
time_range: TimeRange
2 changes: 2 additions & 0 deletions src/virtualship/expedition/schedule.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@
import pydantic
import yaml

from .area_of_interest import AreaOfInterest
from .waypoint import Waypoint


class Schedule(pydantic.BaseModel):
"""Schedule of the virtual ship."""

waypoints: list[Waypoint]
area_of_interest: AreaOfInterest

model_config = pydantic.ConfigDict(extra="forbid")

Expand Down
9 changes: 9 additions & 0 deletions src/virtualship/static/schedule.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,12 @@
area_of_interest:
spatial_range:
minimum_longitude: -5
maximum_longitude: 5
minimum_latitude: -5
maximum_latitude: 5
time_range:
start_time: 2023-01-01 00:00:00
end_time: 2023-02-01 00:00:00
waypoints:
- instrument: CTD
location:
Expand Down