Skip to content
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
5c5bf9a
draft script for converting MFP CSV to YAML schedule
iuryt Jan 28, 2025
40c7ff9
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jan 28, 2025
366dede
add openpyxl
iuryt Feb 3, 2025
d046444
add mfp_to_yaml function
iuryt Feb 3, 2025
e3199fa
add new command to init to accept mfp file as input
iuryt Feb 3, 2025
d9fe46a
delete files from scripts/
iuryt Feb 3, 2025
7dc9bd7
deleted scripts files
iuryt Feb 3, 2025
a79433c
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Feb 3, 2025
11332f8
export the schedule body instead of saving file
iuryt Feb 13, 2025
ad54992
change name of cli param and adapt for new mfp_to_yaml function
iuryt Feb 13, 2025
66adb18
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Feb 13, 2025
2672afa
add warning message for time entry on yaml
iuryt Feb 13, 2025
a370641
change to pydantic and change name of variables
iuryt Feb 13, 2025
b87d944
add XBT
iuryt Feb 13, 2025
eba08b8
accept nonetype time
iuryt Feb 13, 2025
c0a52ac
change to Waypoint to BaseModel and add field_serializer for instrume…
iuryt Feb 13, 2025
526d2af
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Feb 13, 2025
c51043d
remove restriction for version
iuryt Feb 14, 2025
f3daaa7
add checking for columns from excel file
iuryt Feb 14, 2025
4c59420
add unit tests
iuryt Feb 14, 2025
b67b15d
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Feb 14, 2025
6f63cd4
Add update comments and var naming
VeckoTheGecko Feb 14, 2025
222df85
Remove buffering from mfp conversion
VeckoTheGecko Feb 14, 2025
c94567b
update references to Waypoint
VeckoTheGecko Feb 14, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ dependencies:
- pip
- pyyaml
- copernicusmarine >= 2
- openpyxl >= 3.1.5

# linting
- pre-commit
Expand Down
31 changes: 27 additions & 4 deletions src/virtualship/cli/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,26 @@
hash_to_filename,
)
from virtualship.expedition.do_expedition import _get_schedule, do_expedition
from virtualship.utils import SCHEDULE, SHIP_CONFIG
from virtualship.utils import SCHEDULE, SHIP_CONFIG, mfp_to_yaml


@click.command()
@click.argument(
"path",
type=click.Path(exists=False, file_okay=False, dir_okay=True),
)
def init(path):
"""Initialize a directory for a new expedition, with an example schedule and ship config files."""
@click.option(
"--mfp-file",
type=str,
default=None,
help='Partially initialise a project from an exported xlsx or csv file from NIOZ\' Marine Facilities Planning tool (specifically the "Export Coordinates > DD" option). User edits are required after initialisation.',
)
def init(path, mfp_file):
"""
Initialize a directory for a new expedition, with an example schedule and ship config files.

If --mfp-file is provided, it will generate the schedule from the MPF file instead.
"""
path = Path(path)
path.mkdir(exist_ok=True)

Expand All @@ -43,7 +53,20 @@ def init(path):
)

config.write_text(utils.get_example_config())
schedule.write_text(utils.get_example_schedule())

if mfp_file:
# Generate schedule.yaml from the MPF file
click.echo(f"Generating schedule from {mfp_file}...")
schedule_body = mfp_to_yaml(mfp_file, schedule)
click.echo(
"\n⚠️ The generated schedule does not contain time values. "
"\nPlease edit 'schedule.yaml' and manually add the necessary time values."
"\n🕒 Expected time format: 'YYYY-MM-DD HH:MM:SS' (e.g., '2023-10-20 01:00:00').\n"
)
else:
# Create a default example schedule
# schedule_body = utils.get_example_schedule()
schedule.write_text(utils.get_example_schedule())

click.echo(f"Created '{config.name}' and '{schedule.name}' at {path}.")

Expand Down
2 changes: 2 additions & 0 deletions src/virtualship/expedition/instrument_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@ class InstrumentType(Enum):
CTD = "CTD"
DRIFTER = "DRIFTER"
ARGO_FLOAT = "ARGO_FLOAT"
XBT = "XBT"

11 changes: 5 additions & 6 deletions src/virtualship/expedition/space_time_region.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,20 +38,19 @@ def _check_lon_lat_domain(self) -> Self:
raise ValueError("minimum_depth must be less than maximum_depth")
return self


class TimeRange(BaseModel):
"""Defines the temporal boundaries for a space-time region."""

start_time: datetime
end_time: datetime
start_time: datetime | None = None
end_time: datetime | None = None

@model_validator(mode="after")
def _check_time_range(self) -> Self:
if not self.start_time < self.end_time:
raise ValueError("start_time must be before end_time")
if self.start_time and self.end_time:
if not self.start_time < self.end_time:
raise ValueError("start_time must be before end_time")
return self
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@VeckoTheGecko @ammedd
While using pydantic, I had to change this to accept nonetype time, but this means that when we do schedule.from_yaml() for fetch it will not give any error, right? What should we do?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I thought that pydantic would have a way of disabling validation during the initialisation of the object, but looking further at the documentation its looking that that isn't possible ... . Longterm it would be good to have start_time and end_time not none, but perhaps thats something for a future PR



class SpaceTimeRegion(BaseModel):
"""An space-time region with spatial and temporal boundaries."""

Expand Down
106 changes: 106 additions & 0 deletions src/virtualship/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from importlib.resources import files
from typing import TextIO

import numpy as np
import pandas as pd
import yaml
from pydantic import BaseModel

Expand Down Expand Up @@ -37,3 +39,107 @@ def _dump_yaml(model: BaseModel, stream: TextIO) -> str | None:
def _generic_load_yaml(data: str, model: BaseModel) -> BaseModel:
"""Load a yaml string into a pydantic model."""
return model.model_validate(yaml.safe_load(data))


def mfp_to_yaml(excel_file_path: str, yaml_output_path: str):
"""
Generates a YAML file with spatial and temporal information based on instrument data from MFP excel file.

Parameters
----------
- excel_file_path (str): Path to the Excel file containing coordinate and instrument data.

The function:
1. Reads instrument and location data from the Excel file.
2. Determines the maximum depth and buffer based on the instruments present.
3. Ensures longitude and latitude values remain valid after applying buffer adjustments.
4. returns the yaml information.

"""

# Importing Schedule and related models from expedition module
from virtualship.expedition.schedule import Schedule
from virtualship.expedition.space_time_region import SpaceTimeRegion, SpatialRange, TimeRange
from virtualship.expedition.waypoint import Waypoint, Location
from virtualship.expedition.instrument_type import InstrumentType

# Read data from Excel
coordinates_data = pd.read_excel(
excel_file_path,
usecols=["Station Type", "Name", "Latitude", "Longitude", "Instrument"],
)
coordinates_data = coordinates_data.dropna()

# Define maximum depth (in meters) and buffer (in degrees) for each instrument
instrument_properties = {
"XBT": {"maximum_depth": 2000, "buffer": 1},
"CTD": {"maximum_depth": 5000, "buffer": 1},
"DRIFTER": {"maximum_depth": 1, "buffer": 5},
"ARGO_FLOAT": {"maximum_depth": 2000, "buffer": 5},
}

# Extract unique instruments from dataset using a set
unique_instruments = set()

for instrument_list in coordinates_data["Instrument"]:
instruments = instrument_list.split(", ") # Split by ", " to get individual instruments
unique_instruments |= set(instruments) # Union update with set of instruments

# Determine the maximum depth based on the unique instruments
maximum_depth = max(
instrument_properties.get(inst, {"maximum_depth": 0})["maximum_depth"]
for inst in unique_instruments
)
minimum_depth = 0

# Determine the buffer based on the maximum buffer of the instruments present
buffer = max(
instrument_properties.get(inst, {"buffer": 0})["buffer"]
for inst in unique_instruments
)

# Adjusted spatial range
min_longitude = coordinates_data["Longitude"].min() - buffer
max_longitude = coordinates_data["Longitude"].max() + buffer
min_latitude = coordinates_data["Latitude"].min() - buffer
max_latitude = coordinates_data["Latitude"].max() + buffer


spatial_range = SpatialRange(
minimum_longitude=coordinates_data["Longitude"].min() - buffer,
maximum_longitude=coordinates_data["Longitude"].max() + buffer,
minimum_latitude=coordinates_data["Latitude"].min() - buffer,
maximum_latitude=coordinates_data["Latitude"].max() + buffer,
minimum_depth=0,
maximum_depth=maximum_depth,
)


# Create space-time region object
space_time_region = SpaceTimeRegion(
spatial_range=spatial_range,
time_range=TimeRange(),
)

# Generate waypoints
waypoints = []
for _, row in coordinates_data.iterrows():
instruments = [InstrumentType(instrument) for instrument in row["Instrument"].split(", ")]
waypoints.append(
Waypoint(
instrument=instruments,
location=Location(latitude=row["Latitude"], longitude=row["Longitude"]),
)
)

# Create Schedule object
schedule = Schedule(
waypoints=waypoints,
space_time_region=space_time_region,
)

# Save to YAML file
schedule.to_yaml(yaml_output_path)