Skip to content

Commit

Permalink
fix a few typing errors
Browse files Browse the repository at this point in the history
  • Loading branch information
leoschwarz committed Nov 7, 2024
1 parent 0ba25bd commit 7ee1dd6
Show file tree
Hide file tree
Showing 5 changed files with 24 additions and 6 deletions.
6 changes: 5 additions & 1 deletion app_runner/src/app_runner/app_runner/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,13 @@ def run_process(self, chunk_dir: Path) -> None:

def run_register_outputs(self, chunk_dir: Path, workunit_ref: int | Path, reuse_default_resource: bool) -> None:
workunit_definition = WorkunitDefinition.from_ref(workunit_ref, client=self._client)
registration = workunit_definition.registration
if registration is None:
msg = "Workunit definition does not provide registration information"
raise ValueError(msg)
register_outputs(
outputs_yaml=chunk_dir / "outputs.yml",
workunit_id=workunit_definition.registration.workunit_id,
workunit_id=registration.workunit_id,
client=self._client,
ssh_user=self._ssh_user,
reuse_default_resource=reuse_default_resource,
Expand Down
5 changes: 4 additions & 1 deletion app_runner/src/app_runner/cli/outputs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from __future__ import annotations


import cyclopts

from bfabric import Bfabric
Expand Down Expand Up @@ -32,6 +31,10 @@ def register(

specs_list = OutputsSpec.read_yaml(outputs_yaml)
workunit = Workunit.find(id=workunit_id, client=client)
if workunit is None:
msg = f"Workunit with id {workunit_id} not found"
raise ValueError(msg)

register_all(
client=client,
workunit=workunit,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ def _dispatch_jobs_dataset_flow(self, definition: WorkunitDefinition, params: di
if config is None:
raise ValueError("dataset_flow is not configured")
dataset = Dataset.find(id=definition.execution.dataset, client=self._client)
if dataset is None:
msg = f"Dataset with id {definition.execution.dataset} not found"
raise ValueError(msg)
dataset_df = dataset.to_polars()
resources = Resource.find_all(ids=dataset_df[config.resource_column].unique().to_list(), client=self._client)
paths = []
Expand Down
13 changes: 9 additions & 4 deletions app_runner/src/app_runner/input_preparation/prepare.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
from __future__ import annotations

from typing import TYPE_CHECKING, Literal

from bfabric.entities import Resource, Dataset
from loguru import logger

from bfabric.entities import Resource, Dataset
from app_runner.input_preparation.integrity import IntegrityState
from app_runner.input_preparation.list_inputs import list_input_states
from app_runner.input_preparation.spec import (
ResourceSpec,
DatasetSpec,
InputSpecType,
InputsSpec,
)
from app_runner.input_preparation.integrity import IntegrityState
from app_runner.input_preparation.list_inputs import list_input_states
from app_runner.util.checksums import md5sum
from app_runner.util.scp import scp
from typing import TYPE_CHECKING, Literal

if TYPE_CHECKING:
from pathlib import Path
Expand Down Expand Up @@ -55,8 +55,13 @@ def clean_all(self, specs: list[InputSpecType]) -> None:

def prepare_resource(self, spec: ResourceSpec) -> None:
resource = Resource.find(id=spec.id, client=self._client)
if resource is None:
msg = f"Resource with id {spec.id} not found"
raise ValueError(msg)

# determine path to copy from
# TODO as we have seen sometimes a faster approach would be to copy from the NFS mount, but this needs to be
# configured or recognized somehow
scp_uri = f"{resource.storage.scp_prefix}{resource['relativepath']}"

# determine path to copy to
Expand Down
3 changes: 3 additions & 0 deletions app_runner/src/app_runner/output_registration/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@ def register_outputs(

# register all specs
workunit = Workunit.find(id=workunit_id, client=client)
if workunit is None:
msg = f"Workunit with id {workunit_id} not found"
raise ValueError(msg)
register_all(
client=client,
workunit=workunit,
Expand Down

0 comments on commit 7ee1dd6

Please sign in to comment.