Skip to content

Commit

Permalink
feat: slide_etl doesn't use dask if it's just one slide
Browse files Browse the repository at this point in the history
  • Loading branch information
raylim committed Nov 8, 2023
1 parent 6c07692 commit 9d89c70
Showing 1 changed file with 37 additions and 22 deletions.
59 changes: 37 additions & 22 deletions src/luna/pathology/cli/slide_etl.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,37 +133,29 @@ def slide_etl(
Returns:
DataFrame[SlideSchema]: dataframe containing the metadata of all the slides
"""
sb = SlideBuilder(storage_options, output_storage_options=output_storage_options)
if isinstance(slide_urls, str):
slide_urls = [slide_urls]
return __slide_etl(
sb, slide_urls, project_name, comment, output_urlpath, no_copy
)

client = get_or_create_dask_client()
sb = SlideBuilder(storage_options, output_storage_options=output_storage_options)

futures = [
client.submit(
sb.get_slide,
url,
project_name=project_name,
comment=comment,
__slide_etl,
sb,
slide_url,
project_name,
comment,
output_urlpath,
no_copy,
)
for url in slide_urls
for slide_url in slide_urls
]
progress(futures)
slides = client.gather(futures)
if not no_copy and output_urlpath:
futures = [
client.submit(
sb.copy_slide,
slide,
output_urlpath,
)
for slide in slides
]

df = DataFrame[SlideSchema](
pd.json_normalize([x.__dict__ for x in client.gather(futures)])
)
return df
dfs = client.gather(futures)
return pd.concat(dfs)


class SlideBuilder:
Expand Down Expand Up @@ -238,6 +230,29 @@ def get_slide(self, url, project_name="", comment="") -> Slide:
# def estimate_stain(self, slide):


def __slide_etl(
sb: SlideBuilder,
slide_url: str,
project_name: str,
comment: str = "",
output_urlpath: str = "",
no_copy: bool = False,
) -> DataFrame:
slide = sb.get_slide(
slide_url,
project_name=project_name,
comment=comment,
)
if not no_copy and output_urlpath:
slide = sb.copy_slide(
slide,
output_urlpath,
)

df = DataFrame[SlideSchema](pd.json_normalize(slide.__dict__))
return df


def fire_cli():
fire.Fire(cli)

Expand Down

0 comments on commit 9d89c70

Please sign in to comment.