Skip to content

Commit

Permalink
refactor: Update saving and loading of selected areas to use Parquet …
Browse files Browse the repository at this point in the history
…format
  • Loading branch information
janezlapajne committed Aug 28, 2024
1 parent c458f23 commit e323b6a
Showing 1 changed file with 4 additions and 6 deletions.
10 changes: 4 additions & 6 deletions source/helpers/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ def save_selected_areas(selected_areas: list[Pixels], category: str, label: str)
category_areas_dir.mkdir(parents=True, exist_ok=True)
logger.info("Saving selected areas to: %s" % category_areas_dir)
for idx, area in enumerate(selected_areas):
with open(category_areas_dir / f"{idx}.pkl", "wb") as f:
pickle.dump(area.df, f)
area.save_to_parquet(category_areas_dir / f"{idx}.parquet")


def load_selected_areas(category: str) -> list[Pixels]:
Expand All @@ -55,8 +54,7 @@ def load_selected_areas(category: str) -> list[Pixels]:
labels = category_areas_dir.glob("*")
for label in labels:
category_areas_dir_ = category_areas_dir / label.stem
for area_file in sorted(category_areas_dir_.glob("*.pkl")):
with open(area_file, "rb") as f:
area_df = pickle.load(f)
selected_areas.append(Pixels(area_df))
for area_file in sorted(category_areas_dir_.glob("*.parquet")):
area_df = Pixels.load_from_parquet(area_file)
selected_areas.append(area_df)
return selected_areas

0 comments on commit e323b6a

Please sign in to comment.