diff --git a/imdataset_creator/__init__.py b/imdataset_creator/__init__.py index 499e9fe..4f36383 100644 --- a/imdataset_creator/__init__.py +++ b/imdataset_creator/__init__.py @@ -2,6 +2,5 @@ from .alphanumeric_sort import alphanumeric_sort from .config_handler import ConfigHandler from .configs import FilterData, MainConfig -from .datarules.base_rules import ExprDict, File, Filter, Input, Output, Producer, Rule -from .datarules.dataset_builder import DatasetBuilder, chunk_split +from .datarules import DatasetBuilder, ExprDict, File, Filter, Input, Output, Producer, Rule, chunk_split from .scenarios import FileScenario, OutputScenario diff --git a/imdataset_creator/config_handler.py b/imdataset_creator/config_handler.py index 59b9d97..cc6cced 100644 --- a/imdataset_creator/config_handler.py +++ b/imdataset_creator/config_handler.py @@ -2,7 +2,7 @@ from pathlib import Path from .configs import MainConfig, _repr_indent -from .datarules.base_rules import Input, Output, Producer, Rule +from .datarules import Input, Output, Producer, Rule from .file import File from .scenarios import FileScenario, OutputScenario diff --git a/imdataset_creator/datarules/__init__.py b/imdataset_creator/datarules/__init__.py index 52b2bba..78bf7ae 100644 --- a/imdataset_creator/datarules/__init__.py +++ b/imdataset_creator/datarules/__init__.py @@ -1 +1,3 @@ from . import base_rules, data_rules, dataset_builder, image_rules +from .base_rules import ExprDict, File, Filter, Input, Output, Producer, Rule +from .dataset_builder import DatasetBuilder, chunk_split diff --git a/imdataset_creator/gui/main_window.py b/imdataset_creator/gui/main_window.py index 0cc3f1f..29fc7e5 100644 --- a/imdataset_creator/gui/main_window.py +++ b/imdataset_creator/gui/main_window.py @@ -8,8 +8,8 @@ from pprint import pprint from polars import DataFrame, concat -from PySide6.QtCore import QRect, Qt, QThread, Signal, Slot -from PySide6.QtGui import QAction, QIcon, QKeySequence +from PySide6.QtCore import Qt, QThread, Signal, Slot +from PySide6.QtGui import QAction, QKeySequence from PySide6.QtWidgets import ( QFileDialog, QGridLayout, @@ -23,25 +23,14 @@ ) from rich import print as rprint -from imdataset_creator.datarules.dataset_builder import chunk_split - -from .. import DatasetBuilder, File, Input +from .. import DatasetBuilder, File from ..configs import MainConfig +from ..datarules import chunk_split from .err_dialog import catch_errors -from .input_view import InputList, InputView -from .output_filters import FilterView -from .output_view import OutputList, OutputView -from .producer_views import FileInfoProducerView, HashProducerView, ImShapeProducerView, ProducerList, ProducerView -from .rule_views import ( - BlacklistWhitelistView, - ChannelRuleView, - HashRuleView, - ResRuleView, - RuleList, - RuleView, - StatRuleView, - TotalLimitRuleView, -) +from .input_view import InputList +from .output_view import OutputList +from .producer_views import ProducerList +from .rule_views import RuleList log = logging.getLogger() @@ -109,8 +98,6 @@ def __init__(self, parent, cfg_path=Path("config.json")): # self.run_population_button.clicked.connect(self.run_population) self.output_list = OutputList(self) - self.output_list.set_text("Outputs") - self.output_list.register_item(OutputView) self.lists = QSplitter(self) self.lists.addWidget(self.input_list) diff --git a/imdataset_creator/gui/output_filters.py b/imdataset_creator/gui/output_filters.py index 89fe108..8df31c1 100644 --- a/imdataset_creator/gui/output_filters.py +++ b/imdataset_creator/gui/output_filters.py @@ -14,7 +14,7 @@ QWidget, ) -from ..datarules.base_rules import Filter +from ..datarules import Filter from ..image_filters import destroyers, resizer from .frames import FlowItem, FlowList, MiniCheckList, tooltip diff --git a/imdataset_creator/gui/output_view.py b/imdataset_creator/gui/output_view.py index 9281499..0e5df08 100644 --- a/imdataset_creator/gui/output_view.py +++ b/imdataset_creator/gui/output_view.py @@ -7,7 +7,7 @@ from PySide6.QtWidgets import QCheckBox, QFileDialog, QLabel, QLineEdit from ..configs import OutputData -from ..datarules.base_rules import Output +from ..datarules import Output from .frames import FlowList from .input_view import InputView from .output_filters import FilterList @@ -81,3 +81,8 @@ def from_config(cls, cfg: OutputData, parent=None): class OutputList(FlowList): items: list[OutputView] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.set_text("Outputs") + self.register_item(OutputView) diff --git a/imdataset_creator/image_filters/destroyers.py b/imdataset_creator/image_filters/destroyers.py index 0d12b4a..ee2f807 100644 --- a/imdataset_creator/image_filters/destroyers.py +++ b/imdataset_creator/image_filters/destroyers.py @@ -13,7 +13,7 @@ from numpy import ndarray from ..configs.configtypes import FilterData -from ..datarules.base_rules import Filter +from ..datarules import Filter log = logging.getLogger() diff --git a/imdataset_creator/image_filters/resizer.py b/imdataset_creator/image_filters/resizer.py index 82e6188..c5ff04f 100644 --- a/imdataset_creator/image_filters/resizer.py +++ b/imdataset_creator/image_filters/resizer.py @@ -1,14 +1,13 @@ from dataclasses import dataclass from enum import Enum from random import choice -from typing import Literal import cv2 import numpy as np from imdataset_creator.configs.configtypes import FilterData -from ..datarules.base_rules import Filter +from ..datarules import Filter np_gen = np.random.default_rng() diff --git a/imdataset_creator/scenarios.py b/imdataset_creator/scenarios.py index 741d455..f975715 100644 --- a/imdataset_creator/scenarios.py +++ b/imdataset_creator/scenarios.py @@ -6,7 +6,8 @@ import numpy as np from .configs.keyworded import fancy_repr -from .datarules.base_rules import File, Filter +from .datarules import Filter +from .file import File # IDK Wtf to call these things other than scenarios diff --git a/manage_dataframe.ipynb b/manage_dataframe.ipynb index 94fccb5..9e9a112 100644 --- a/manage_dataframe.ipynb +++ b/manage_dataframe.ipynb @@ -11,8 +11,8 @@ "\n", "import polars as pl\n", "\n", - "from imdataset_creator.datarules.dataset_builder import DatasetBuilder\n", - "from imdataset_creator.datarules.base_rules import Rule" + "from imdataset_creator.datarules import DatasetBuilder\n", + "from imdataset_creator.datarules import Rule" ] }, { @@ -106,66 +106,6 @@ "plt.xlabel(\"modifiedtime\")\n", "plt.ylabel(\"sum resolution\")" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Link a list of files based on data from the database" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "input_folder = Path(\"/mnt/Toshiba/.Grabber/\")\n", - "output = input_folder.with_name(f\"{input_folder.name}-linked\")\n", - "category = \"hash\"\n", - "config_path = Path(\"database_config.toml\")\n", - "overwrite = True\n", - "\n", - "populate=False # you'll need to specify a rule to add to the builder if you use this\n", - "# define rules here\n", - "# from dataset_filters.external_filters import HashRule\n", - "rule_list: list[Rule] = [\n", - "# HashFilter()\n", - " ]\n", - "# ^^ these filters do not change the output size. They only dictate what columns are available,\n", - "# if what you want is not already." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Run\n", - "output.mkdir(exist_ok=True)\n", - "exts = [\".jpg\", \".jpeg\", \".png\", \".webp\"]\n", - "filelist = [i for i in input_folder.rglob(\"*\") if i.suffix in exts]\n", - "db = DatasetBuilder(Path(\"filedb.arrow\"))\n", - "##### add filters here ##### \n", - "if populate:\n", - " db.add_rules(rule_list)\n", - " if db.rules:\n", - " db.populate_df(map(str, filelist))\n", - "############################\n", - "assert category in db.df.columns, f\"selected category is not in {db.df.columns}\"\n", - "file_data = db.df.filter(pl.col(\"path\").is_in(list(map(str, filelist))))\n", - "\n", - "with tqdm(file_data.iter_rows(named=True), total=len(file_data)) as t:\n", - " for data in t:\n", - " pth = Path(data[\"path\"])\n", - " hash_ = str(data[category])\n", - " new_path: Path = (output / f\"{hash_}_{pth.stem}\").with_suffix(pth.suffix)\n", - " if not new_path.exists() or overwrite:\n", - " new_path.unlink()\n", - " new_path.symlink_to(pth)\n", - " t.set_description_str(hash_)\n" - ] } ], "metadata": {