From 23fea9cc1019ebb4266564521bbf0242bbb1005c Mon Sep 17 00:00:00 2001 From: zeptofine Date: Fri, 1 Dec 2023 13:17:17 -0500 Subject: [PATCH] new ruff rules --- pdm.lock | 22 +++--- pyproject.toml | 68 ++++++++++--------- src/imdataset_creator/__main__.py | 2 +- src/imdataset_creator/config_handler.py | 25 ++----- src/imdataset_creator/datarules/base_rules.py | 12 ++-- .../datarules/dataset_builder.py | 2 +- .../datarules/image_rules.py | 14 +++- src/imdataset_creator/file.py | 23 ++++--- src/imdataset_creator/gui/config_inputs.py | 6 +- src/imdataset_creator/gui/main_window.py | 2 - src/imdataset_creator/gui/minichecklist.py | 5 +- 11 files changed, 99 insertions(+), 82 deletions(-) mode change 100755 => 100644 src/imdataset_creator/gui/minichecklist.py diff --git a/pdm.lock b/pdm.lock index ac2656c..741ab4c 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default"] strategy = ["cross_platform"] lock_version = "4.4" -content_hash = "sha256:6862214b26a0af56c999897fb0f628a8f02bfc55039f4aae96fcca08c384f076" +content_hash = "sha256:14fa7067ab34f838f77cef73035579eaf59f3bc59e618544a7337634c104005e" [[package]] name = "bracex" @@ -76,6 +76,16 @@ files = [ {file = "ImageHash-4.3.1.tar.gz", hash = "sha256:7038d1b7f9e0585beb3dd8c0a956f02b95a346c0b5f24a9e8cc03ebadaf0aa70"}, ] +[[package]] +name = "imagesize" +version = "1.4.1" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +summary = "Getting image size from png/jpeg/jpeg2000/gif file" +files = [ + {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, + {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -143,16 +153,6 @@ files = [ {file = "opencv_python-4.8.1.78-cp37-abi3-win_amd64.whl", hash = "sha256:b983197f97cfa6fcb74e1da1802c7497a6f94ed561aba6980f1f33123f904956"}, ] -[[package]] -name = "packaging" -version = "23.2" -requires_python = ">=3.7" -summary = "Core utilities for Python packages" -files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, -] - [[package]] name = "pathvalidate" version = "3.2.0" diff --git a/pyproject.toml b/pyproject.toml index cb425bb..79c9ed9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,20 +4,20 @@ authors = [ {name = "zeptofine", email = "xpsychonauticonx@gmail.com"}, ] dependencies = [ - "ImageHash<5.0.0,>=4.3.1", - "Pillow<11.0.0,>=10.0.1", - "PySide6-Essentials<7.0.0,>=6.5.2", - "ffmpeg-python<1.0.0,>=0.2.0", - "imagesize<2.0.0,>=1.4.1", - "numpy<2.0.0,>=1.26.0", - "opencv-python<5.0.0.0,>=4.8.1.78", - "pathvalidate<4.0.0,>=3.2.0", - "polars<1.0.0,>=0.19.3", - "pyarrow<14.0.0,>=13.0.0", - "python-dateutil<3.0.0,>=2.8.2", - "rich<14.0.0,>=13.5.3", - "typer<1.0.0,>=0.9.0", - "wcmatch<9.0,>=8.5", + "ImageHash<5.0.0,>=4.3.1", + "Pillow<11.0.0,>=10.0.1", + "PySide6-Essentials<7.0.0,>=6.5.2", + "ffmpeg-python<1.0.0,>=0.2.0", + "imagesize<2.0.0,>=1.4.1", + "numpy<2.0.0,>=1.26.0", + "opencv-python<5.0.0.0,>=4.8.1.78", + "pathvalidate<4.0.0,>=3.2.0", + "polars<1.0.0,>=0.19.3", + "pyarrow<14.0.0,>=13.0.0", + "python-dateutil<3.0.0,>=2.8.2", + "rich<14.0.0,>=13.5.3", + "typer<1.0.0,>=0.9.0", + "wcmatch<9.0,>=8.5", ] description = "" license = {text = "GPL"} @@ -38,29 +38,33 @@ build-backend = "pdm.backend" requires = ["pdm-backend"] [tool.ruff] -extend-ignore = ["F401", "E501", "B905"] +extend-ignore = ["F401", "E501", "B905", "A003", "DTZ006", "DTZ005"] extend-select = [ - "A", - "ASYNC", - "UP", - "I", - "B", - "ICN", - "F", - "RET", - "SIM", - "NPY", - "PERF", - "RUF", - "PIE", - "C4", - "E", # pycodestyle + "A", # flake8-builtins + "ASYNC", # flake8-async + "UP", # pyupgrade + "I", # isort + "B", # flake8-bugbear + "ICN", # flake8-import-conventions + "F", # pyflakes + "RET", # flake8-return + "SIM", # flake8-simplify + "NPY", # NumPy-specific rules + "PERF", # perflint + "RUF", # Ruff-specific rules + "PIE", # flake8-pie + "DTZ", # flake8-datetimez + "C4", # flake8-comprehensions + "E", # Error + "EXE", # flake8-executable + "TCH", # flake8-type-checking "W", # pycodestyle "FA", # flake8-future-annotations "ISC", # flake8-implicit-str-concat - "G", # flake8-logging-format "Q", # flake8-quotes - "SLF", # flake8-self + "SLF", # flake8-self #Specific pylint rules # "PL", # Pylint + "PLR1711", # useless-return + "PLR1714", # repeated-equality-comparison ] fixable = ["ALL"] line-length = 120 diff --git a/src/imdataset_creator/__main__.py b/src/imdataset_creator/__main__.py index 3715c3c..bb9e948 100644 --- a/src/imdataset_creator/__main__.py +++ b/src/imdataset_creator/__main__.py @@ -1,6 +1,6 @@ import json import logging -from datetime import datetime +from datetime import UTC, datetime from multiprocessing import Pool, cpu_count, freeze_support from pathlib import Path from random import sample diff --git a/src/imdataset_creator/config_handler.py b/src/imdataset_creator/config_handler.py index ce39845..1b1b176 100644 --- a/src/imdataset_creator/config_handler.py +++ b/src/imdataset_creator/config_handler.py @@ -13,14 +13,12 @@ class ConfigHandler: def __init__(self, cfg: MainConfig): # generate `Input`s - self.inputs: list[Input] = [Input.from_cfg(folder["data"]) for folder in cfg["inputs"]] + self.inputs = [Input.from_cfg(folder["data"]) for folder in cfg["inputs"]] # generate `Producer`s - self.producers: list[Producer] = [ - Producer.all_producers[p["name"]].from_cfg(p["data"]) for p in cfg["producers"] - ] + self.producers = [Producer.all_producers[p["name"]].from_cfg(p["data"]) for p in cfg["producers"]] # generate `Rule`s - self.rules: list[Rule] = [Rule.all_rules[r["name"]].from_cfg(r["data"]) for r in cfg["rules"]] + self.rules = [Rule.all_rules[r["name"]].from_cfg(r["data"]) for r in cfg["rules"]] # generate test kwargs tkwargs = {} @@ -29,7 +27,7 @@ def __init__(self, cfg: MainConfig): tkwargs[name] = production.template # generate `Output`s - self.outputs: list[Output] = [Output.from_cfg(folder["data"], tkwargs) for folder in cfg["output"]] + self.outputs = [Output.from_cfg(folder["data"], tkwargs) for folder in cfg["output"]] @overload def gather_images(self, sort=True, reverse=False) -> Generator[tuple[Path, list[Path]], None, None]: @@ -39,20 +37,13 @@ def gather_images(self, sort=True, reverse=False) -> Generator[tuple[Path, list[ def gather_images(self, sort=False, reverse=False) -> Generator[tuple[Path, PathGenerator], None, None]: ... - def gather_images( - self, sort=False, reverse=False - ) -> Generator[tuple[Path, PathGenerator | list[Path]], None, None]: + def gather_images(self, sort=False, reverse=False): for input_ in self.inputs: gen = input_.run() if sort: yield ( input_.folder, - list( - map( - Path, - sorted(map(str, gen), key=alphanumeric_sort, reverse=reverse), - ) - ), + list(map(Path, sorted(map(str, gen), key=alphanumeric_sort, reverse=reverse))), ) else: yield input_.folder, gen @@ -71,7 +62,5 @@ def parse_files(self, files: Iterable[File]) -> Generator[FileScenario, None, No return (FileScenario(file, out_s) for file in files if (out_s := self.get_outputs(file))) def __repr__(self) -> str: - attrlist: list[str] = [ - f"{key}={val!r}" for key, val in vars(self).items() if all(k not in key for k in ("__",)) - ] + attrlist = [f"{key}={val!r}" for key, val in vars(self).items() if all(k not in key for k in ("__",))] return f"{self.__class__.__name__}({', '.join(attrlist)})" diff --git a/src/imdataset_creator/datarules/base_rules.py b/src/imdataset_creator/datarules/base_rules.py index 65b82c0..69bbd9c 100644 --- a/src/imdataset_creator/datarules/base_rules.py +++ b/src/imdataset_creator/datarules/base_rules.py @@ -6,18 +6,22 @@ from collections.abc import Callable, Generator, Iterable from dataclasses import dataclass from pathlib import Path -from types import MappingProxyType -from typing import Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar -import numpy as np import wcmatch.glob as wglob from pathvalidate import validate_filepath from polars import DataFrame, DataType, Expr, PolarsDataType from ..configs import Keyworded -from ..configs.configtypes import InputData, OutputData from ..file import File, SafeFormatter +if TYPE_CHECKING: + from types import MappingProxyType + + import numpy as np + + from ..configs.configtypes import InputData, OutputData + PartialDataFrame = DataFrame FullDataFrame = DataFrame DataTypeSchema = dict[str, DataType | type] diff --git a/src/imdataset_creator/datarules/dataset_builder.py b/src/imdataset_creator/datarules/dataset_builder.py index a49df56..4ef5415 100644 --- a/src/imdataset_creator/datarules/dataset_builder.py +++ b/src/imdataset_creator/datarules/dataset_builder.py @@ -277,7 +277,7 @@ def get_unfinished(self) -> LazyFrame: def get_unfinished_existing(self) -> LazyFrame: return self.get_unfinished().filter(pl.col("path").apply(os.path.exists)) - def filter(self, lst: Collection[str]) -> DataFrame: # noqa: A003 + def filter(self, lst: Collection[str]) -> DataFrame: if len(self.unready_rules): warnings.warn( f"{len(self.unready_rules)} filters are not initialized and will not be populated", diff --git a/src/imdataset_creator/datarules/image_rules.py b/src/imdataset_creator/datarules/image_rules.py index cf6773b..7de03c9 100644 --- a/src/imdataset_creator/datarules/image_rules.py +++ b/src/imdataset_creator/datarules/image_rules.py @@ -1,11 +1,10 @@ from __future__ import annotations import os -from collections.abc import Callable from enum import Enum, StrEnum from functools import cache from types import MappingProxyType -from typing import Literal, Self +from typing import TYPE_CHECKING, Literal, Self import imagehash import polars as pl @@ -24,6 +23,11 @@ combine_expr_conds, ) +if TYPE_CHECKING: + from collections.abc import Callable + + import numpy as np + def whash_db4(img) -> imagehash.ImageHash: return imagehash.whash(img, mode="db4") @@ -43,6 +47,12 @@ def get_hwc(pth): } +def get_hwc_from_np(img: np.ndarray): + h, w = img.shape[:2] + c = 1 if img.ndim == 2 else img.shape[2] + return h, w, c + + class ImShapeProducer(Producer): produces = MappingProxyType( { diff --git a/src/imdataset_creator/file.py b/src/imdataset_creator/file.py index e01000a..e9916fe 100644 --- a/src/imdataset_creator/file.py +++ b/src/imdataset_creator/file.py @@ -1,14 +1,17 @@ from __future__ import annotations import re -from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from string import Formatter -from typing import Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar from pathvalidate import sanitize_filename -from typing_extensions import SupportsIndex + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping, Sequence + + from typing_extensions import SupportsIndex class InvalidFormatError(Exception): @@ -41,8 +44,6 @@ def get_field(self, field_name: str, _: Sequence[Any], kwargs: Mapping[str, Any] escaped_split = re.compile(r"(?[^\?:]+)\?(?P(?:[^:])*):?(?P.*)$") # present?yes:no -replacement_fmt = re.compile(r"'(?P[^']+)'='(?P[^']*)'") def condition_format(pth: str, match: re.Match) -> str: @@ -56,12 +57,18 @@ def condition_format(pth: str, match: re.Match) -> str: return match.group("false") +def replacement_format(pth: str, match: re.Match) -> str: + """ + Inline replacement condition. 'from'='to' + replaces every instance of `from` to `to`. + """ + return pth.replace(match.group("from"), match.group("to")) + + class MalleablePath(str): mpath_format_conditions: ClassVar[dict[re.Pattern, Callable[[str, re.Match], str]]] = { re.compile(r"^(?P[^\?:]+)\?(?P(?:[^:!])*):?(?P.*)$"): (condition_format), - re.compile(r"^'(?P[^']+)'='(?P[^']*)'$"): ( # replaces to - lambda pth, m: pth.replace(m.group("from"), m.group("to")) - ), + re.compile(r"^'(?P[^']+)'='(?P[^']*)'$"): (replacement_format), } def __format__(self, format_spec: str): diff --git a/src/imdataset_creator/gui/config_inputs.py b/src/imdataset_creator/gui/config_inputs.py index 3e78dd4..a7e1fc9 100644 --- a/src/imdataset_creator/gui/config_inputs.py +++ b/src/imdataset_creator/gui/config_inputs.py @@ -3,7 +3,7 @@ import contextlib import functools from copy import deepcopy as objcopy -from typing import Generic, Iterable, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar from PySide6.QtCore import QRect, Qt, Signal, Slot from PySide6.QtGui import QAction, QMouseEvent @@ -26,6 +26,9 @@ from ..configs.keyworded import Keyworded from .settings_inputs import BaseInput, ItemSettings, SettingsBox, SettingsItem, apply_tooltip +if TYPE_CHECKING: + from collections.abc import Iterable + JSON_SERIALIZABLE = dict | list | tuple | str | int | float | bool | None @@ -317,7 +320,6 @@ def update_menu(self): self.add_box.setEnabled(True) elif self.add_box.isVisible(): self.add_box.setEnabled(False) - return def add_item_to_menu(self, item: ItemDeclaration): self.add_menu.addAction(item.title, lambda: self.initialize_item(item)) diff --git a/src/imdataset_creator/gui/main_window.py b/src/imdataset_creator/gui/main_window.py index 76f97ef..e817831 100644 --- a/src/imdataset_creator/gui/main_window.py +++ b/src/imdataset_creator/gui/main_window.py @@ -1,5 +1,3 @@ -#!/usr/bin/python - import json import logging import os diff --git a/src/imdataset_creator/gui/minichecklist.py b/src/imdataset_creator/gui/minichecklist.py old mode 100755 new mode 100644 index dad1986..7f7a15a --- a/src/imdataset_creator/gui/minichecklist.py +++ b/src/imdataset_creator/gui/minichecklist.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Collection +from typing import TYPE_CHECKING from PySide6.QtWidgets import ( QCheckBox, @@ -8,6 +8,9 @@ QGridLayout, ) +if TYPE_CHECKING: + from collections.abc import Collection + class MiniCheckList(QFrame): def __init__(self, items: Collection[str], *args, **kwargs):