From 2e59957f281d73afe3b993025238f1ae9afd872f Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 10:17:48 +0200 Subject: [PATCH 01/37] Modernize project configuration (removed setup.{cfg,py}) --- .flake8 | 8 +++++ ANNOUNCE.txt | 4 +++ MANIFEST.in | 7 ----- grin/__init__.py | 27 +---------------- pyproject.toml | 77 +++++++++++++++++++++++++++++++++++++++++++++++- setup.cfg | 67 ----------------------------------------- setup.py | 4 --- tox.ini | 4 +-- 8 files changed, 91 insertions(+), 107 deletions(-) create mode 100644 .flake8 delete mode 100644 MANIFEST.in delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..3481edb --- /dev/null +++ b/.flake8 @@ -0,0 +1,8 @@ +[flake8] +exclude = build,.git,.hg,.tox,.lib,__pycache__ +# E203 doesn't work for slicing +# W503 talks about operator formatting which is too opinionated. +ignore = E203, W503, C819 +max-complexity = 18 +max-line-length = 120 +select = B,C,E,F,W,T4,B9 diff --git a/ANNOUNCE.txt b/ANNOUNCE.txt index fe2f530..74ea928 100644 --- a/ANNOUNCE.txt +++ b/ANNOUNCE.txt @@ -4,6 +4,10 @@ files, primarily source code. Download: http://pypi.python.org/pypi/grin3 License: BSD +grin dev. + + * modernize project setup/configuration + grin 2.6.1 is a bug-fix release. * detect bad gzip files (thanks Charles Cazabon) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index adcbc9e..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,7 +0,0 @@ -include ANNOUNCE.txt -include LICENSE.txt -include MANIFEST.in -include README.rst -include THANKS.txt -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] diff --git a/grin/__init__.py b/grin/__init__.py index 131a771..99cafb9 100644 --- a/grin/__init__.py +++ b/grin/__init__.py @@ -32,32 +32,7 @@ from .recognizer import GZIP_MAGIC, FileRecognizer, get_recognizer # noqa: F401 from .utils import get_line_offsets, get_regex, is_binary_string # noqa: F401 - -def get_version(version=None): - "Returns a PEP 386-compliant version number from VERSION." - assert len(version) == 5 - assert version[3] in ("alpha", "beta", "rc", "final") - - # Now build the two parts of the version number: - # main = X.Y[.Z] - # sub = .devN - for pre-alpha releases - # | {a|b|c}N - for alpha, beta and rc releases - - parts = 2 if version[2] == 0 else 3 - main = ".".join(str(x) for x in version[:parts]) - - sub = "" - - if version[3] != "final": - mapping = {"alpha": "a", "beta": "b", "rc": "c"} - sub = mapping[version[3]] + str(version[4]) - - return str(main + sub) - - -# Constants -VERSION = (2, 6, 1, "final", 0) -__version__ = get_version(VERSION) +__version__ = "2.6.1" __author__ = "Robert Kern" __author_email__ = "robert.kern@enthought.com" __maintainer__ = "Raffaele Salmaso" diff --git a/pyproject.toml b/pyproject.toml index 6cd42c0..7566e1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,69 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "grin3" +dynamic = ["version"] +description = "A grep program configured the way I like it. (python3 port)" +readme = "README.rst" +requires-python = ">=3.5" +license = { file = "LICENSE.txt" } +authors = [{ name = "Robert Kern", email = "robert.kern@enthought.com" }] +maintainers = [{ name = "Raffaele Salmaso", email = "raffaele@salmaso.org" }] +classifiers = [ + "License :: OSI Approved :: BSD License", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Utilities", +] + +[project.urls] +Github = "https://github.com/rsalmaso/grin3" +Gitlab = "https://gitlab.com/rsalmaso/grin3" + +[project.scripts] +grin = "grin.grin:main" +grind = "grin.grind:main" + +[dependency-groups] +dev = ["build", "black", "flake8", "isort", "mypy", "twine"] + +[tool.hatch.version] +path = "grin/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["grin"] +exclude = ["**/*.so", "**/*.py~"] + +[tool.hatch.build.targets.sdist] +include = [ + "/grin", + "/tests", + "/examples", + "/README.rst", + "/LICENSE.txt", + "/ANNOUNCE.txt", + "/THANKS.txt", + "/.flake8", + "/tox.ini", + "/pyproject.toml", +] +exclude = ["**/*.so", "**/*.py~", "**/__pycache__"] + [tool.black] exclude = ''' /( @@ -6,5 +72,14 @@ exclude = ''' ''' include = '\.pyi?$' line-length = 120 -safe = true target-version = ['py35'] + +[tool.isort] +combine_as_imports = true +default_section = "THIRDPARTY" +force_grid_wrap = 0 +include_trailing_comma = true +indent = 4 +line_length = 120 +multi_line_output = 3 +use_parentheses = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b2c14d3..0000000 --- a/setup.cfg +++ /dev/null @@ -1,67 +0,0 @@ -[metadata] -name = grin3 -version = attr: grin.__version__ -url = https://pypi.org/project/grin3/ -project_urls = - Github = https://github.com/rsalmaso/grin3 - Gitlab = https://gitlab.com/rsalmaso/grin3 -author = Robert Kern -author_email = robert.kern@enthought.com -maintainer = Raffaele Salmaso -maintainer_email = raffaele@salmaso.org -description = A grep program configured the way I like it. (python3 port) -long_description = file: README.rst -long_description_content_type = text/x-rst -classifiers = - License :: OSI Approved :: BSD License - Development Status :: 5 - Production/Stable - Environment :: Console - Intended Audience :: Developers - Operating System :: OS Independent - Programming Language :: Python - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: Implementation :: CPython - Programming Language :: Python :: Implementation :: PyPy - Topic :: Utilities - -[options] -python_requires = >=3.5 -packages = - grin - -[options.entry_points] -console_scripts = - grin = grin.grin:main - grind = grin.grind:main - -[wheel] -universal = 0 - -[bdist_wheel] -universal = 0 - -[flake8] -exclude = build,.git,.hg,.tox,.lib,__pycache__ -# E203 doesn't work for slicing -# W503 talks about operator formatting which is too opinionated. -ignore = E203, W503, C819 -max-complexity = 18 -max-line-length = 120 -select = B,C,E,F,W,T4,B9 - -[isort] -combine_as_imports = True -default_section = THIRDPARTY -force_grid_wrap = 0 -include_trailing_comma = True -indent = 4 -line_length = 120 -multi_line_output = 3 -use_parentheses = True diff --git a/setup.py b/setup.py deleted file mode 100644 index b024da8..0000000 --- a/setup.py +++ /dev/null @@ -1,4 +0,0 @@ -from setuptools import setup - - -setup() diff --git a/tox.ini b/tox.ini index 5099108..4acc5f5 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ changedir = {toxinidir} skip_install = True deps = black -commands = black --check --diff examples/ grin/ tests/ setup.py +commands = black --check --diff examples/ grin/ tests/ [testenv:flake8] changedir = {toxinidir} @@ -24,4 +24,4 @@ deps = flake8-commas flake8-isort flake8-quotes -commands = flake8 --show-source examples/ grin/ tests/ setup.py +commands = flake8 --show-source examples/ grin/ tests/ From 5289498c6dd0a654a2e823fe411edfffbf33d760 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 10:18:16 +0200 Subject: [PATCH 02/37] Drop vagrant support --- .gitignore | 1 - .hgignore | 1 - ANNOUNCE.txt | 1 + README.rst | 17 +++++------------ Vagrantfile | 53 ---------------------------------------------------- 5 files changed, 6 insertions(+), 67 deletions(-) delete mode 100644 Vagrantfile diff --git a/.gitignore b/.gitignore index ba8b7aa..c4adb75 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .tox/ -.vagrant/ build/ dist/ grin3.egg-info/ diff --git a/.hgignore b/.hgignore index e424ca0..be5e454 100644 --- a/.hgignore +++ b/.hgignore @@ -1,6 +1,5 @@ syntax: regexp .tox/ -.vagrant/ build/ dist/ grin3.egg-info/ diff --git a/ANNOUNCE.txt b/ANNOUNCE.txt index 74ea928..0ef1d39 100644 --- a/ANNOUNCE.txt +++ b/ANNOUNCE.txt @@ -7,6 +7,7 @@ files, primarily source code. grin dev. * modernize project setup/configuration + * drop vagrant support grin 2.6.1 is a bug-fix release. diff --git a/README.rst b/README.rst index 9d46786..fd2ae79 100644 --- a/README.rst +++ b/README.rst @@ -272,21 +272,14 @@ Running unittests:: tox ~~~ -Run all tests into dedicated virtualenvs, and check code style. +Run all tests in dedicated virtualenvs, and check code style. The ``tox-uv`` +plugin provisions the interpreters (including the Python 3.15 beta) via uv:: -vagrant -~~~~~~~ + $ uv run --group dev tox -There is a simple vagrant config to install a preconfigured system with everything needed to run tests locally. +To run the suite against a single interpreter, e.g. the 3.15 beta:: -Just run:: - - $ vagrant up - # and wait for provisioning - $ vagrant ssh - $ tox -e ALL - -It will run all tests suite on all supported python versions + $ uv run --python 3.15 -m unittest discover tests To Do diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index aae61f5..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,53 +0,0 @@ -# -*- mode: ruby -*- -# vim: set ft=ruby ts=2 sw=2 : - -require "fileutils" - -VAGRANTFILE_API_VERSION = "2" - -$script = <<-SCRIPT -export DEBIAN_FRONTEND=noninteractive -add-apt-repository --no-update --yes ppa:deadsnakes/ppa -add-apt-repository --no-update --yes ppa:git-core/ppa -add-apt-repository --no-update --yes ppa:pypy/ppa - -apt update -apt dist-upgrade -y - -# python-software-properties -apt install --yes \ - git \ - python3.5 python3.5-dev python3.5-venv \ - python3.6 python3.6-dev python3.6-venv \ - python3.7 python3.7-dev python3.7-venv \ - python3.8 python3.8-dev python3.8-venv \ - python3.9 python3.9-dev python3.9-venv \ - pypy3 pypy3-dev \ - python3-pip - -mkdir -p /etc/mercurial -echo "[extensions]\nevolve =\ntopic =\n" >> /etc/mercurial/hgrc -python3 -m pip install mercurial -python3 -m pip install hg-evolve -python3 -m pip install tox - -# Fix locale to allow saving unicoded filenames -echo 'LANG=en_US.UTF-8' > /etc/default/locale - -# Start in project dir by default -echo "\n\ncd /vagrant" >> /home/vagrant/.bashrc -SCRIPT - -# ensure log directory exists -$vm_log_dir = File.join(Dir.pwd, ".vagrant", "log") -FileUtils.mkdir_p $vm_log_dir - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/focal64" - config.vm.provision :shell, inline: $script - - config.vm.provider "virtualbox" do |vb| - vb.customize ["modifyvm", :id, "--memory", 2048] - vb.customize ["modifyvm", :id, "--uartmode1", "file", File.join($vm_log_dir, "focal.log")] - end -end From 3da5cc8a6bbe4640a7b4fd7eaa481701b9d89e08 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 10:23:49 +0200 Subject: [PATCH 03/37] Python minimum supported version set to 3.10 --- ANNOUNCE.txt | 1 + pyproject.toml | 15 +++++++-------- tox.ini | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ANNOUNCE.txt b/ANNOUNCE.txt index 0ef1d39..88ab47e 100644 --- a/ANNOUNCE.txt +++ b/ANNOUNCE.txt @@ -8,6 +8,7 @@ grin dev. * modernize project setup/configuration * drop vagrant support + * require Python >= 3.10 grin 2.6.1 is a bug-fix release. diff --git a/pyproject.toml b/pyproject.toml index 7566e1b..b6ff3a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "grin3" dynamic = ["version"] description = "A grep program configured the way I like it. (python3 port)" readme = "README.rst" -requires-python = ">=3.5" +requires-python = ">=3.10" license = { file = "LICENSE.txt" } authors = [{ name = "Robert Kern", email = "robert.kern@enthought.com" }] maintainers = [{ name = "Raffaele Salmaso", email = "raffaele@salmaso.org" }] @@ -19,13 +19,12 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Utilities", @@ -40,7 +39,7 @@ grin = "grin.grin:main" grind = "grin.grind:main" [dependency-groups] -dev = ["build", "black", "flake8", "isort", "mypy", "twine"] +dev = ["build", "black", "flake8", "isort", "mypy", "tox", "tox-uv", "twine"] [tool.hatch.version] path = "grin/__init__.py" @@ -72,7 +71,7 @@ exclude = ''' ''' include = '\.pyi?$' line-length = 120 -target-version = ['py35'] +target-version = ['py310'] [tool.isort] combine_as_imports = true diff --git a/tox.ini b/tox.ini index 4acc5f5..d6d0fec 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] -envlist = py35, py36, py37, py38, py39, py310, py311, pypy3 +requires = tox-uv>=1 +envlist = py310, py311, py312, py313, py314, py315, pypy3 skipsdist = True skip_missing_interpreters = True From 55be49fbf971993810c726364bc96a08aa054f2e Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 10:42:37 +0200 Subject: [PATCH 04/37] Converted .txt and .rst files to .md --- ANNOUNCE.txt | 112 ------------- CHANGELOG.md | 105 ++++++++++++ LICENSE.txt => LICENSE.md | 0 README.md | 325 ++++++++++++++++++++++++++++++++++++++ README.rst | 288 --------------------------------- THANKS.txt => THANKS.md | 3 +- examples/README.md | 7 + examples/README.txt | 10 -- pyproject.toml | 12 +- 9 files changed, 444 insertions(+), 418 deletions(-) delete mode 100644 ANNOUNCE.txt create mode 100644 CHANGELOG.md rename LICENSE.txt => LICENSE.md (100%) create mode 100644 README.md delete mode 100644 README.rst rename THANKS.txt => THANKS.md (95%) create mode 100644 examples/README.md delete mode 100644 examples/README.txt diff --git a/ANNOUNCE.txt b/ANNOUNCE.txt deleted file mode 100644 index 88ab47e..0000000 --- a/ANNOUNCE.txt +++ /dev/null @@ -1,112 +0,0 @@ -grin is a grep-like tool for recursively searching through text -files, primarily source code. - - Download: http://pypi.python.org/pypi/grin3 - License: BSD - -grin dev. - - * modernize project setup/configuration - * drop vagrant support - * require Python >= 3.10 - -grin 2.6.1 is a bug-fix release. - - * detect bad gzip files (thanks Charles Cazabon) - -grin 2.6.0 is a feature release. - - * add support for Python 3.10 - * add support for Python 3.11 - -grin 2.5.0 is a feature release. - - * add -a/--ascii option (thanks Charles Cazabon) - * add -w/--word-regexp option (thanks Charles Cazabon) - -grin 2.4.1 is a bug-fix release. - - * minor code tweaks - -grin 2.4.0 is a feature release. - - grin 2.4.X will be a deprecation release(s) where many options and code will be deprecated and - hidden by default (their usage will be under --help-verbose cli flag) - - * remove deprecated grin.default_options, use grin.Options() instead - * deprecate --[no-]skip-backup-files options, use --skip-exts instead - * deprecate --with-filename option (as it is the default option) - * deprecate --without-filename option as an alias for --no-filename option - * deprecate --line-number option (as it is the default option) - * deprecate --skip-hidden-dirs option (as it is the default option) - * deprecate --no-follow option (as it is the default option) - * deprecate --skip-hidden-files option (as it is the default option) - * add --color {auto,no,always} option - * deprecate --no-color, --force-color and --use-color options, use --color option instead - -grin 2.3.1 is a bug-fix release. - - * correct default Options.re_flags value - -grin 2.3.0 is a feature release. - - * add -F/--fixed-string option (thanks Charles Cazabon) - -grin 2.2.1 is a bug-fix release. - - * fix missing grind argument - -grin 2.2.0 is a feature release. - - * support for python3.9 - * support for pypy3 - * add -x, --encoding option (defaults to terminal output encoding) - -grin 2.1.0 is a feature release. - - * speedup file access - * default_options is now an object - * exclude by default node_modules,.class,target directories - * support for stable python3.8 - -grin 2.0.2 is a bug-fix release. - - * fix encoding - -grin 2.0.1 is a bug-fix release. - - * fix setup and wheel build - * update docs - -grin 2.0.0 is a bug-fix release. - - * refactor code into a module - * refactor test code from nose to unittest - * add Vagrant configuration - * add tox support - * adopt black formatter - * removed TypeError - * drop python3.4 support - * added support for python3.7 and python3.8 (dev) - -grin 1.2.3 is a bug-fix release. - - * removed UnicodeDecodeError - -grin 1.2.2 is a python3 only port. - - * python3 support added. - * python2 support dropped. - -grin 1.2.1 is a bug-fix release. - - * Windows defaults to not coloring the output. (Paul Pelzl) - report. - - * Fix the reading of gzip files. (Brandon Craig Rhodes) - - * Quit gracefully when piping to a program that exits prematurely. - (Brandon Craig Rhodes) - - * Sort the basenames of files during traversal in order to maintain - a repeatable ordering. (Brandon Craig Rhodes) diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c162502 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,105 @@ +# Changelog + +## dev + +* modernize project setup/configuration +* drop vagrant support +* require Python >= 3.10 + +## 2.6.1 + +* detect bad gzip files (thanks Charles Cazabon) + +## 2.6.0 + +* add support for Python 3.10 +* add support for Python 3.11 + +## 2.5.0 + +* add -a/--ascii option (thanks Charles Cazabon) +* add -w/--word-regexp option (thanks Charles Cazabon) + +## 2.4.1 + +* minor code tweaks + +## 2.4.0 + +grin 2.4.X will be a deprecation release(s) where many options and code will be +deprecated and hidden by default (their usage will be under --help-verbose cli +flag) + +* remove deprecated grin.default_options, use grin.Options() instead +* deprecate --[no-]skip-backup-files options, use --skip-exts instead +* deprecate --with-filename option (as it is the default option) +* deprecate --without-filename option as an alias for --no-filename option +* deprecate --line-number option (as it is the default option) +* deprecate --skip-hidden-dirs option (as it is the default option) +* deprecate --no-follow option (as it is the default option) +* deprecate --skip-hidden-files option (as it is the default option) +* add --color {auto,no,always} option +* deprecate --no-color, --force-color and --use-color options, use --color option instead + +## 2.3.1 + +* correct default Options.re_flags value + +## 2.3.0 + +* add -F/--fixed-string option (thanks Charles Cazabon) + +## 2.2.1 + +* fix missing grind argument + +## 2.2.0 + +* support for python3.9 +* support for pypy3 +* add -x, --encoding option (defaults to terminal output encoding) + +## 2.1.0 + +* speedup file access +* default_options is now an object +* exclude by default node_modules,.class,target directories +* support for stable python3.8 + +## 2.0.2 + +* fix encoding + +## 2.0.1 + +* fix setup and wheel build +* update docs + +## 2.0.0 + +* refactor code into a module +* refactor test code from nose to unittest +* add Vagrant configuration +* add tox support +* adopt black formatter +* removed TypeError +* drop python3.4 support +* added support for python3.7 and python3.8 (dev) + +## 1.2.3 + +* removed UnicodeDecodeError + +## 1.2.2 + +python3 only port. + +* python3 support added. +* python2 support dropped. + +## 1.2.1 + +* Windows defaults to not coloring the output. (Paul Pelzl) +* Fix the reading of gzip files. (Brandon Craig Rhodes) +* Quit gracefully when piping to a program that exits prematurely. (Brandon Craig Rhodes) +* Sort the basenames of files during traversal in order to maintain a repeatable ordering. (Brandon Craig Rhodes) diff --git a/LICENSE.txt b/LICENSE.md similarity index 100% rename from LICENSE.txt rename to LICENSE.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..c248255 --- /dev/null +++ b/README.md @@ -0,0 +1,325 @@ +# grin + +*this is a port of original grin to python3* + +I wrote grin to help me search directories full of source code. The venerable +GNU [grep] and [find] are great tools, but they fall just a little short for my +normal use cases. + +The main problem I had with GNU [grep] is that I had no way to exclude certain +directories that I knew had nothing of interest for me, like .svn/, CVS/ and +build/. The results from those directories obscured the results I was actually +interested in. There are tools like [ack], which skip these directories, but +[ack] also only grepped files with extensions that it knew about. Furthermore, +it had not implemented the context lines feature, which I had grown accustomed +to. Recent development has added these features, but I had already released grin +by the time I found out. + +One can construct a GNU [find] command that will exclude .svn/ and the rest, but +the only reliable way I am aware of runs [grep] on each file independently. The +startup cost of invoking many separate [grep] processes is relatively large. + +Also, I was bored. It seems to be catching. Perl has [ack], Ruby has [rak], and +now Python has grin. + +I wrote grin to get exactly the features I wanted: + +* Recurse directories by default. +* Do not go into directories with specified names. +* Do not search files with specified extensions. +* Be able to show context lines before and after matched lines. +* Python regex syntax (one can quibble as to whether this is a feature or my + laziness for using the regex library provided with my implementation + language, but as a Python programmer, this is the syntax I am most familiar + with). +* Unless suppressed via a command line option, display the filename regardless + of the number of files. +* Accept a file (or stdin) with a list of newline-separated filenames. This + allows one to use [find] to feed grin a list of filenames which might have + embedded spaces quite easily. +* Grep through gzipped text files. +* Be useful as a library to build custom tools quickly. + +I have also exposed the directory recursion logic as the command-line tool +"grind" in homage to [find]. It will recurse through directories matching a glob +pattern to file names and printing out the matches. It shares the directory and +file extension skipping settings that grin uses. + +For configuration, you can specify the environment variables GRIN_ARGS and +GRIND_ARGS. These should just contain command-line options of their respective +programs. These will be prepended to the command-line arguments actually given. +Options given later will override options given earlier, so all options +explicitly in the command-line will override those in the environment variable. +For example, if I want to default to two lines of context and no skipped +directories, I would have this line in my bashrc: + +```bash +export GRIN_ARGS="-C 2 --no-skip-dirs" +``` + +## Installation + +Install using [pip]: + +```bash +$ python3 -m pip install grin3 +``` + +Running the tests: + +``` +$ python3 -m unittest discover tests +---------------------------------------------------------------------- +Ran 50 tests in 0.010s + +OK +``` + +There is one little tweak to the installation that you may want to consider. By +default, setuptools installs scripts indirectly; the scripts installed to +$prefix/bin or Python3x\Scripts use setuptools' pkg_resources module to load +the exact version of grin egg that installed the script, then runs the script's +main() function. This is not usually a bad feature, but it can add substantial +startup overhead for a small command-line utility like grin. If you want the +response of grin to be snappier, I recommend installing custom scripts that just +import the grin module and run the appropriate main() function. See the files +examples/grin and examples/grind for examples. + +## Using grin + +To recursively search the current directory for a regex: + +```bash +$ grin some_regex +``` + +To search an explicit set of files: + +```bash +$ grin some_regex file1.txt path/to/file2.txt +``` + +To recursively search an explicit set of directories: + +```bash +$ grin some_regex dir1/ dir2/ +``` + +To search data piped to stdin: + +```bash +$ cat somefile | grin some_regex - +``` + +To make the regex case-insensitive: + +```bash +$ grin -i some_regex +``` + +To search for a fixed-string pattern containing regex metacharacters +without having to manually escape them: + +```bash +$ grin -F '[string_with_regex_metachars(' +``` + +By default, grin uses Unicode definitions of digits (\d,\D), word boundaries +(\b,\B), whitespace (\s,\S), etc. To force ASCII-only interpretation of these +character classes: + +```bash +$ grin -a pattern +``` + +To search for a pattern that begins and ends on a word boundary (no partial-word +matches): + +```bash +$ grin -w myword +``` + +To output 2 lines of context before, after, or both before and after the +matches: + +```bash +$ grin -B 2 some_regex +$ grin -A 2 some_regex +$ grin -C 2 some_regex +``` + +To only search Python .py files: + +```bash +$ grin -I "*.py" some_regex +``` + +To suppress the line numbers which are printed by default: + +```bash +$ grin -N some_regex +``` + +To just show the names of the files that contain matches rather than the matches +themselves: + +```bash +$ grin -l some_regex +``` + +To suppress the use of color highlighting: + +```bash +# Note that grin does its best to only use color when it detects that it is +# outputting to a real terminal. If the output is being piped to a file or +# a pager, then no color will be used. +$ grin --color no some_regex +``` + +To force the use of color highlighting when piping the output to something that +is capable of understanding ANSI color escapes: + +```bash +$ grin --color always some_regex | less -R +``` + +To avoid recursing into directories named either CVS or RCS: + +```bash +$ grin -d CVS,RCS some_regex +``` + +By default grin skips a large number of files. To suppress all of this behavior +and search everything: + +```bash +$ grin -sbSDE some_regex +``` + +To search for files newer than some_file.txt: + +```bash +# If no subdirectory or file in the list contains whitespace: +$ grin some_regex `find . -newer some_file.txt` + +# If a subdirectory or file in the list may contain whitespace: +$ find . -newer some_file.txt | grin -f - some_regex +``` + +## Using grind + +To find files matching the glob "foo\*.py" in this directory or any subdirectory +using same the default rules as grin: + +```bash +$ grind "foo*.py" +``` + +To suppress all of the default rules and not skip any files or directories while +searching: + +```bash +$ grind -sbSDE "foo*.py" +``` + +To find all files that are not skipped by the default rules: + +```bash +$ grind +``` + +To start the search in a particular set of directories instead of the current +one (not the -- separator): + +```bash +$ grind --dirs thisdir that/dir -- "foo*.py" +``` + +## Using grin as a Library + +One of the goals I had when writing grin was to be able to use it as a library +to write custom tools. You can see one example that I quickly hacked up in +examples/grinimports.py . It reuses almost all of grin's infrastructure, except +that it preprocesses Python files to extract and normalize just the import +statements. This lets you conveniently and robustly search for import +statements. Look at "grinimports.py --help" for more information. + +examples/grinpython.py allows you to search through Python files and specify +whether you want to search through actual Python code, comments or string +literals in any combination. For example: + +``` +$ grinpython.py -i --strings grep grin.py +grin.py: + 188 : """ Grep a single file for a regex by iterating over the lines in a file. + 292 : """ Do a full grep. +... + +$ grinpython.py -i --comments grep grin.py +grin.py: + 979 : # something we want to grep. + +$ grinpython.py -i --python-code grep grin.py +grin.py: + 187 : class GrepText: + 291 : def do_grep(self, fp): +... +``` + +Similarly, it should be straightforward to write small tools like this which +extract and search text metadata from binary files. + +## Development, bugs and such + +The source code is managed with mercurial and was hosted on bitbucket, +now it is waiting for https://foss.heptapod.net/ repository, +and it is available via git mirrors on + +* https://github.com/rsalmaso/grin3 +* https://gitlab.com/rsalmaso/grin3 + +You are free to open a PR/MR/Bug where you want. + +The source code is formatted with [black], [isort] and [flake8]. + +## Testing + +### tests + +Running unittests: + +``` +$ python3 -m unittest discover tests +---------------------------------------------------------------------- +Ran 50 tests in 0.010s + +OK +``` + +### tox + +Run all tests in dedicated virtualenvs, and check code style. The `tox-uv` +plugin provisions the interpreters (including the Python 3.15 beta) via uv: + +```bash +$ uv run --group dev tox +``` + +To run the suite against a single interpreter, e.g. the 3.15 beta: + +```bash +$ uv run --python 3.15 -m unittest discover tests +``` + +## To Do + +* Figure out the story for grepping UTF-8, UTF-16 and UTF-32 Unicode text files. + +[grep]: http://www.gnu.org/software/grep/ +[ack]: http://search.cpan.org/~petdance/ack/ack +[rak]: http://rak.rubyforge.org/ +[find]: http://www.gnu.org/software/findutils/ +[pip]: https://pip.pypa.io/en/stable/ +[black]: https://pypi.org/project/black/ +[isort]: https://pypi.org/project/isort/ +[flake8]: https://pypi.org/project/flake8/ diff --git a/README.rst b/README.rst deleted file mode 100644 index fd2ae79..0000000 --- a/README.rst +++ /dev/null @@ -1,288 +0,0 @@ -==== -grin -==== - -[this is a port of original grin to python3] - -I wrote grin to help me search directories full of source code. The venerable -GNU grep_ and find_ are great tools, but they fall just a little short for my -normal use cases. - -The main problem I had with GNU grep_ is that I had no way to exclude certain -directories that I knew had nothing of interest for me, like .svn/, CVS/ and -build/. The results from those directories obscured the results I was actually -interested in. There are tools like ack_, which skip these directories, but ack_ -also only grepped files with extensions that it knew about. Furthermore, it had -not implemented the context lines feature, which I had grown accustomed to. -Recent development has added these features, but I had already released grin by -the time I found out. - -One can construct a GNU find_ command that will exclude .svn/ and the rest, but -the only reliable way I am aware of runs grep_ on each file independently. The -startup cost of invoking many separate grep_ processes is relatively large. - -Also, I was bored. It seems to be catching. Perl has ack_, Ruby has rak_, and -now Python has grin. - -I wrote grin to get exactly the features I wanted: - - * Recurse directories by default. - * Do not go into directories with specified names. - * Do not search files with specified extensions. - * Be able to show context lines before and after matched lines. - * Python regex syntax (one can quibble as to whether this is a feature or my - laziness for using the regex library provided with my implementation - language, but as a Python programmer, this is the syntax I am most familiar - with). - * Unless suppressed via a command line option, display the filename regardless - of the number of files. - * Accept a file (or stdin) with a list of newline-separated filenames. This - allows one to use find_ to feed grin a list of filenames which might have - embedded spaces quite easily. - * Grep through gzipped text files. - * Be useful as a library to build custom tools quickly. - -I have also exposed the directory recursion logic as the command-line tool -"grind" in homage to find_. It will recurse through directories matching a glob -pattern to file names and printing out the matches. It shares the directory and -file extension skipping settings that grin uses. - -For configuration, you can specify the environment variables GRIN_ARGS and -GRIND_ARGS. These should just contain command-line options of their respective -programs. These will be prepended to the command-line arguments actually given. -Options given later will override options given earlier, so all options -explicitly in the command-line will override those in the environment variable. -For example, if I want to default to two lines of context and no skipped -directories, I would have this line in my bashrc:: - - export GRIN_ARGS="-C 2 --no-skip-dirs" - -.. _grep : http://www.gnu.org/software/grep/ -.. _ack : http://search.cpan.org/~petdance/ack/ack -.. _rak: http://rak.rubyforge.org/ -.. _find : http://www.gnu.org/software/findutils/ - - -Installation ------------- - -Install using pip_:: - - $ python3 -m pip install grin3 - -Running the tests:: - - $ python3 -m unittest discover tests - ---------------------------------------------------------------------- - Ran 24 tests in 0.010s - - OK - - -There is one little tweak to the installation that you may want to consider. By -default, setuptools installs scripts indirectly; the scripts installed to -$prefix/bin or Python3x\Scripts use setuptools' pkg_resources module to load -the exact version of grin egg that installed the script, then runs the script's -main() function. This is not usually a bad feature, but it can add substantial -startup overhead for a small command-line utility like grin. If you want the -response of grin to be snappier, I recommend installing custom scripts that just -import the grin module and run the appropriate main() function. See the files -examples/grin and examples/grind for examples. - -.. _pip : https://pip.pypa.io/en/stable/ - - -Using grin ----------- - -To recursively search the current directory for a regex:: - - $ grin some_regex - -To search an explicit set of files:: - - $ grin some_regex file1.txt path/to/file2.txt - -To recursively search an explicit set of directories:: - - $ grin some_regex dir1/ dir2/ - -To search data piped to stdin:: - - $ cat somefile | grin some_regex - - -To make the regex case-insensitive:: - - $ grin -i some_regex - -To search for a fixed-string pattern containing regex metacharacters -without having to manually escape them:: - - $ grin -F '[string_with_regex_metachars(' - -By default, grin uses Unicode definitions of digits (\d,\D), word boundaries (\b,\B), -whitespace (\s,\S), etc. To force ASCII-only interpretation of these character classes: - - $ grin -a pattern - -To search for a pattern that begins and ends on a word boundary (no partial- -word matches): - - $ grin -w myword - -To output 2 lines of context before, after, or both before and after the -matches:: - - $ grin -B 2 some_regex - $ grin -A 2 some_regex - $ grin -C 2 some_regex - -To only search Python .py files:: - - $ grin -I "*.py" some_regex - -To suppress the line numbers which are printed by default:: - - $ grin -N some_regex - -To just show the names of the files that contain matches rather than the matches -themselves:: - - $ grin -l some_regex - -To suppress the use of color highlighting:: - - # Note that grin does its best to only use color when it detects that it is - # outputting to a real terminal. If the output is being piped to a file or - # a pager, then no color will be used. - $ grin --color no some_regex - -To force the use of color highlighting when piping the output to something that -is capable of understanding ANSI color escapes:: - - $ grin --color always some_regex | less -R - -To avoid recursing into directories named either CVS or RCS:: - - $ grin -d CVS,RCS some_regex - -By default grin skips a large number of files. To suppress all of this behavior -and search everything:: - - $ grin -sbSDE some_regex - -To search for files newer than some_file.txt:: - - # If no subdirectory or file in the list contains whitespace: - $ grin some_regex `find . -newer some_file.txt` - - # If a subdirectory or file in the list may contain whitespace: - $ find . -newer some_file.txt | grin -f - some_regex - - -Using grind ------------ - -To find files matching the glob "foo*.py" in this directory or any subdirectory -using same the default rules as grin:: - - $ grind "foo*.py" - -To suppress all of the default rules and not skip any files or directories while -searching:: - - $ grind -sbSDE "foo*.py" - -To find all files that are not skipped by the default rules:: - - $ grind - -To start the search in a particular set of directories instead of the current -one (not the -- separator):: - - $ grind --dirs thisdir that/dir -- "foo*.py" - - -Using grin as a Library ------------------------ - -One of the goals I had when writing grin was to be able to use it as a library -to write custom tools. You can see one example that I quickly hacked up in -examples/grinimports.py . It reuses almost all of grin's infrastructure, except -that it preprocesses Python files to extract and normalize just the import -statements. This lets you conveniently and robustly search for import -statements. Look at "grinimports.py --help" for more information. - -examples/grinpython.py allows you to search through Python files and specify whether you want to search through actual Python code, comments or string literals in any combination. For example:: - - $ grinpython.py -i --strings grep grin.py - grin.py: - 188 : """ Grep a single file for a regex by iterating over the lines in a file. - 292 : """ Do a full grep. - ... - - $ grinpython.py -i --comments grep grin.py - grin.py: - 979 : # something we want to grep. - - $ grinpython.py -i --python-code grep grin.py - grin.py: - 187 : class GrepText: - 291 : def do_grep(self, fp): - ... - -Similarly, it should be straightforward to write small tools like this which -extract and search text metadata from binary files. - - -Development, bugs and such --------------------------- - -The source code is managed with mercurial and was hosted on bitbucket, -now it is waiting for https://foss.heptapod.net/ repository, -and it is available via git mirrors on - - https://github.com/rsalmaso/grin3 - - https://gitlab.com/rsalmaso/grin3 - -You are free to open a PR/MR/Bug where you want. - -The source code is formatted with `black_`, `isort_` and `flake8_` - -.. _black : https://pypi.org/project/black/ -.. _isort : https://pypi.org/project/isort/ -.. _flake8 : https://pypi.org/project/flake8/ - - -Testing -------- - -tests -~~~~~ - -Running unittests:: - - $ python3 -m unittest discover tests - ---------------------------------------------------------------------- - Ran 24 tests in 0.010s - - OK - -tox -~~~ - -Run all tests in dedicated virtualenvs, and check code style. The ``tox-uv`` -plugin provisions the interpreters (including the Python 3.15 beta) via uv:: - - $ uv run --group dev tox - -To run the suite against a single interpreter, e.g. the 3.15 beta:: - - $ uv run --python 3.15 -m unittest discover tests - - -To Do ------ - -* Figure out the story for grepping UTF-8, UTF-16 and UTF-32 Unicode text files. diff --git a/THANKS.txt b/THANKS.md similarity index 95% rename from THANKS.txt rename to THANKS.md index 5d791af..a46bbf3 100644 --- a/THANKS.txt +++ b/THANKS.md @@ -1,5 +1,4 @@ -Acknowledgements ----------------- +# Acknowledgements Charles Cazabon for -F/--fixed-string option. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..b33128c --- /dev/null +++ b/examples/README.md @@ -0,0 +1,7 @@ +# Descriptions of Examples + +* **grin, grind** — Scripts that avoid the pkg_resources startup overhead. +* **grinimports.py** — Tool that will take Python files, extract all of their + import statements, normalize the import statements, and grep for the given + regex on those normalized statements. See "grinimports.py --help" for more + information. diff --git a/examples/README.txt b/examples/README.txt deleted file mode 100644 index 46dc1f1..0000000 --- a/examples/README.txt +++ /dev/null @@ -1,10 +0,0 @@ -Descriptions of Examples ------------------------- - -grin, grind : - Scripts that avoid the pkg_resources startup overhead. - -grinimports.py : - Tool that will take Python files, extract all of their import statements, - normalize the import statements, and grep for the given regex on those - normalized statements. See "grinimports.py --help" for more information. diff --git a/pyproject.toml b/pyproject.toml index b6ff3a1..4c9b316 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,9 @@ build-backend = "hatchling.build" name = "grin3" dynamic = ["version"] description = "A grep program configured the way I like it. (python3 port)" -readme = "README.rst" +readme = "README.md" requires-python = ">=3.10" -license = { file = "LICENSE.txt" } +license = { file = "LICENSE.md" } authors = [{ name = "Robert Kern", email = "robert.kern@enthought.com" }] maintainers = [{ name = "Raffaele Salmaso", email = "raffaele@salmaso.org" }] classifiers = [ @@ -53,10 +53,10 @@ include = [ "/grin", "/tests", "/examples", - "/README.rst", - "/LICENSE.txt", - "/ANNOUNCE.txt", - "/THANKS.txt", + "/README.md", + "/LICENSE.md", + "/CHANGELOG.md", + "/THANKS.md", "/.flake8", "/tox.ini", "/pyproject.toml", From 17385fd0db6f4c08e7794f826fe365bbe71b2fa5 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 11:04:01 +0200 Subject: [PATCH 05/37] Switch from black/flake8/isort to ruff check/format --- .flake8 | 8 ---- .hgignore | 1 + CHANGELOG.md | 1 + README.md | 6 +-- examples/grinimports.py | 5 ++- examples/grinpython.py | 4 +- grin/__init__.py | 2 +- grin/grind.py | 2 +- grin/main.py | 6 +-- grin/recognizer.py | 8 ++-- pyproject.toml | 69 +++++++++++++++++++++++++---------- tests/test_file_recognizer.py | 8 ++-- tests/test_grep.py | 2 +- tox.ini | 20 +++------- 14 files changed, 80 insertions(+), 62 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 3481edb..0000000 --- a/.flake8 +++ /dev/null @@ -1,8 +0,0 @@ -[flake8] -exclude = build,.git,.hg,.tox,.lib,__pycache__ -# E203 doesn't work for slicing -# W503 talks about operator formatting which is too opinionated. -ignore = E203, W503, C819 -max-complexity = 18 -max-line-length = 120 -select = B,C,E,F,W,T4,B9 diff --git a/.hgignore b/.hgignore index be5e454..b5b4071 100644 --- a/.hgignore +++ b/.hgignore @@ -1,4 +1,5 @@ syntax: regexp +.ruff_cache/ .tox/ build/ dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c162502..144fc53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * modernize project setup/configuration * drop vagrant support * require Python >= 3.10 +* switch to ruff (replaces black/isort/flake8) ## 2.6.1 diff --git a/README.md b/README.md index c248255..8f4c6f2 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ and it is available via git mirrors on You are free to open a PR/MR/Bug where you want. -The source code is formatted with [black], [isort] and [flake8]. +The source code is linted and formatted with [ruff]. ## Testing @@ -320,6 +320,4 @@ $ uv run --python 3.15 -m unittest discover tests [rak]: http://rak.rubyforge.org/ [find]: http://www.gnu.org/software/findutils/ [pip]: https://pip.pypa.io/en/stable/ -[black]: https://pypi.org/project/black/ -[isort]: https://pypi.org/project/isort/ -[flake8]: https://pypi.org/project/flake8/ +[ruff]: https://docs.astral.sh/ruff/ diff --git a/examples/grinimports.py b/examples/grinimports.py index 95fd737..63277ef 100755 --- a/examples/grinimports.py +++ b/examples/grinimports.py @@ -3,15 +3,16 @@ Transform Python files into normalized import statements for grepping. """ +from io import StringIO import os import shlex import sys -from io import StringIO import compiler -import grin from compiler.visitor import ASTVisitor, walk +import grin + __version__ = "1.2" diff --git a/examples/grinpython.py b/examples/grinpython.py index f8d6b20..f4709e2 100755 --- a/examples/grinpython.py +++ b/examples/grinpython.py @@ -3,12 +3,12 @@ Transform Python code by omitting strings, comments, and/or code. """ +from io import BytesIO import os import shlex import string import sys import tokenize -from io import BytesIO import grin @@ -64,7 +64,7 @@ def __call__(self, filename, mode="rb"): try: gen = tokenize.generate_tokens(f.readline) old_end = (1, 0) - for kind, token, start, end, line in gen: + for kind, token, start, end, _line in gen: if old_end[0] == start[0]: dx = start[1] - old_end[1] else: diff --git a/grin/__init__.py b/grin/__init__.py index 99cafb9..bd3bf4b 100644 --- a/grin/__init__.py +++ b/grin/__init__.py @@ -29,7 +29,7 @@ from .grind import get_grind_arg_parser # noqa: F401 from .main import GrepText # noqa: F401 from .options import Options # noqa: F401 -from .recognizer import GZIP_MAGIC, FileRecognizer, get_recognizer # noqa: F401 +from .recognizer import FileRecognizer, get_recognizer, GZIP_MAGIC # noqa: F401 from .utils import get_line_offsets, get_regex, is_binary_string # noqa: F401 __version__ = "2.6.1" diff --git a/grin/grind.py b/grin/grind.py index 37b6b57..af521e5 100644 --- a/grin/grind.py +++ b/grin/grind.py @@ -186,7 +186,7 @@ def main(argv=None): fr = get_recognizer(args) for dir in args.dirs: - for filename, k in fr.walk(dir): + for filename, _ in fr.walk(dir): if fnmatch.fnmatch(os.path.basename(filename), args.glob): output(filename) except KeyboardInterrupt: diff --git a/grin/main.py b/grin/main.py index 5383e84..27172db 100644 --- a/grin/main.py +++ b/grin/main.py @@ -27,15 +27,15 @@ import bisect import gzip +from io import UnsupportedOperation import itertools import os import re import stat import sys -from io import UnsupportedOperation from .colors import COLOR_STYLE, colorize -from .datablock import EMPTY_DATABLOCK, DataBlock +from .datablock import DataBlock, EMPTY_DATABLOCK from .options import Options from .utils import get_line_offsets, to_str @@ -131,7 +131,7 @@ def read_block_with_context(self, prev, fp, fp_size, encoding): else: before_start = prev.end - 1 before_count = 0 - for i in range(self.options.before_context): + for _ in range(self.options.before_context): ofs = prev.data.rfind("\n", 0, before_start) before_start = ofs before_count += 1 diff --git a/grin/recognizer.py b/grin/recognizer.py index 76e3d13..a147879 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -69,8 +69,8 @@ def __init__( self, skip_hidden_dirs=False, skip_hidden_files=False, - skip_dirs=set(), - skip_exts=set(), + skip_dirs=None, + skip_exts=None, skip_symlink_dirs=True, skip_symlink_files=True, binary_bytes=4096, @@ -78,7 +78,9 @@ def __init__( ): self.skip_hidden_dirs = skip_hidden_dirs self.skip_hidden_files = skip_hidden_files - self.skip_dirs = skip_dirs + self.skip_dirs = skip_dirs if skip_dirs is not None else set() + if skip_exts is None: + skip_exts = set() # For speed, split extensions into the simple ones, that are # compatible with os.path.splitext and hence can all be diff --git a/pyproject.toml b/pyproject.toml index 4c9b316..c4e8747 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ grin = "grin.grin:main" grind = "grin.grind:main" [dependency-groups] -dev = ["build", "black", "flake8", "isort", "mypy", "tox", "tox-uv", "twine"] +dev = ["build", "mypy", "ruff", "tox", "tox-uv", "twine"] [tool.hatch.version] path = "grin/__init__.py" @@ -57,28 +57,59 @@ include = [ "/LICENSE.md", "/CHANGELOG.md", "/THANKS.md", - "/.flake8", "/tox.ini", "/pyproject.toml", ] exclude = ["**/*.so", "**/*.py~", "**/__pycache__"] -[tool.black] -exclude = ''' -/( - __pycache__ -)/ -''' -include = '\.pyi?$' +[tool.ruff] line-length = 120 -target-version = ['py310'] +target-version = "py310" -[tool.isort] -combine_as_imports = true -default_section = "THIRDPARTY" -force_grid_wrap = 0 -include_trailing_comma = true -indent = 4 -line_length = 120 -multi_line_output = 3 -use_parentheses = true +[tool.ruff.format] +indent-style = "space" +line-ending = "lf" +quote-style = "double" + +[tool.ruff.lint] +ignore = [ + "B018", # Found useless expression. Either assign it to a variable or remove it. + "B026", # Star-arg unpacking after a keyword argument is strongly discouraged + "B904", # Within an except clause, raise exceptions with raise ... from err or raise ... from None to + # distinguish them from errors in exception handling. + "B905", # zip() without an explicit strict= parameter set. strict=True causes the resulting iterator + # to raise a ValueError if the arguments are exhausted at differing lengths. + "E722", # Do not use bare except, specify exception instead +] +fixable = [ + "I", +] +unfixable = [ + "F401", # don't remove unused import +] +select = [ + "B", # flake8-bugbear + "C9", # mccabe + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "Q", + "W", # pycodestyle +] + +[tool.ruff.lint.flake8-quotes] +inline-quotes = "double" + +[tool.ruff.lint.isort] +combine-as-imports = false +force-sort-within-sections = true +force-wrap-aliases = true +known-first-party = [ + "grin", +] +known-third-party = [ +] +order-by-type = false + +[tool.ruff.lint.mccabe] +max-complexity = 18 diff --git a/tests/test_file_recognizer.py b/tests/test_file_recognizer.py index a3d09b2..b7faed6 100644 --- a/tests/test_file_recognizer.py +++ b/tests/test_file_recognizer.py @@ -29,18 +29,18 @@ Test the file recognizer capabilities. """ +from contextlib import contextmanager import errno import gzip import os +from pathlib import Path import shutil import socket import sys -from contextlib import contextmanager -from pathlib import Path from tempfile import TemporaryDirectory from unittest import TestCase -from grin import GZIP_MAGIC, FileRecognizer +from grin import FileRecognizer, GZIP_MAGIC @contextmanager @@ -253,7 +253,7 @@ def cleanup(cls): except Exception as e: print("Could not delete %s: %s" % (filename, e), file=sys.stderr) os.unlink("socket_test") - for dirpath, dirnames, filenames in os.walk("tree"): + for dirpath, dirnames, _filenames in os.walk("tree"): # Make sure every directory can be deleted for dirname in dirnames: os.chmod(os.path.join(dirpath, dirname), 0o700) diff --git a/tests/test_grep.py b/tests/test_grep.py index bad4c2b..b4a0226 100644 --- a/tests/test_grep.py +++ b/tests/test_grep.py @@ -26,8 +26,8 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import gzip -import re from io import BytesIO +import re from unittest import TestCase import grin diff --git a/tox.ini b/tox.ini index d6d0fec..7d19134 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox-uv>=1 -envlist = py310, py311, py312, py313, py314, py315, pypy3 +envlist = py310, py311, py312, py313, py314, py315, pypy3, ruff skipsdist = True skip_missing_interpreters = True @@ -10,19 +10,11 @@ setenv = PYTHONDONTWRITEBYTECODE = 1 usedevelop = True -[testenv:black] +[testenv:ruff] changedir = {toxinidir} skip_install = True deps = - black -commands = black --check --diff examples/ grin/ tests/ - -[testenv:flake8] -changedir = {toxinidir} -skip_install = True -deps = - flake8 - flake8-commas - flake8-isort - flake8-quotes -commands = flake8 --show-source examples/ grin/ tests/ + ruff +commands = + ruff check examples/ grin/ tests/ + ruff format --check --diff examples/ grin/ tests/ From 0c780bae14dea34cfb3a4a635b75a05a2da58079 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Wed, 17 Feb 2021 09:36:15 +0100 Subject: [PATCH 06/37] Rename is_binary_string param from bytes to data --- grin/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/grin/utils.py b/grin/utils.py index a1cce22..0f54dc1 100644 --- a/grin/utils.py +++ b/grin/utils.py @@ -50,18 +50,18 @@ def to_str(s, encoding="utf8"): return s.decode("latin1") -def is_binary_string(bytes): +def is_binary_string(data): """Determine if a string is classified as binary rather than text. Parameters ---------- - bytes : str + data : str Returns ------- is_binary : bool """ - nontext = bytes.translate(ALLBYTES, TEXTCHARS) + nontext = data.translate(ALLBYTES, TEXTCHARS) return bool(nontext) From 1589059507169a25d89fc5032366900d6e5aa207 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Wed, 17 Feb 2021 22:05:58 +0100 Subject: [PATCH 07/37] Require kwargs params for FileRecognizer.__init__ --- grin/recognizer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/grin/recognizer.py b/grin/recognizer.py index a147879..8f8e167 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -67,6 +67,7 @@ class FileRecognizer: def __init__( self, + *, skip_hidden_dirs=False, skip_hidden_files=False, skip_dirs=None, From ebc6987bbe1aefc670192455bba4f89b44c2511d Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 11:25:45 +0200 Subject: [PATCH 08/37] Convert Options to dataclass --- grin/options.py | 45 ++++++++++++++++++++++----------------------- tests/test_grep.py | 27 +++++++++++++++++---------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/grin/options.py b/grin/options.py index d8ff458..fe23a09 100644 --- a/grin/options.py +++ b/grin/options.py @@ -25,29 +25,28 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__all__ = ["Options"] +from dataclasses import dataclass, field +import re +__all__ = ["Options"] -class Options(dict): - """Simple options.""" - def __init__(self, *args, **kwargs): - dict.__init__(self, *args, **kwargs) - self.setdefault("before_context", 0) - self.setdefault("after_context", 0) - self.setdefault("show_line_numbers", True) - self.setdefault("show_match", True) - self.setdefault("show_filename", True) - self.setdefault("show_emacs", False) - self.setdefault("skip_hidden_dirs", False) - self.setdefault("skip_backup_files", True) - self.setdefault("skip_hidden_files", False) - self.setdefault("skip_dirs", set()) - self.setdefault("skip_exts", set()) - self.setdefault("skip_symlink_dirs", True) - self.setdefault("skip_symlink_files", True) - self.setdefault("binary_bytes", 4096) - self.setdefault("re_flags", []) - self.setdefault("fixed_string", False) - self.setdefault("word_regexp", False) - self.__dict__ = self +@dataclass +class Options: + before_context: int = 0 + after_context: int = 0 + show_line_numbers: bool = True + show_match: bool = True + show_filename: bool = True + show_emacs: bool = False + skip_hidden_dirs: bool = False + skip_hidden_files: bool = False + skip_backup_files: bool = True + skip_dirs: set[str] = field(default_factory=set) + skip_exts: set[str] = field(default_factory=set) + skip_symlink_dirs: bool = True + skip_symlink_files: bool = True + binary_bytes: int = 4096 + re_flags: list[re.RegexFlag] = field(default_factory=list) + fixed_string: bool = False + word_regexp: bool = False diff --git a/tests/test_grep.py b/tests/test_grep.py index b4a0226..ae49551 100644 --- a/tests/test_grep.py +++ b/tests/test_grep.py @@ -25,6 +25,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from argparse import Namespace import gzip from io import BytesIO import re @@ -32,6 +33,17 @@ import grin + +def make_regex(regex, *, re_flags=None, fixed_string=False, word_regexp=False): + args = Namespace( + regex=regex, + re_flags=re_flags if re_flags is not None else [], + fixed_string=fixed_string, + word_regexp=word_regexp, + ) + return grin.utils.get_regex(args) + + all_foo = b"""foo foo foo @@ -425,8 +437,7 @@ def test_one_line_after_context_no_lines_before(self): def test_fixed_string_option(self): # -F/--fixed-string works with unescaped regex metachars - options = grin.Options(fixed_string=True, regex="foo(", re_flags=[], before_context=0, after_context=0) - regex_with_metachars = grin.GrepText(grin.utils.get_regex(options)) + regex_with_metachars = grin.GrepText(make_regex("foo(", fixed_string=True)) self.assertEqual( regex_with_metachars.do_grep(BytesIO(regex_metachar_foo)), [(2, 0, "def foo(...):\n", [(4, 8)])], @@ -436,8 +447,7 @@ def test_ascii(self): # -a/--ascii # No match when in ascii mode - options = grin.Options(regex=r"\d", re_flags=[re.A], before_context=0, after_context=0) - regex_unicode = grin.GrepText(grin.utils.get_regex(options)) + regex_unicode = grin.GrepText(make_regex(r"\d", re_flags=[re.A])) self.assertEqual( regex_unicode.do_grep(BytesIO(unicode_digits)), [], @@ -445,8 +455,7 @@ def test_ascii(self): # [(1, 0, 'an Arabic-Indic digit ٢ on the\n', [(22, 23)])] # Unicode (default) - options = grin.Options(regex=r"\d", re_flags=[], before_context=0, after_context=0) - regex_unicode = grin.GrepText(grin.utils.get_regex(options)) + regex_unicode = grin.GrepText(make_regex(r"\d")) self.assertEqual( regex_unicode.do_grep(BytesIO(unicode_digits)), [(1, 0, "an Arabic-Indic digit ٢ on the\n", [(22, 23)])], @@ -456,16 +465,14 @@ def test_word_match_option(self): # -w/--word-regexp # Not a word-match - options = grin.Options(word_regexp=True, regex="tes", re_flags=[], before_context=0, after_context=0) - regex_on_word_boundaries = grin.GrepText(grin.utils.get_regex(options)) + regex_on_word_boundaries = grin.GrepText(make_regex("tes", word_regexp=True)) self.assertEqual( regex_on_word_boundaries.do_grep(BytesIO(word_boundaries)), [], ) # Word-match - options = grin.Options(word_regexp=True, regex="test", re_flags=[], before_context=0, after_context=0) - regex_on_word_boundaries = grin.GrepText(grin.utils.get_regex(options)) + regex_on_word_boundaries = grin.GrepText(make_regex("test", word_regexp=True)) self.assertEqual( regex_on_word_boundaries.do_grep(BytesIO(word_boundaries)), [(1, 0, "This is a test.\n", [(10, 14)])], From 41eddae783ef06e8d8c392a5310df6729798b55d Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 18 Feb 2021 09:55:47 +0100 Subject: [PATCH 09/37] Convert FileRecognizer to dataclass --- grin/recognizer.py | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/grin/recognizer.py b/grin/recognizer.py index 8f8e167..2109b7f 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -25,6 +25,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from dataclasses import dataclass, field import fnmatch import gzip import os @@ -38,6 +39,7 @@ GZIP_MAGIC = b"\037\213" +@dataclass class FileRecognizer: """Configurable way to determine what kind of file something is. @@ -65,41 +67,30 @@ class FileRecognizer: fnmatch pattern to match file names against """ - def __init__( - self, - *, - skip_hidden_dirs=False, - skip_hidden_files=False, - skip_dirs=None, - skip_exts=None, - skip_symlink_dirs=True, - skip_symlink_files=True, - binary_bytes=4096, - include=None, - ): - self.skip_hidden_dirs = skip_hidden_dirs - self.skip_hidden_files = skip_hidden_files - self.skip_dirs = skip_dirs if skip_dirs is not None else set() - if skip_exts is None: - skip_exts = set() - + skip_hidden_dirs: bool = False + skip_hidden_files: bool = False + skip_dirs: set[str] = field(default_factory=set) + skip_exts: set[str] = field(default_factory=set) + skip_symlink_dirs: bool = True + skip_symlink_files: bool = True + binary_bytes: int = 4096 + include: str | None = None + skip_exts_simple: set[str] = field(init=False) + skip_exts_endswith: list[str] = field(init=False) + + def __post_init__(self): # For speed, split extensions into the simple ones, that are # compatible with os.path.splitext and hence can all be # checked for in a single set-lookup, and the weirdos that # can't and therefore must be checked for one at a time. self.skip_exts_simple = set() self.skip_exts_endswith = list() - for ext in skip_exts: + for ext in self.skip_exts: if os.path.splitext("foo.bar" + ext)[1] == ext: self.skip_exts_simple.add(ext) else: self.skip_exts_endswith.append(ext) - self.skip_symlink_dirs = skip_symlink_dirs - self.skip_symlink_files = skip_symlink_files - self.binary_bytes = binary_bytes - self.include = include - def is_binary(self, filename): """Determine if a given file is binary or not. From 7f209de0278aadc88f3a882b189b4df46181d874 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Tue, 16 Feb 2021 01:09:21 +0100 Subject: [PATCH 10/37] Rework color management Add Color enum type and Style dataclass Move colorize from freestanding function as Style method --- grin/colors.py | 106 ++++++++++++++++++++++++++++++------------------- grin/main.py | 12 +++--- 2 files changed, 72 insertions(+), 46 deletions(-) diff --git a/grin/colors.py b/grin/colors.py index ab99f59..6aef0a8 100644 --- a/grin/colors.py +++ b/grin/colors.py @@ -26,27 +26,28 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__all__ = ["colorize"] - - -COLOR_TABLE = [ - "black", - "red", - "green", - "yellow", - "blue", - "magenta", - "cyan", - "white", - "default", -] -COLOR_STYLE = { - "filename": dict(fg="green", bold=True), - "searchterm": dict(fg="black", bg="yellow"), -} +__all__ = ["Color", "Style", "STYLES"] + + +from dataclasses import dataclass +from enum import IntEnum +from typing import Optional + +class Color(IntEnum): + BLACK = 0 + RED = 1 + GREEN = 2 + YELLOW = 3 + BLUE = 4 + MAGENTA = 5 + CYAN = 6 + WHITE = 7 + DEFAULT = 8 -def colorize(s, fg=None, bg=None, bold=False, underline=False, reverse=False): + +@dataclass +class Style: """Wraps a string with ANSI color escape sequences corresponding to the style parameters given. @@ -54,7 +55,6 @@ def colorize(s, fg=None, bg=None, bold=False, underline=False, reverse=False): Parameters ---------- - s : str fg : str Foreground color of the text. One of (black, red, green, yellow, blue, magenta, cyan, white, default) @@ -66,25 +66,51 @@ def colorize(s, fg=None, bg=None, bold=False, underline=False, reverse=False): Whether or not to underline the text. reverse : bool Whether or not to show the text in reverse video. - - Returns - ------- - A string with embedded color escape sequences. """ - style_fragments = [] - if fg in COLOR_TABLE: - # Foreground colors go from 30-39 - style_fragments.append(COLOR_TABLE.index(fg) + 30) - if bg in COLOR_TABLE: - # Background colors go from 40-49 - style_fragments.append(COLOR_TABLE.index(bg) + 40) - if bold: - style_fragments.append(1) - if underline: - style_fragments.append(4) - if reverse: - style_fragments.append(7) - style_start = "\x1b[" + ";".join(map(str, style_fragments)) + "m" - style_end = "\x1b[0m" - return style_start + s + style_end + bg: Optional[Color] = None + fg: Optional[Color] = None + bold: bool = False + underline: bool = False + reverse: bool = False + + @property + def start(self): + style_fragments = [] + if self.fg is not None: + # Foreground colors go from 30-39 + style_fragments.append(self.fg + 30) + if self.bg is not None: + # Background colors go from 40-49 + style_fragments.append(self.bg + 40) + if self.bold: + style_fragments.append(1) + if self.underline: + style_fragments.append(4) + if self.reverse: + style_fragments.append(7) + return "\x1b[" + ";".join(map(str, style_fragments)) + "m" + + @property + def end(self): + return "\x1b[0m" + + def apply(self, text): + """Wraps a string with ANSI color escape sequences. + + Parameters + ---------- + text : str + + Returns + ------- + A string with embedded color escape sequences. + """ + + return self.start + text + self.end + + +STYLES = { + "filename": Style(fg=Color.GREEN, bold=True), + "searchterm": Style(fg=Color.BLACK, bg=Color.YELLOW), +} diff --git a/grin/main.py b/grin/main.py index 27172db..8134ebf 100644 --- a/grin/main.py +++ b/grin/main.py @@ -34,8 +34,8 @@ import stat import sys -from .colors import COLOR_STYLE, colorize -from .datablock import DataBlock, EMPTY_DATABLOCK +from .colors import STYLES +from .datablock import EMPTY_DATABLOCK, DataBlock from .options import Options from .utils import get_line_offsets, to_str @@ -324,7 +324,7 @@ def report(self, context_lines, filename=None): if self.options.show_filename and filename is not None and not self.options.show_emacs: line = "%s:\n" % filename if self.options.use_color: - line = colorize(line, **COLOR_STYLE.get("filename", {})) + line = STYLES["filename"].apply(line) lines.append(line) if self.options.show_emacs: template = "%(filename)s:%(lineno)s: %(line)s" @@ -333,15 +333,15 @@ def report(self, context_lines, filename=None): else: template = "%(line)s" for i, kind, line, spans in context_lines: - if self.options.use_color and kind == MATCH and "searchterm" in COLOR_STYLE: - style = COLOR_STYLE["searchterm"] + if self.options.use_color and kind == MATCH and "searchterm" in STYLES: + style = STYLES["searchterm"] orig_line = line[:] total_offset = 0 for start, end in spans: old_substring = orig_line[start:end] start += total_offset end += total_offset - color_substring = colorize(old_substring, **style) + color_substring = style.apply(old_substring) line = line[:start] + color_substring + line[end:] total_offset += len(color_substring) - len(old_substring) From 14ba4d5619e7036f191f10e0a95d6bdba5c4353c Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 6 Aug 2020 16:41:29 +0200 Subject: [PATCH 11/37] Move EMPTY_DATABLOCK definition --- grin/datablock.py | 19 +++++++++---------- grin/main.py | 6 ++++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/grin/datablock.py b/grin/datablock.py index 3e4f22b..f933ed8 100644 --- a/grin/datablock.py +++ b/grin/datablock.py @@ -25,9 +25,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__all__ = ["DataBlock", "EMPTY_DATABLOCK"] +from dataclasses import dataclass +__all__ = ["DataBlock"] + +@dataclass class DataBlock: """This class holds a block of data read from a file, along with some preceding and trailing context. @@ -47,12 +50,8 @@ class DataBlock: True if this is the final block in the file """ - def __init__(self, data="", start=0, end=0, before_count=0, is_last=False): - self.data = data - self.start = start - self.end = end - self.before_count = before_count - self.is_last = is_last - - -EMPTY_DATABLOCK = DataBlock() + data: str = "" + start: int = 0 + end: int = 0 + before_count: int = 0 + is_last: bool = False diff --git a/grin/main.py b/grin/main.py index 8134ebf..936e3b5 100644 --- a/grin/main.py +++ b/grin/main.py @@ -35,7 +35,7 @@ import sys from .colors import STYLES -from .datablock import EMPTY_DATABLOCK, DataBlock +from .datablock import DataBlock from .options import Options from .utils import get_line_offsets, to_str @@ -46,10 +46,12 @@ MATCH = 0 POST = 1 - # Target amount of data to read into memory at a time. READ_BLOCKSIZE = 16 * 1024 * 1024 +# Initialize an empty datablock +EMPTY_DATABLOCK = DataBlock() + class GrepText: """Grep a single file for a regex by iterating over the lines in a file. From 683c7ba096203657224d86302668d0aeb2978fdc Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 6 Aug 2020 16:49:09 +0200 Subject: [PATCH 12/37] Inline GrepText.options initialization --- grin/main.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/grin/main.py b/grin/main.py index 936e3b5..604725f 100644 --- a/grin/main.py +++ b/grin/main.py @@ -67,11 +67,8 @@ def __init__(self, regex, options=None): self.regex = regex # An equivalent regex with multiline enabled. self.regex_m = re.compile(regex.pattern, regex.flags | re.MULTILINE) - # The options object from parsing the configuration and command line. - if options is None: - options = Options() - self.options = options + self.options = Options() if options is None else options def read_block_with_context(self, prev, fp, fp_size, encoding): """Read a block of data from the file, along with some surrounding From 6659209d38e5038525d901a366e78b35f5b24577 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Fri, 7 Aug 2020 10:00:53 +0200 Subject: [PATCH 13/37] Always use Options class --- examples/grinpython.py | 63 +++++++++++++---------- grin/grin.py | 110 +++++++++++++++++++++++++++-------------- grin/grind.py | 54 +++++++++++++------- grin/options.py | 7 +++ grin/recognizer.py | 24 +++++---- grin/utils.py | 8 +-- 6 files changed, 170 insertions(+), 96 deletions(-) diff --git a/examples/grinpython.py b/examples/grinpython.py index f4709e2..4585f32 100755 --- a/examples/grinpython.py +++ b/examples/grinpython.py @@ -3,6 +3,7 @@ Transform Python code by omitting strings, comments, and/or code. """ +from dataclasses import dataclass, field from io import BytesIO import os import shlex @@ -15,24 +16,21 @@ __version__ = "1.2" +@dataclass class Transformer: - """ - Transform Python files to remove certain features. - """ - - def __init__(self, python_code, comments, strings): - # Keep code. - self.python_code = python_code - # Keep comments. - self.comments = comments - # Keep strings. - self.strings = strings + """Transform Python files to remove certain features.""" + python_code: bool # Keep code. + comments: bool # Keep comments. + strings: bool # Keep strings. + # A table for the translate() function that replaces all non-whitespace + # characters with spaces. + space_table: str = field(init=False) + + def __post_init__(self): table = [" "] * 256 for s in string.whitespace: table[ord(s)] = s - # A table for the translate() function that replaces all non-whitespace - # characters with spaces. self.space_table = "".join(table) def keep_token(self, kind): @@ -83,9 +81,7 @@ def __call__(self, filename, mode="rb"): def get_grinpython_arg_parser(parser=None): - """ - Create the command-line parser. - """ + """Create the command-line parser.""" parser = grin.get_grin_arg_parser(parser) parser.set_defaults(include="*.py") parser.description = "Search Python code with strings, comments, and/or code removed." @@ -110,24 +106,39 @@ def get_grinpython_arg_parser(parser=None): return parser -def grinpython_main(argv=None): +@dataclass +class GrinpythonOptions(grin.options.Options): + python_code: bool = False + comments: bool = False + strings: bool = False + + +def get_grinpython_config(argv): if argv is None: # Look at the GRIN_ARGS environment variable for more arguments. env_args = shlex.split(os.getenv("GRIN_ARGS", "")) argv = [sys.argv[0]] + env_args + sys.argv[1:] parser = get_grinpython_arg_parser() args = parser.parse_args(argv[1:]) - if args.context is not None: - args.before_context = args.context - args.after_context = args.context _isatty = not args.no_color and sys.stdout.isatty() and (os.environ.get("TERM") != "dumb") - args.use_color = args.force_color or _isatty + options = GrinpythonOptions( + before_context=args.context if args.context is not None else 0, + after_context=args.context if args.context is not None else 0, + use_color=args.force_color or _isatty, + python_code=args.python_code, + comments=args.comments, + strings=args.strings, + ) + return options + - xform = Transformer(args.python_code, args.comments, args.strings) +def main(argv=None): + options = get_grinpython_config(argv) + xform = Transformer(python_code=options.python_code, comments=options.comments, strings=options.strings) - regex = grin.get_regex(args) - g = grin.GrepText(regex, args) - for filename, kind in grin.get_filenames(args): + regex = grin.get_regex(options) + g = grin.GrepText(regex, options) + for filename, kind in grin.get_filenames(options): if kind == "text": # Ignore gzipped files. report = g.grep_a_file(filename, opener=xform) @@ -135,4 +146,4 @@ def grinpython_main(argv=None): if __name__ == "__main__": - grinpython_main() + main() diff --git a/grin/grin.py b/grin/grin.py index 75675fb..b4769c0 100644 --- a/grin/grin.py +++ b/grin/grin.py @@ -26,6 +26,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse +from dataclasses import dataclass import fnmatch import gzip import os @@ -34,18 +35,24 @@ import sys from .main import GrepText +from .options import Options from .recognizer import get_recognizer from .utils import deprecate_option, get_regex __all__ = ["get_filenames", "get_grin_arg_parser"] -def get_filenames(args): +@dataclass +class GrinOptions(Options): + pass + + +def get_filenames(options): """Generate the filenames to grep. Parameters ---------- - args : Namespace + args : Options The commandline arguments object. Yields @@ -60,22 +67,22 @@ def get_filenames(args): """ files = [] # If the user has given us a file with filenames, consume them first. - if args.files_from_file is not None: - if args.files_from_file == "-": + if options.files_from_file is not None: + if options.files_from_file == "-": files_file = sys.stdin should_close = False - elif os.path.exists(args.files_from_file): - files_file = open(args.files_from_file) + elif os.path.exists(options.files_from_file): + files_file = open(options.files_from_file) should_close = True else: - raise IOError(2, "No such file: %r" % args.files_from_file) + raise IOError(2, "No such file: %r" % options.files_from_file) try: # Remove '' # XXX: how can I detect bad filenames? One user accidentally ran # grin -f against a binary file and got an unhelpful error message # later. - if args.null_separated: + if options.null_separated: files.extend([x.strip() for x in files_file.read().split("\0")]) else: files.extend([x.strip() for x in files_file]) @@ -84,8 +91,8 @@ def get_filenames(args): files_file.close() # Now add the filenames provided on the command line itself. - files.extend(args.files) - if args.sys_path: + files.extend(options.files) + if options.sys_path: files.extend(sys.path) # Make sure we don't have any empty strings lying around. # Also skip certain special null files which may be added by programs like @@ -103,18 +110,18 @@ def get_filenames(args): # Go over our list of filenames and see if we can recognize each as # something we want to grep. - fr = get_recognizer(args) + fr = get_recognizer(options) for fn in files: # Special case text stdin. if fn == "-": yield fn, "text" continue kind = fr.recognize(fn, None) - if kind in ("text", "gzip") and fnmatch.fnmatch(os.path.basename(fn), args.include): + if kind in ("text", "gzip") and fnmatch.fnmatch(os.path.basename(fn), options.include): yield fn, kind elif kind == "directory": for filename, k in fr.walk(fn): - if k in ("text", "gzip") and fnmatch.fnmatch(os.path.basename(filename), args.include): + if k in ("text", "gzip") and fnmatch.fnmatch(os.path.basename(filename), options.include): yield filename, k # XXX: warn about other files? # XXX: handle binary? @@ -389,35 +396,62 @@ def get_grin_arg_parser(parser=None): return parser +def get_grin_config(argv=None): + if argv is None: + # Look at the GRIN_ARGS environment variable for more arguments. + env_args = shlex.split(os.getenv("GRIN_ARGS", "")) + argv = [sys.argv[0]] + env_args + sys.argv[1:] + parser = get_grin_arg_parser() + args = parser.parse_args(argv[1:]) + use_term_color = not args.no_color and sys.stdout.isatty() and (os.environ.get("TERM") != "dumb") + + # handle deprecated --{no,use,force}-color options + isatty = sys.stdout.isatty() and (os.environ.get("TERM") != "dumb") + if "--no-color" in argv or "--use-color" in argv or "--force-color" in argv: + args.use_color = args.force_color or (not args.no_color and isatty) + else: + args.use_color = (args.color == "auto" and isatty) or args.color == "always" + + return GrinOptions( + before_context=args.context if args.context is not None else args.before_context, + after_context=args.context if args.context is not None else args.after_context, + use_color=args.force_color or use_term_color, + show_line_numbers=args.show_line_numbers, + show_match=args.show_match, + show_filename=args.show_filename, + show_emacs=args.show_emacs, + skip_hidden_dirs=args.skip_hidden_dirs, + skip_hidden_files=args.skip_hidden_files, + skip_backup_files=args.skip_backup_files, + skip_dirs=set([x for x in args.skip_dirs.split(",") if x]), + skip_exts=set([x for x in args.skip_exts.split(",") if x]), + skip_symlink_files=not args.follow_symlinks, + skip_symlink_dirs=not args.follow_symlinks, + re_flags=args.re_flags, + regex=args.regex, + files=args.files, + files_from_file=args.files_from_file, + null_separated=args.null_separated, + include=args.include, + sys_path=args.sys_path, + ) + + def main(argv=None): try: - if argv is None: - # Look at the GRIN_ARGS environment variable for more arguments. - env_args = shlex.split(os.getenv("GRIN_ARGS", "")) - argv = [sys.argv[0]] + env_args + sys.argv[1:] - parser = get_grin_arg_parser() - args = parser.parse_args(argv[1:]) - if args.context is not None: - args.before_context = args.context - args.after_context = args.context - - # handle deprecated --{no,use,force}-color options - isatty = sys.stdout.isatty() and (os.environ.get("TERM") != "dumb") - if "--no-color" in sys.argv or "--use-color" in sys.argv or "--force-color" in sys.argv: - args.use_color = args.force_color or (not args.no_color and isatty) - else: - args.use_color = (args.color == "auto" and isatty) or args.color == "always" + options = get_grin_config(argv) + regex = get_regex(options) + g = GrepText(regex, options) - regex = get_regex(args) - g = GrepText(regex, args) openers = dict(text=open, gzip=gzip.open) - for filename, kind in get_filenames(args): - try: - report = g.grep_a_file(filename, opener=openers[kind], encoding=args.encoding) - except (OSError, EOFError): - if kind != "gzip": - raise # probably shouldn't happen; something weird - report = g.grep_a_file(filename, opener=openers["text"], encoding=args.encoding) + for filename, kind in get_filenames(options): + if kind in ["text", "gzip"]: + try: + report = g.grep_a_file(filename, opener=openers[kind]) + except (OSError, EOFError): + if kind != "gzip": + raise # probably shouldn't happen; something weird + report = g.grep_a_file(filename, opener=openers["text"]) sys.stdout.write(report) except KeyboardInterrupt: raise SystemExit(0) diff --git a/grin/grind.py b/grin/grind.py index af521e5..56674bb 100644 --- a/grin/grind.py +++ b/grin/grind.py @@ -26,17 +26,25 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse +from dataclasses import dataclass, field import fnmatch import os import shlex import sys +from .options import Options from .recognizer import get_recognizer from .utils import deprecate_option __all__ = ["get_grind_arg_parser"] +@dataclass +class GrindOptions(Options): + dirs: list[str] = field(default_factory=list) + glob: str = "*" + + def get_grind_arg_parser(parser=None): """Create the command-line parser for the find-like companion program.""" from . import __version__ @@ -166,28 +174,38 @@ def print_null(filename): sys.stdout.write("\0") +def get_grind_config(argv=None): + if argv is None: + # Look at the GRIND_ARGS environment variable for more arguments. + env_args = shlex.split(os.getenv("GRIND_ARGS", "")) + argv = [sys.argv[0]] + env_args + sys.argv[1:] + parser = get_grind_arg_parser() + args = parser.parse_args(argv[1:]) + options = GrindOptions( + dirs=args.dirs, + glob=args.glob, + null_separated=args.null_separated, + sys_path=args.sys_path, + skip_hidden_dirs=args.skip_hidden_dirs, + skip_hidden_files=args.skip_hidden_files, + skip_backup_files=args.skip_backup_files, + skip_dirs=set([x for x in args.skip_dirs.split(",") if x]), + skip_exts=set([x for x in args.skip_exts.split(",") if x]), + skip_symlink_files=not args.follow_symlinks, + skip_symlink_dirs=not args.follow_symlinks, + ) + return options + + def main(argv=None): try: - if argv is None: - # Look at the GRIND_ARGS environment variable for more arguments. - env_args = shlex.split(os.getenv("GRIND_ARGS", "")) - argv = [sys.argv[0]] + env_args + sys.argv[1:] - parser = get_grind_arg_parser() - args = parser.parse_args(argv[1:]) - + options = get_grind_config(argv) # Define the output function. - if args.null_separated: - output = print_null - else: - output = print - - if args.sys_path: - args.dirs.extend(sys.path) - - fr = get_recognizer(args) - for dir in args.dirs: + output = print_null if options.null_separated else print + fr = get_recognizer(options) + for dir in options.dirs: for filename, _ in fr.walk(dir): - if fnmatch.fnmatch(os.path.basename(filename), args.glob): + if fnmatch.fnmatch(os.path.basename(filename), options.glob): output(filename) except KeyboardInterrupt: raise SystemExit(0) diff --git a/grin/options.py b/grin/options.py index fe23a09..5df5467 100644 --- a/grin/options.py +++ b/grin/options.py @@ -50,3 +50,10 @@ class Options: re_flags: list[re.RegexFlag] = field(default_factory=list) fixed_string: bool = False word_regexp: bool = False + use_color: bool = False + regex: str = "" + files: list[str] = field(default_factory=list) + sys_path: bool = False + files_from_file: str | None = None + null_separated: bool = False + include: str = "*" diff --git a/grin/recognizer.py b/grin/recognizer.py index 2109b7f..0a941be 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -293,28 +293,32 @@ def walk(self, startpath, direntry=None): yield fn, k -def get_recognizer(args): +def get_recognizer(options): """Get the file recognizer object from the configured options.""" # always convert to set - skip_dirs = set([x for x in args.skip_dirs.split(",") if x] if isinstance(args.skip_dirs, str) else args.skip_dirs) - skip_exts = set([x for x in args.skip_exts.split(",") if x] if isinstance(args.skip_exts, str) else args.skip_exts) + skip_dirs = set( + [x for x in options.skip_dirs.split(",") if x] if isinstance(options.skip_dirs, str) else options.skip_dirs, + ) + skip_exts = set( + [x for x in options.skip_exts.split(",") if x] if isinstance(options.skip_exts, str) else options.skip_exts, + ) # handle deprecated --[no-]-skip-backup-files option - if args.skip_backup_files: + if options.skip_backup_files: skip_exts.add("~") - elif not args.skip_backup_files: + elif not options.skip_backup_files: try: skip_exts.remove("~") except KeyError: pass return FileRecognizer( - skip_hidden_files=args.skip_hidden_files, - skip_hidden_dirs=args.skip_hidden_dirs, + skip_hidden_files=options.skip_hidden_files, + skip_hidden_dirs=options.skip_hidden_dirs, skip_dirs=skip_dirs, skip_exts=skip_exts, - skip_symlink_files=not args.follow_symlinks, - skip_symlink_dirs=not args.follow_symlinks, - include=getattr(args, "include", None), + skip_symlink_files=options.skip_symlink_files, + skip_symlink_dirs=options.skip_symlink_dirs, + include=options.include, ) diff --git a/grin/utils.py b/grin/utils.py index 0f54dc1..c116415 100644 --- a/grin/utils.py +++ b/grin/utils.py @@ -92,13 +92,13 @@ def get_line_offsets(block): line_count += 1 -def get_regex(args): +def get_regex(options): """Get the compiled regex object to search with.""" # Combine all of the flags. flags = 0 - for flag in args.re_flags: + for flag in options.re_flags: flags |= flag - pattern = re.escape(args.regex) if args.fixed_string else args.regex - if args.word_regexp: + pattern = re.escape(options.regex) if options.fixed_string else options.regex + if options.word_regexp: pattern = r"\b" + pattern + r"\b" return re.compile(pattern, flags) From 83234a4ab51f421013446163c2233f91b707885c Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Wed, 17 Feb 2021 09:14:03 +0100 Subject: [PATCH 14/37] Rework GrepText match context data structure --- grin/main.py | 104 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 64 insertions(+), 40 deletions(-) diff --git a/grin/main.py b/grin/main.py index 604725f..c2d54ad 100644 --- a/grin/main.py +++ b/grin/main.py @@ -26,6 +26,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import bisect +from dataclasses import dataclass, field import gzip from io import UnsupportedOperation import itertools @@ -33,6 +34,7 @@ import re import stat import sys +from typing import Any from .colors import STYLES from .datablock import DataBlock @@ -52,6 +54,58 @@ # Initialize an empty datablock EMPTY_DATABLOCK = DataBlock() +# Type aliases +BlockContextType = list[tuple[int, int, str, list[tuple[int, int]] | None]] +LineOffsetsType = list[int] | None +LineCountType = int | None + + +@dataclass +class BuildMatchContext: + block: DataBlock + before: int + after: int + regex: Any + line_num_offset: int + block_context: BlockContextType = field(default_factory=list) + line_offsets: LineOffsetsType = None + line_count: LineCountType = None + + def _call(self, match): + match_line_num = bisect.bisect(self.line_offsets, match.start() + self.block.start) - 1 + before_count = min(self.before, match_line_num) + after_count = min(self.after, (len(self.line_offsets) - 1) - match_line_num - 1) + match_line = self.block.data[self.line_offsets[match_line_num] : self.line_offsets[match_line_num + 1]] + spans = [m.span() for m in self.regex.finditer(match_line)] + + before_ctx = [ + ( + i + self.line_num_offset, + PRE, + self.block.data[self.line_offsets[i] : self.line_offsets[i + 1]], + None, + ) + for i in range(match_line_num - before_count, match_line_num) + ] + after_ctx = [ + ( + i + self.line_num_offset, + POST, + self.block.data[self.line_offsets[i] : self.line_offsets[i + 1]], + None, + ) + for i in range(match_line_num + 1, match_line_num + after_count + 1) + ] + match_ctx = [(match_line_num + self.line_num_offset, MATCH, match_line, spans)] + return before_ctx + match_ctx + after_ctx + + def __call__(self, match): + # Computing line offsets is expensive, so we do it lazily. We don't + # take the extra CPU hit unless there's a regex match in the file. + if self.line_offsets is None: + (self.line_offsets, self.line_count) = get_line_offsets(self.block) + self.block_context += self._call(match) + class GrepText: """Grep a single file for a regex by iterating over the lines in a file. @@ -227,56 +281,26 @@ def do_grep_block(self, block, line_num_offset): Returns ------- - Tuple of the form + tuple of the form (line_count, list of (lineno, type (POST/PRE/MATCH), line, spans). 'line_count' is either the number of lines in the block, or None if the line_count was not computed. For each 4-tuple of type MATCH, **spans** is a list of (start,end) positions of substrings that matched the pattern. """ - before = self.options.before_context - after = self.options.after_context - block_context = [] - line_offsets = None - line_count = None - - def build_match_context(match): - match_line_num = bisect.bisect(line_offsets, match.start() + block.start) - 1 - before_count = min(before, match_line_num) - after_count = min(after, (len(line_offsets) - 1) - match_line_num - 1) - match_line = block.data[line_offsets[match_line_num] : line_offsets[match_line_num + 1]] - spans = [m.span() for m in self.regex.finditer(match_line)] - - before_ctx = [ - ( - i + line_num_offset, - PRE, - block.data[line_offsets[i] : line_offsets[i + 1]], - None, - ) - for i in range(match_line_num - before_count, match_line_num) - ] - after_ctx = [ - ( - i + line_num_offset, - POST, - block.data[line_offsets[i] : line_offsets[i + 1]], - None, - ) - for i in range(match_line_num + 1, match_line_num + after_count + 1) - ] - match_ctx = [(match_line_num + line_num_offset, MATCH, match_line, spans)] - return before_ctx + match_ctx + after_ctx + build_match_context = BuildMatchContext( + block=block, + before=self.options.before_context, + after=self.options.after_context, + line_num_offset=line_num_offset, + regex=self.regex, + ) # Using re.MULTILINE here, so ^ and $ will work as expected. for match in self.regex_m.finditer(block.data[block.start : block.end]): - # Computing line offsets is expensive, so we do it lazily. We don't - # take the extra CPU hit unless there's a regex match in the file. - if line_offsets is None: - (line_offsets, line_count) = get_line_offsets(block) - block_context += build_match_context(match) + build_match_context(match) - return (line_count, block_context) + return (build_match_context.line_count, build_match_context.block_context) def uniquify_context(self, context): """Remove duplicate lines from the list of context lines.""" From b5817adab96b120e00c370eb71a69da25dffe3ce Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Wed, 17 Feb 2021 09:22:17 +0100 Subject: [PATCH 15/37] Add mypy config --- .gitignore | 1 + .hgignore | 1 + CHANGELOG.md | 1 + pyproject.toml | 29 +++++++++++++++++++++++++++++ 4 files changed, 32 insertions(+) diff --git a/.gitignore b/.gitignore index c4adb75..20ff8f8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ pip-wheel-metadata/ .venv/ .idea/ __pycache__ +.mypy_cache *~ *- *.swp diff --git a/.hgignore b/.hgignore index b5b4071..821c2e7 100644 --- a/.hgignore +++ b/.hgignore @@ -8,6 +8,7 @@ pip-wheel-metadata/ ^\.venv/ ^\.idea __pycache__ +.mypy_cache syntax: glob *~ diff --git a/CHANGELOG.md b/CHANGELOG.md index 144fc53..3e50e08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * drop vagrant support * require Python >= 3.10 * switch to ruff (replaces black/isort/flake8) +* add mypy config ## 2.6.1 diff --git a/pyproject.toml b/pyproject.toml index c4e8747..b2ac31f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,3 +113,32 @@ order-by-type = false [tool.ruff.lint.mccabe] max-complexity = 18 + +[tool.mypy] +# Specify the target platform details in config, so your developers are +# free to run mypy on Windows, Linux, or macOS and get consistent +# results. +python_version = "3.10" +platform = "linux" + +# sensible formatting +show_column_numbers = true +show_error_codes = true + +# show error messages from unrelated files +follow_imports = "skip" + +# suppress errors about unsatisfied imports +ignore_missing_imports = true + +# be strict +strict = true +strict_optional = true +warn_no_return = true +warn_unused_configs = false +disallow_untyped_defs = true +check_untyped_defs = true + +# No incremental mode +no_incremental = true +cache_dir = "/dev/null" From 2caaae40bd3ea2ff2945f74f2cee5a8938bd7acd Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Wed, 17 Feb 2021 09:29:14 +0100 Subject: [PATCH 16/37] Add mypy check to tox.ini --- CHANGELOG.md | 2 +- pyproject.toml | 7 +++---- tox.ini | 9 ++++++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e50e08..2747cf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * drop vagrant support * require Python >= 3.10 * switch to ruff (replaces black/isort/flake8) -* add mypy config +* add mypy config and tox env ## 2.6.1 diff --git a/pyproject.toml b/pyproject.toml index b2ac31f..894739b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,13 +131,12 @@ follow_imports = "skip" # suppress errors about unsatisfied imports ignore_missing_imports = true -# be strict -strict = true +strict = false strict_optional = true warn_no_return = true warn_unused_configs = false -disallow_untyped_defs = true -check_untyped_defs = true +disallow_untyped_defs = false +check_untyped_defs = false # No incremental mode no_incremental = true diff --git a/tox.ini b/tox.ini index 7d19134..e44db86 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox-uv>=1 -envlist = py310, py311, py312, py313, py314, py315, pypy3, ruff +envlist = py310, py311, py312, py313, py314, py315, pypy3, ruff, mypy skipsdist = True skip_missing_interpreters = True @@ -18,3 +18,10 @@ deps = commands = ruff check examples/ grin/ tests/ ruff format --check --diff examples/ grin/ tests/ + +[testenv:mypy] +changedir = {toxinidir} +skip_install = True +deps = + mypy +commands = mypy grin/ tests/ examples/ From 9ac7fb6ff8289e24e3efd21ed3f966b779604807 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 18 Feb 2021 13:21:03 +0100 Subject: [PATCH 17/37] Add typing to codebase --- CHANGELOG.md | 2 + examples/grinimports.py | 82 +++++++++++---------------- examples/grinpython.py | 24 ++++---- grin/__init__.py | 14 +++++ grin/colors.py | 11 ++-- grin/grin.py | 20 ++++--- grin/grind.py | 8 +-- grin/main.py | 61 ++++++++++---------- grin/options.py | 4 +- grin/recognizer.py | 31 +++++----- grin/types.py | 40 +++++++++++++ grin/utils.py | 15 +++-- pyproject.toml | 7 ++- tests/test_file_recognizer.py | 104 +++++++++++++++++++--------------- tests/test_grep.py | 37 ++++++------ 15 files changed, 266 insertions(+), 194 deletions(-) create mode 100644 grin/types.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2747cf5..262df78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ * require Python >= 3.10 * switch to ruff (replaces black/isort/flake8) * add mypy config and tox env +* add type hints (mypy strict) +* port grinimports example from compiler (py2) to ast (py3) ## 2.6.1 diff --git a/examples/grinimports.py b/examples/grinimports.py index 63277ef..7450316 100755 --- a/examples/grinimports.py +++ b/examples/grinimports.py @@ -3,88 +3,83 @@ Transform Python files into normalized import statements for grepping. """ +import argparse +import ast from io import StringIO -import os -import shlex import sys -import compiler -from compiler.visitor import ASTVisitor, walk - import grin __version__ = "1.2" -def normalize_From(node): +def normalize_from(node: ast.ImportFrom) -> list[str]: """ Return a list of strings of Python 'from' statements, one import on each line. """ statements = [] - children = node.getChildren() - module = "." * node.level + node.modname - for name, asname in children[1]: - line = "from %s import %s" % (module, name) - if asname is not None: - line += " as %s" % asname + module = "." * node.level + (node.module or "") + for alias in node.names: + line = "from %s import %s" % (module, alias.name) + if alias.asname is not None: + line += " as %s" % alias.asname line += "\n" statements.append(line) return statements -def normalize_Import(node): +def normalize_import(node: ast.Import) -> list[str]: """ Return a list of strings of Python 'import' statements, one import on each line. """ statements = [] - children = node.getChildren() - for name, asname in children[0]: - line = "import %s" % (name) - if asname is not None: - line += " as %s" % asname + for alias in node.names: + line = "import %s" % alias.name + if alias.asname is not None: + line += " as %s" % alias.asname line += "\n" statements.append(line) return statements -class ImportPuller(ASTVisitor): +class ImportPuller(ast.NodeVisitor): """ Extract import statements from an AST. """ - def __init__(self): - ASTVisitor.__init__(self) - self.statements = [] + def __init__(self) -> None: + self.statements: list[str] = [] - def visitFrom(self, node): - self.statements.extend(normalize_From(node)) + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + self.statements.extend(normalize_from(node)) - def visitImport(self, node): - self.statements.extend(normalize_Import(node)) + def visit_Import(self, node: ast.Import) -> None: + self.statements.extend(normalize_import(node)) - def as_string(self): + def as_string(self) -> str: """ Concatenate all of the 'import' and 'from' statements. """ return "".join(self.statements) -def normalize_file(filename, *args): +def normalize_file(filename: str, *args: object) -> StringIO: """ Import-normalize a file. If the file is not parseable, an empty filelike object will be returned. """ try: - ast = compiler.parseFile(filename) + with open(filename, "rb") as f: + tree = ast.parse(f.read(), filename=filename) except Exception: return StringIO("") ip = ImportPuller() - walk(ast, ip) + ip.visit(tree) return StringIO(ip.as_string()) -def get_grinimports_arg_parser(parser=None): +def get_grinimports_arg_parser(parser: argparse.ArgumentParser | None = None) -> argparse.ArgumentParser: """ Create the command-line parser. """ @@ -135,25 +130,16 @@ def somefunction(): return parser -def grinimports_main(argv=None): - if argv is None: - # Look at the GRIN_ARGS environment variable for more arguments. - env_args = shlex.split(os.getenv("GRIN_ARGS", "")) - argv = [sys.argv[0]] + env_args + sys.argv[1:] - parser = get_grinimports_arg_parser() - args = parser.parse_args(argv[1:]) - if args.context is not None: - args.before_context = args.context - args.after_context = args.context - _isatty = not args.no_color and sys.stdout.isatty() and (os.environ.get("TERM") != "dumb") - args.use_color = args.force_color or _isatty - - regex = grin.get_regex(args) - g = grin.GrepText(regex, args) - for filename, kind in grin.get_filenames(args): +def grinimports_main(argv: list[str] | None = None) -> None: + options = grin.grin.get_grin_config(argv) + options.include = "*.py" + + regex = grin.get_regex(options) + g = grin.GrepText(regex, options) + for filename, kind in grin.get_filenames(options): if kind == "text": # Ignore gzipped files. - report = g.grep_a_file(filename, opener=normalize_file) + report = g.grep_a_file(filename, opener=normalize_file) # type: ignore[arg-type] sys.stdout.write(report) diff --git a/examples/grinpython.py b/examples/grinpython.py index 4585f32..f423097 100755 --- a/examples/grinpython.py +++ b/examples/grinpython.py @@ -3,13 +3,15 @@ Transform Python code by omitting strings, comments, and/or code. """ +import argparse from dataclasses import dataclass, field -from io import BytesIO +from io import StringIO import os import shlex import string import sys import tokenize +from typing import IO import grin @@ -27,13 +29,13 @@ class Transformer: # characters with spaces. space_table: str = field(init=False) - def __post_init__(self): + def __post_init__(self) -> None: table = [" "] * 256 for s in string.whitespace: table[ord(s)] = s self.space_table = "".join(table) - def keep_token(self, kind): + def keep_token(self, kind: int) -> bool: """ Return True if we should keep the token in the output. """ @@ -46,19 +48,19 @@ def keep_token(self, kind): else: return self.python_code - def replace_with_spaces(self, s): + def replace_with_spaces(self, s: str) -> str: """ Replace all non-newline characters in a string with spaces. """ return s.translate(self.space_table) - def __call__(self, filename, mode="rb"): + def __call__(self, filename: str, mode: str = "rt") -> IO[str]: """ Open a file and convert it to a filelike object with transformed contents. """ - g = BytesIO() - f = open(filename, mode) + g: StringIO = StringIO() + f: IO[str] = open(filename, mode) try: gen = tokenize.generate_tokens(f.readline) old_end = (1, 0) @@ -80,7 +82,7 @@ def __call__(self, filename, mode="rb"): return g -def get_grinpython_arg_parser(parser=None): +def get_grinpython_arg_parser(parser: argparse.ArgumentParser | None = None) -> argparse.ArgumentParser: """Create the command-line parser.""" parser = grin.get_grin_arg_parser(parser) parser.set_defaults(include="*.py") @@ -113,7 +115,7 @@ class GrinpythonOptions(grin.options.Options): strings: bool = False -def get_grinpython_config(argv): +def get_grinpython_config(argv: list[str] | None = None) -> GrinpythonOptions: if argv is None: # Look at the GRIN_ARGS environment variable for more arguments. env_args = shlex.split(os.getenv("GRIN_ARGS", "")) @@ -132,7 +134,7 @@ def get_grinpython_config(argv): return options -def main(argv=None): +def main(argv: list[str] | None = None) -> None: options = get_grinpython_config(argv) xform = Transformer(python_code=options.python_code, comments=options.comments, strings=options.strings) @@ -141,7 +143,7 @@ def main(argv=None): for filename, kind in grin.get_filenames(options): if kind == "text": # Ignore gzipped files. - report = g.grep_a_file(filename, opener=xform) + report = g.grep_a_file(filename, opener=xform) # type: ignore sys.stdout.write(report) diff --git a/grin/__init__.py b/grin/__init__.py index bd3bf4b..cc6af31 100644 --- a/grin/__init__.py +++ b/grin/__init__.py @@ -32,6 +32,20 @@ from .recognizer import FileRecognizer, get_recognizer, GZIP_MAGIC # noqa: F401 from .utils import get_line_offsets, get_regex, is_binary_string # noqa: F401 +__all__ = [ + "FileRecognizer", + "GZIP_MAGIC", + "GrepText", + "Options", + "get_filenames", + "get_grin_arg_parser", + "get_grind_arg_parser", + "get_line_offsets", + "get_recognizer", + "get_regex", + "is_binary_string", +] + __version__ = "2.6.1" __author__ = "Robert Kern" __author_email__ = "robert.kern@enthought.com" diff --git a/grin/colors.py b/grin/colors.py index 6aef0a8..38b17c7 100644 --- a/grin/colors.py +++ b/grin/colors.py @@ -31,7 +31,6 @@ from dataclasses import dataclass from enum import IntEnum -from typing import Optional class Color(IntEnum): @@ -68,14 +67,14 @@ class Style: Whether or not to show the text in reverse video. """ - bg: Optional[Color] = None - fg: Optional[Color] = None + bg: Color | None = None + fg: Color | None = None bold: bool = False underline: bool = False reverse: bool = False @property - def start(self): + def start(self) -> str: style_fragments = [] if self.fg is not None: # Foreground colors go from 30-39 @@ -92,10 +91,10 @@ def start(self): return "\x1b[" + ";".join(map(str, style_fragments)) + "m" @property - def end(self): + def end(self) -> str: return "\x1b[0m" - def apply(self, text): + def apply(self, text: str) -> str: """Wraps a string with ANSI color escape sequences. Parameters diff --git a/grin/grin.py b/grin/grin.py index b4769c0..f834a34 100644 --- a/grin/grin.py +++ b/grin/grin.py @@ -33,10 +33,12 @@ import re import shlex import sys +from typing import cast, Iterator from .main import GrepText from .options import Options from .recognizer import get_recognizer +from .types import Opener from .utils import deprecate_option, get_regex __all__ = ["get_filenames", "get_grin_arg_parser"] @@ -47,7 +49,7 @@ class GrinOptions(Options): pass -def get_filenames(options): +def get_filenames(options: Options) -> Iterator[tuple[str, str]]: """Generate the filenames to grep. Parameters @@ -98,11 +100,11 @@ def get_filenames(options): # Also skip certain special null files which may be added by programs like # Emacs. if sys.platform == "win32": - upper_bad = set(["NUL:", "NUL"]) - raw_bad = set([""]) + upper_bad: set[str] = set(["NUL:", "NUL"]) + raw_bad: set[str] = set([""]) else: - upper_bad = set() - raw_bad = set(["", "/dev/null"]) + upper_bad: set[str] = set() + raw_bad: set[str] = set(["", "/dev/null"]) files = [fn for fn in files if fn not in raw_bad and fn.upper() not in upper_bad] if len(files) == 0: # Add the current directory at least. @@ -127,7 +129,7 @@ def get_filenames(options): # XXX: handle binary? -def get_grin_arg_parser(parser=None): +def get_grin_arg_parser(parser: argparse.ArgumentParser | None = None) -> argparse.ArgumentParser: """Create the command-line parser.""" from . import __version__ @@ -396,7 +398,7 @@ def get_grin_arg_parser(parser=None): return parser -def get_grin_config(argv=None): +def get_grin_config(argv: list[str] | None = None) -> GrinOptions: if argv is None: # Look at the GRIN_ARGS environment variable for more arguments. env_args = shlex.split(os.getenv("GRIN_ARGS", "")) @@ -437,13 +439,13 @@ def get_grin_config(argv=None): ) -def main(argv=None): +def main(argv: list[str] | None = None) -> None: try: options = get_grin_config(argv) regex = get_regex(options) g = GrepText(regex, options) - openers = dict(text=open, gzip=gzip.open) + openers: dict[str, Opener] = {"text": cast(Opener, open), "gzip": cast(Opener, gzip.open)} for filename, kind in get_filenames(options): if kind in ["text", "gzip"]: try: diff --git a/grin/grind.py b/grin/grind.py index 56674bb..8340df2 100644 --- a/grin/grind.py +++ b/grin/grind.py @@ -45,7 +45,7 @@ class GrindOptions(Options): glob: str = "*" -def get_grind_arg_parser(parser=None): +def get_grind_arg_parser(parser: argparse.ArgumentParser | None = None) -> argparse.ArgumentParser: """Create the command-line parser for the find-like companion program.""" from . import __version__ @@ -167,14 +167,14 @@ def get_grind_arg_parser(parser=None): return parser -def print_null(filename): +def print_null(filename: str) -> None: # Note that the final filename will have a trailing NUL, just like # "find -print0" does. sys.stdout.write(filename) sys.stdout.write("\0") -def get_grind_config(argv=None): +def get_grind_config(argv: list[str] | None = None) -> GrindOptions: if argv is None: # Look at the GRIND_ARGS environment variable for more arguments. env_args = shlex.split(os.getenv("GRIND_ARGS", "")) @@ -197,7 +197,7 @@ def get_grind_config(argv=None): return options -def main(argv=None): +def main(argv: list[str] | None = None) -> None: try: options = get_grind_config(argv) # Define the output function. diff --git a/grin/main.py b/grin/main.py index c2d54ad..5835c75 100644 --- a/grin/main.py +++ b/grin/main.py @@ -34,8 +34,9 @@ import re import stat import sys -from typing import Any +from typing import cast +from . import types from .colors import STYLES from .datablock import DataBlock from .options import Options @@ -65,41 +66,43 @@ class BuildMatchContext: block: DataBlock before: int after: int - regex: Any + regex: re.Pattern[str] line_num_offset: int block_context: BlockContextType = field(default_factory=list) line_offsets: LineOffsetsType = None line_count: LineCountType = None - def _call(self, match): - match_line_num = bisect.bisect(self.line_offsets, match.start() + self.block.start) - 1 - before_count = min(self.before, match_line_num) - after_count = min(self.after, (len(self.line_offsets) - 1) - match_line_num - 1) - match_line = self.block.data[self.line_offsets[match_line_num] : self.line_offsets[match_line_num + 1]] + def _call(self, match: re.Match[str]) -> BlockContextType: + match_line_num: int = bisect.bisect(cast(list[int], self.line_offsets), match.start() + self.block.start) - 1 + before_count: int = min(self.before, match_line_num) + after_count: int = min(self.after, (len(cast(list[int], self.line_offsets)) - 1) - match_line_num - 1) + match_line: str = self.block.data[ + cast(list[int], self.line_offsets)[match_line_num] : cast(list[int], self.line_offsets)[match_line_num + 1] + ] spans = [m.span() for m in self.regex.finditer(match_line)] - before_ctx = [ + before_ctx: BlockContextType = [ ( i + self.line_num_offset, PRE, - self.block.data[self.line_offsets[i] : self.line_offsets[i + 1]], + self.block.data[cast(list[int], self.line_offsets)[i] : cast(list[int], self.line_offsets)[i + 1]], None, ) for i in range(match_line_num - before_count, match_line_num) ] - after_ctx = [ + after_ctx: BlockContextType = [ ( i + self.line_num_offset, POST, - self.block.data[self.line_offsets[i] : self.line_offsets[i + 1]], + self.block.data[cast(list[int], self.line_offsets)[i] : cast(list[int], self.line_offsets)[i + 1]], None, ) for i in range(match_line_num + 1, match_line_num + after_count + 1) ] - match_ctx = [(match_line_num + self.line_num_offset, MATCH, match_line, spans)] + match_ctx: BlockContextType = [(match_line_num + self.line_num_offset, MATCH, match_line, spans)] return before_ctx + match_ctx + after_ctx - def __call__(self, match): + def __call__(self, match: re.Match[str]) -> None: # Computing line offsets is expensive, so we do it lazily. We don't # take the extra CPU hit unless there's a regex match in the file. if self.line_offsets is None: @@ -116,7 +119,7 @@ class GrepText: options : Options or similar """ - def __init__(self, regex, options=None): + def __init__(self, regex: re.Pattern[str], options: Options | None = None) -> None: # The compiled regex. self.regex = regex # An equivalent regex with multiline enabled. @@ -124,7 +127,13 @@ def __init__(self, regex, options=None): # The options object from parsing the configuration and command line. self.options = Options() if options is None else options - def read_block_with_context(self, prev, fp, fp_size, encoding): + def read_block_with_context( + self, + prev: DataBlock | None, + fp: types.ReadableFile, + fp_size: int | None, + encoding: str, + ) -> DataBlock: """Read a block of data from the file, along with some surrounding context. @@ -216,7 +225,7 @@ def read_block_with_context(self, prev, fp, fp_size, encoding): ) return result - def do_grep(self, fp, encoding="utf8"): + def do_grep(self, fp: types.ReadableFile, encoding: str = "utf8") -> BlockContextType: """Do a full grep. Parameters @@ -268,7 +277,7 @@ def do_grep(self, fp, encoding="utf8"): unique_context = self.uniquify_context(context) return unique_context - def do_grep_block(self, block, line_num_offset): + def do_grep_block(self, block: DataBlock, line_num_offset: int) -> tuple[int | None, BlockContextType]: """Grep a single block of file content. Parameters @@ -302,7 +311,7 @@ def do_grep_block(self, block, line_num_offset): return (build_match_context.line_count, build_match_context.block_context) - def uniquify_context(self, context): + def uniquify_context(self, context: BlockContextType) -> BlockContextType: """Remove duplicate lines from the list of context lines.""" context.sort() unique_context = [] @@ -319,7 +328,7 @@ def uniquify_context(self, context): return unique_context - def report(self, context_lines, filename=None): + def report(self, context_lines: BlockContextType, filename: str | None = None) -> str: """Return a string showing the results. Parameters @@ -356,7 +365,7 @@ def report(self, context_lines, filename=None): else: template = "%(line)s" for i, kind, line, spans in context_lines: - if self.options.use_color and kind == MATCH and "searchterm" in STYLES: + if self.options.use_color and kind == MATCH and spans is not None and "searchterm" in STYLES: style = STYLES["searchterm"] orig_line = line[:] total_offset = 0 @@ -382,7 +391,7 @@ def report(self, context_lines, filename=None): text = "".join(lines) return text - def grep_a_file(self, filename, opener=open, encoding="utf8"): + def grep_a_file(self, filename: str, opener: types.Opener = open, encoding: str = "utf8") -> str: """Grep a single file that actually exists on the file system. Parameters @@ -400,16 +409,10 @@ def grep_a_file(self, filename, opener=open, encoding="utf8"): The grep results as text. """ # Special-case stdin as "-". - if filename == "-": - f = sys.stdin - filename = "" - else: - # Always open in binary mode - f = opener(filename, "rb") + f, filename = (sys.stdin, "") if filename == "-" else (opener(filename, "rb"), filename) try: unique_context = self.do_grep(f, encoding) finally: if filename != "-": f.close() - report = self.report(unique_context, filename) - return report + return self.report(unique_context, filename) diff --git a/grin/options.py b/grin/options.py index 5df5467..c550cf0 100644 --- a/grin/options.py +++ b/grin/options.py @@ -42,8 +42,8 @@ class Options: skip_hidden_dirs: bool = False skip_hidden_files: bool = False skip_backup_files: bool = True - skip_dirs: set[str] = field(default_factory=set) - skip_exts: set[str] = field(default_factory=set) + skip_dirs: set[str] | str = field(default_factory=set) + skip_exts: set[str] | str = field(default_factory=set) skip_symlink_dirs: bool = True skip_symlink_files: bool = True binary_bytes: int = 4096 diff --git a/grin/recognizer.py b/grin/recognizer.py index 0a941be..c6adba3 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -30,7 +30,10 @@ import gzip import os import stat +from typing import Generator, IO +from .options import Options +from .types import DirEntry from .utils import is_binary_string __all__ = ["FileRecognizer", "get_recognizer"] @@ -63,7 +66,7 @@ class FileRecognizer: binary_bytes : int The number of bytes to check at the beginning and end of a file for binary characters. - include : Optional[str] + include : str | None fnmatch pattern to match file names against """ @@ -78,7 +81,7 @@ class FileRecognizer: skip_exts_simple: set[str] = field(init=False) skip_exts_endswith: list[str] = field(init=False) - def __post_init__(self): + def __post_init__(self) -> None: # For speed, split extensions into the simple ones, that are # compatible with os.path.splitext and hence can all be # checked for in a single set-lookup, and the weirdos that @@ -91,7 +94,7 @@ def __post_init__(self): else: self.skip_exts_endswith.append(ext) - def is_binary(self, filename): + def is_binary(self, filename: str) -> bool: """Determine if a given file is binary or not. Parameters @@ -105,7 +108,7 @@ def is_binary(self, filename): with open(filename, "rb") as fp: return self._is_binary_file(fp) - def _is_binary_file(self, f): + def _is_binary_file(self, f: gzip.GzipFile | IO[bytes]) -> bool: """Determine if a given filelike object has binary data or not. Parameters @@ -124,7 +127,7 @@ def _is_binary_file(self, f): return True return is_binary_string(data) - def is_gzipped_text(self, filename): + def is_gzipped_text(self, filename: str) -> bool: """Determine if a given file is a gzip-compressed text file or not. If the uncompressed file is binary and not text, then this will return @@ -139,20 +142,20 @@ def is_gzipped_text(self, filename): is_gzipped_text : bool """ is_gzipped_text = False - with open(filename, "rb") as fp: - marker = fp.read(2) + with open(filename, "rb") as binary_file: + marker = binary_file.read(2) if marker == GZIP_MAGIC: - with gzip.open(filename) as fp: + with gzip.open(filename) as gzip_file: try: - is_gzipped_text = not self._is_binary_file(fp) + is_gzipped_text = not self._is_binary_file(gzip_file) except IOError: # We saw the GZIP_MAGIC marker, but it is not actually a gzip # file. is_gzipped_text = False return is_gzipped_text - def recognize(self, filename, direntry): + def recognize(self, filename: str, direntry: DirEntry = None) -> str: """Determine what kind of thing a filename represents. It will also determine what a directory walker should do with the @@ -210,7 +213,7 @@ def recognize(self, filename, direntry): except OSError: return "unreadable" - def recognize_directory(self, filename, direntry): + def recognize_directory(self, filename: str, direntry: DirEntry = None) -> str: """Determine what to do with a directory.""" basename = os.path.split(filename)[-1] if self.skip_hidden_dirs and basename.startswith(".") and basename not in (".", ".."): @@ -226,7 +229,7 @@ def recognize_directory(self, filename, direntry): return "skip" return "directory" - def recognize_file(self, filename, direntry): + def recognize_file(self, filename: str, direntry: DirEntry = None) -> str: """Determine what to do with a file.""" basename = os.path.split(filename)[-1] if self.skip_hidden_files and basename.startswith("."): @@ -261,7 +264,7 @@ def recognize_file(self, filename, direntry): except (OSError, IOError): return "unreadable" - def walk(self, startpath, direntry=None): + def walk(self, startpath: str, direntry: DirEntry = None) -> Generator[tuple[str, str], None, None]: """Walk the tree from a given start path yielding all of the files (not directories) and their kinds underneath it depth first. @@ -293,7 +296,7 @@ def walk(self, startpath, direntry=None): yield fn, k -def get_recognizer(options): +def get_recognizer(options: Options) -> FileRecognizer: """Get the file recognizer object from the configured options.""" # always convert to set diff --git a/grin/types.py b/grin/types.py new file mode 100644 index 0000000..1c7223d --- /dev/null +++ b/grin/types.py @@ -0,0 +1,40 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import gzip +import os +from typing import Callable, IO + +# A file being grepped: stdin or a file opened in text or binary mode. +# Binary content is decoded on the fly (see grin.utils.to_str), so both +# text and byte streams are accepted. +ReadableFile = IO[str] | IO[bytes] | gzip.GzipFile + +# A callable that opens a path for grepping, like the builtin open(). +Opener = Callable[[str | bytes | int, str], ReadableFile] + +DirEntry = os.DirEntry[str] | None diff --git a/grin/utils.py b/grin/utils.py index c116415..970eca4 100644 --- a/grin/utils.py +++ b/grin/utils.py @@ -29,6 +29,9 @@ import re import sys +from .datablock import DataBlock +from .options import Options + __all__ = ["get_line_offsets", "get_regex", "is_binary_string", "to_str"] @@ -37,11 +40,11 @@ ALLBYTES = bytes(range(256)) -def deprecate_option(msg): +def deprecate_option(msg: str) -> str: return msg if "--help-verbose" in sys.argv else argparse.SUPPRESS -def to_str(s, encoding="utf8"): +def to_str(s: str | bytes, encoding: str = "utf8") -> str: if isinstance(s, str): return s try: @@ -50,12 +53,12 @@ def to_str(s, encoding="utf8"): return s.decode("latin1") -def is_binary_string(data): +def is_binary_string(data: bytes) -> bool: """Determine if a string is classified as binary rather than text. Parameters ---------- - data : str + data : bytes Returns ------- @@ -65,7 +68,7 @@ def is_binary_string(data): return bool(nontext) -def get_line_offsets(block): +def get_line_offsets(block: DataBlock) -> tuple[list[int], int]: """Compute the list of offsets in DataBlock 'block' which correspond to the beginnings of new lines. @@ -92,7 +95,7 @@ def get_line_offsets(block): line_count += 1 -def get_regex(options): +def get_regex(options: Options) -> re.Pattern[str]: """Get the compiled regex object to search with.""" # Combine all of the flags. flags = 0 diff --git a/pyproject.toml b/pyproject.toml index 894739b..b2ac31f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,12 +131,13 @@ follow_imports = "skip" # suppress errors about unsatisfied imports ignore_missing_imports = true -strict = false +# be strict +strict = true strict_optional = true warn_no_return = true warn_unused_configs = false -disallow_untyped_defs = false -check_untyped_defs = false +disallow_untyped_defs = true +check_untyped_defs = true # No incremental mode no_incremental = true diff --git a/tests/test_file_recognizer.py b/tests/test_file_recognizer.py index b7faed6..5ba8b14 100644 --- a/tests/test_file_recognizer.py +++ b/tests/test_file_recognizer.py @@ -38,13 +38,21 @@ import socket import sys from tempfile import TemporaryDirectory +from typing import Callable, cast, Generator, IO from unittest import TestCase from grin import FileRecognizer, GZIP_MAGIC +from grin.types import DirEntry + +# The fixture helpers open files for *writing* binary content, unlike grin's +# read-oriented grin.types.Opener. +WriteOpener = Callable[[str, str], IO[bytes]] + +GZIP_OPEN = cast(WriteOpener, gzip.open) @contextmanager -def cd(newdir): +def cd(newdir: str) -> Generator[None, None, None]: prevdir = os.getcwd() os.chdir(os.path.expanduser(newdir)) try: @@ -54,15 +62,19 @@ def cd(newdir): class FilesTextCase(TestCase): + oldcwd: Path + tempdir: "TemporaryDirectory[str]" + _socket_file: socket.socket + @classmethod - def setUpClass(cls): + def setUpClass(cls) -> None: cls.oldcwd = Path.cwd() cls.tempdir = TemporaryDirectory() os.chdir(cls.tempdir.name) cls.setup() @classmethod - def tearDownClass(cls): + def tearDownClass(cls) -> None: cls.cleanup() _socket_file = getattr(cls, "_socket_file", None) if _socket_file: @@ -71,17 +83,17 @@ def tearDownClass(cls): cls.tempdir.cleanup() @classmethod - def empty_file(cls, filename, open=open): + def empty_file(cls, filename: str, open: WriteOpener = open) -> None: with open(filename, "wb"): pass @classmethod - def binary_file(cls, filename, open=open): + def binary_file(cls, filename: str, open: WriteOpener = open) -> None: with open(filename, "wb") as f: f.write(bytes(range(255))) @classmethod - def text_file(cls, filename, open=open): + def text_file(cls, filename: str, open: WriteOpener = open) -> None: lines = [b"foo\n", b"bar\n"] * 100 lines.append(b"baz\n") lines.extend([b"foo\n", b"bar\n"] * 100) @@ -89,7 +101,7 @@ def text_file(cls, filename, open=open): f.writelines(lines) @classmethod - def fake_gzip_file(cls, filename, open=open): + def fake_gzip_file(cls, filename: str, open: WriteOpener = open) -> None: """Write out a binary file that has the gzip magic header bytes, but is not a gzip file. """ @@ -98,7 +110,7 @@ def fake_gzip_file(cls, filename, open=open): f.write(bytes(range(255))) @classmethod - def binary_middle(cls, filename, open=open): + def binary_middle(cls, filename: str, open: WriteOpener = open) -> None: """Write out a file that is text for the first 100 bytes, then 100 binary bytes, then 100 text bytes to test that the recognizer only reads some of the file. @@ -108,36 +120,36 @@ def binary_middle(cls, filename, open=open): f.write(text) @classmethod - def socket_file(cls, filename): + def socket_file(cls, filename: str) -> None: cls._socket_file = socket.socket(socket.AF_UNIX) cls._socket_file.bind(filename) @classmethod - def unreadable_file(cls, filename): + def unreadable_file(cls, filename: str) -> None: """Write a file that does not have read permissions.""" cls.text_file(filename) os.chmod(filename, 0o200) @classmethod - def unreadable_dir(cls, filename): + def unreadable_dir(cls, filename: str) -> None: """Make a directory that does not have read permissions.""" os.mkdir(filename) os.chmod(filename, 0o300) @classmethod - def unexecutable_dir(cls, filename): + def unexecutable_dir(cls, filename: str) -> None: """Make a directory that does not have execute permissions.""" os.mkdir(filename) os.chmod(filename, 0o600) @classmethod - def totally_unusable_dir(cls, filename): + def totally_unusable_dir(cls, filename: str) -> None: """Make a directory that has neither read nor execute permissions.""" os.mkdir(filename) os.chmod(filename, 0o100) @classmethod - def setup(cls): + def setup(cls) -> None: # Make files to test individual recognizers. cls.empty_file("empty") cls.binary_file("binary") @@ -149,11 +161,11 @@ def setup(cls): os.mkdir("dir") cls.binary_file(".binary") cls.text_file(".text") - cls.empty_file("empty.gz", open=gzip.open) - cls.binary_file("binary.gz", open=gzip.open) - cls.text_file("text.gz", open=gzip.open) - cls.binary_file(".binary.gz", open=gzip.open) - cls.text_file(".text.gz", open=gzip.open) + cls.empty_file("empty.gz", open=GZIP_OPEN) + cls.binary_file("binary.gz", open=GZIP_OPEN) + cls.text_file("text.gz", open=GZIP_OPEN) + cls.binary_file(".binary.gz", open=GZIP_OPEN) + cls.text_file(".text.gz", open=GZIP_OPEN) cls.fake_gzip_file("fake.gz") os.mkdir(".dir") os.symlink("binary", "binary_link") @@ -205,7 +217,7 @@ def setup(cls): os.chmod("tree/totally_unusable_dir", 0o100) @classmethod - def cleanup(cls): + def cleanup(cls) -> None: files_to_delete = [ "empty", "binary", @@ -261,28 +273,28 @@ def cleanup(cls): # This class tests what the recognizer does if no DirEntry is provided. # These are overridden in FilesTextCase_DirEntry to provide DirEntry's. - def _recognize(self, filename, fr): + def _recognize(self, filename: str, fr: FileRecognizer) -> str: return fr.recognize(filename, None) - def _recognize_file(self, filename, fr): + def _recognize_file(self, filename: str, fr: FileRecognizer) -> str: return fr.recognize(filename, None) - def _recognize_directory(self, dirname, fr): + def _recognize_directory(self, dirname: str, fr: FileRecognizer) -> str: return fr.recognize_directory(dirname, None) - def test_binary(self): + def test_binary(self) -> None: fr = FileRecognizer() self.assertTrue(fr.is_binary("binary")) self.assertEqual(self._recognize_file("binary", fr), "binary") self.assertEqual(self._recognize("binary", fr), "binary") - def test_text(self): + def test_text(self) -> None: fr = FileRecognizer() self.assertFalse(fr.is_binary("text")) self.assertEqual(self._recognize_file("text", fr), "text") self.assertEqual(self._recognize("text", fr), "text") - def test_gzipped(self): + def test_gzipped(self) -> None: fr = FileRecognizer() self.assertTrue(fr.is_binary("text.gz")) self.assertEqual(self._recognize_file("text.gz", fr), "gzip") @@ -294,7 +306,7 @@ def test_gzipped(self): self.assertEqual(self._recognize_file("fake.gz", fr), "binary") self.assertEqual(self._recognize("fake.gz", fr), "binary") - def test_binary_middle(self): + def test_binary_middle(self) -> None: fr = FileRecognizer(binary_bytes=100) self.assertFalse(fr.is_binary("binary_middle")) self.assertEqual(self._recognize_file("binary_middle", fr), "text") @@ -304,16 +316,16 @@ def test_binary_middle(self): self.assertEqual(self._recognize_file("binary_middle", fr), "binary") self.assertEqual(self._recognize("binary_middle", fr), "binary") - def test_socket(self): + def test_socket(self) -> None: fr = FileRecognizer() self.assertEqual(self._recognize("socket_test", fr), "skip") - def test_dir(self): + def test_dir(self) -> None: fr = FileRecognizer() self.assertEqual(self._recognize_directory("dir", fr), "directory") self.assertEqual(self._recognize("dir", fr), "directory") - def test_skip_symlinks(self): + def test_skip_symlinks(self) -> None: fr = FileRecognizer(skip_symlink_files=True, skip_symlink_dirs=True) self.assertEqual(self._recognize("binary_link", fr), "link") self.assertEqual(self._recognize_file("binary_link", fr), "link") @@ -322,7 +334,7 @@ def test_skip_symlinks(self): self.assertEqual(self._recognize("dir_link", fr), "link") self.assertEqual(self._recognize_directory("dir_link", fr), "link") - def test_do_not_skip_symlinks(self): + def test_do_not_skip_symlinks(self) -> None: fr = FileRecognizer(skip_symlink_files=False, skip_symlink_dirs=False) self.assertEqual(self._recognize("binary_link", fr), "binary") self.assertEqual(self._recognize_file("binary_link", fr), "binary") @@ -331,7 +343,7 @@ def test_do_not_skip_symlinks(self): self.assertEqual(self._recognize("dir_link", fr), "directory") self.assertEqual(self._recognize_directory("dir_link", fr), "directory") - def test_skip_hidden(self): + def test_skip_hidden(self) -> None: fr = FileRecognizer(skip_hidden_files=True, skip_hidden_dirs=True) self.assertEqual(self._recognize(".binary", fr), "skip") self.assertEqual(self._recognize_file(".binary", fr), "skip") @@ -350,7 +362,7 @@ def test_skip_hidden(self): self.assertEqual(self._recognize(".binary.gz", fr), "skip") self.assertEqual(self._recognize_file(".binary.gz", fr), "skip") - def test_skip_weird_exts(self): + def test_skip_weird_exts(self) -> None: fr = FileRecognizer(skip_exts=set()) self.assertEqual(self._recognize_file("text#", fr), "text") self.assertEqual(self._recognize_file("foo.bar.baz", fr), "text") @@ -358,7 +370,7 @@ def test_skip_weird_exts(self): self.assertEqual(self._recognize_file("text#", fr), "skip") self.assertEqual(self._recognize_file("foo.bar.baz", fr), "skip") - def test_do_not_skip_hidden_or_symlinks(self): + def test_do_not_skip_hidden_or_symlinks(self) -> None: fr = FileRecognizer( skip_hidden_files=False, skip_hidden_dirs=False, @@ -382,7 +394,7 @@ def test_do_not_skip_hidden_or_symlinks(self): self.assertEqual(self._recognize(".binary.gz", fr), "binary") self.assertEqual(self._recognize_file(".binary.gz", fr), "binary") - def test_do_not_skip_hidden_but_skip_symlinks(self): + def test_do_not_skip_hidden_but_skip_symlinks(self) -> None: fr = FileRecognizer( skip_hidden_files=False, skip_hidden_dirs=False, @@ -406,7 +418,7 @@ def test_do_not_skip_hidden_but_skip_symlinks(self): self.assertEqual(self._recognize(".binary.gz", fr), "binary") self.assertEqual(self._recognize_file(".binary.gz", fr), "binary") - def test_lack_of_permissions(self): + def test_lack_of_permissions(self) -> None: fr = FileRecognizer() self.assertEqual(self._recognize("unreadable_file", fr), "unreadable") self.assertEqual(self._recognize_file("unreadable_file", fr), "unreadable") @@ -417,7 +429,7 @@ def test_lack_of_permissions(self): self.assertEqual(self._recognize("totally_unusable_dir", fr), "directory") self.assertEqual(self._recognize_directory("totally_unusable_dir", fr), "directory") - def test_symlink_src_unreadable(self): + def test_symlink_src_unreadable(self) -> None: fr = FileRecognizer(skip_symlink_files=False, skip_symlink_dirs=False) self.assertEqual(self._recognize("unreadable_file_link", fr), "unreadable") self.assertEqual(self._recognize_file("unreadable_file_link", fr), "unreadable") @@ -428,7 +440,7 @@ def test_symlink_src_unreadable(self): self.assertEqual(self._recognize("totally_unusable_dir_link", fr), "directory") self.assertEqual(self._recognize_directory("totally_unusable_dir_link", fr), "directory") - def test_skip_ext(self): + def test_skip_ext(self) -> None: fr = FileRecognizer(skip_exts=set([".skip_ext", "~"])) self.assertEqual(self._recognize("text.skip_ext", fr), "skip") self.assertEqual(self._recognize_file("text.skip_ext", fr), "skip") @@ -440,14 +452,14 @@ def test_skip_ext(self): self.assertEqual(self._recognize_directory("dir.skip_ext", fr), "directory") self.assertEqual(self._recognize_file("text~", fr), "skip") - def test_skip_dir(self): + def test_skip_dir(self) -> None: fr = FileRecognizer(skip_dirs=set(["skip_dir", "fake_skip_dir"])) self.assertEqual(self._recognize("skip_dir", fr), "skip") self.assertEqual(self._recognize_directory("skip_dir", fr), "skip") self.assertEqual(self._recognize("fake_skip_dir", fr), "text") self.assertEqual(self._recognize_file("fake_skip_dir", fr), "text") - def test_walking(self): + def test_walking(self) -> None: fr = FileRecognizer( skip_hidden_files=True, skip_hidden_dirs=True, @@ -465,7 +477,7 @@ def test_walking(self): result = sorted(fr.walk("tree")) self.assertEqual(result, truth) - def test_dot(self): + def test_dot(self) -> None: with cd("tree"): fr = FileRecognizer( skip_hidden_files=True, @@ -484,7 +496,7 @@ def test_dot(self): result = sorted(fr.walk(".")) self.assertEqual(result, truth) - def test_dot_dot(self): + def test_dot_dot(self) -> None: with cd("tree/dir"): fr = FileRecognizer( skip_hidden_files=True, @@ -510,7 +522,7 @@ class DirEntryFilesTextCase_DirEntry(FilesTextCase): .recognize_directory and .recognize_file. """ - def _do_recognize(self, filename, fr, method): + def _do_recognize(self, filename: str, fr: FileRecognizer, method: Callable[[str, DirEntry], str]) -> str: # Working backwards from the filename to the DirEntry record that would have produced it. direc, fn = os.path.split(filename) @@ -522,11 +534,11 @@ def _do_recognize(self, filename, fr, method): return method(filename, dire) raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename) - def _recognize(self, filename, fr): + def _recognize(self, filename: str, fr: FileRecognizer) -> str: return self._do_recognize(filename, fr, fr.recognize) - def _recognize_file(self, filename, fr): + def _recognize_file(self, filename: str, fr: FileRecognizer) -> str: return self._do_recognize(filename, fr, fr.recognize) - def _recognize_directory(self, filename, fr): + def _recognize_directory(self, filename: str, fr: FileRecognizer) -> str: return self._do_recognize(filename, fr, fr.recognize_directory) diff --git a/tests/test_grep.py b/tests/test_grep.py index ae49551..549621c 100644 --- a/tests/test_grep.py +++ b/tests/test_grep.py @@ -25,7 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from argparse import Namespace import gzip from io import BytesIO import re @@ -34,14 +33,20 @@ import grin -def make_regex(regex, *, re_flags=None, fixed_string=False, word_regexp=False): - args = Namespace( +def make_regex( + regex: str, + *, + re_flags: list[re.RegexFlag] | None = None, + fixed_string: bool = False, + word_regexp: bool = False, +) -> re.Pattern[str]: + options = grin.Options( regex=regex, re_flags=re_flags if re_flags is not None else [], fixed_string=fixed_string, word_regexp=word_regexp, ) - return grin.utils.get_regex(args) + return grin.utils.get_regex(options) all_foo = b"""foo @@ -186,7 +191,7 @@ def foo(...): class GrepTestCase(TestCase): - def test_non_ascii(self): + def test_non_ascii(self) -> None: non_ascii = grin.GrepText(re.compile("é")) self.assertEqual( non_ascii.do_grep(BytesIO(utf_8_foo), encoding="utf8"), @@ -205,7 +210,7 @@ def test_non_ascii(self): [(0, 0, "Rémy\n", [(1, 2)])], ) - def test_basic_defaults(self): + def test_basic_defaults(self) -> None: # Test the basic defaults, no context. gt_default = grin.GrepText(re.compile("foo")) self.assertEqual( @@ -233,7 +238,7 @@ def test_basic_defaults(self): [(2, 0, "barfoobar\n", [(3, 6)])], ) - def test_symmetric_one_line_context(self): + def test_symmetric_one_line_context(self) -> None: # Symmetric 1-line context. gt_context_1 = grin.GrepText(re.compile("foo"), options=grin.Options(before_context=1, after_context=1)) @@ -288,7 +293,7 @@ def test_symmetric_one_line_context(self): [(1, -1, "bar\n", None), (2, 0, "barfoobar\n", [(3, 6)]), (3, 1, "bar\n", None)], ) - def test_symmetric_two_line_context(self): + def test_symmetric_two_line_context(self) -> None: # Symmetric 2-line context. gt_context_2 = grin.GrepText(re.compile("foo"), options=grin.Options(before_context=2, after_context=2)) @@ -352,7 +357,7 @@ def test_symmetric_two_line_context(self): ], ) - def test_one_line_before_no_lines_after(self): + def test_one_line_before_no_lines_after(self) -> None: # 1 line of before-context, no lines after. gt_before_context_1 = grin.GrepText(re.compile("foo"), options=grin.Options(before_context=1, after_context=0)) @@ -393,7 +398,7 @@ def test_one_line_before_no_lines_after(self): [(1, -1, "bar\n", None), (2, 0, "barfoobar\n", [(3, 6)])], ) - def test_one_line_after_context_no_lines_before(self): + def test_one_line_after_context_no_lines_before(self) -> None: # 1 line of after-context, no lines before. gt_after_context_1 = grin.GrepText(re.compile("foo"), options=grin.Options(before_context=0, after_context=1)) @@ -434,7 +439,7 @@ def test_one_line_after_context_no_lines_before(self): [(2, 0, "barfoobar\n", [(3, 6)]), (3, 1, "bar\n", None)], ) - def test_fixed_string_option(self): + def test_fixed_string_option(self) -> None: # -F/--fixed-string works with unescaped regex metachars regex_with_metachars = grin.GrepText(make_regex("foo(", fixed_string=True)) @@ -443,7 +448,7 @@ def test_fixed_string_option(self): [(2, 0, "def foo(...):\n", [(4, 8)])], ) - def test_ascii(self): + def test_ascii(self) -> None: # -a/--ascii # No match when in ascii mode @@ -461,7 +466,7 @@ def test_ascii(self): [(1, 0, "an Arabic-Indic digit ٢ on the\n", [(22, 23)])], ) - def test_word_match_option(self): + def test_word_match_option(self) -> None: # -w/--word-regexp # Not a word-match @@ -478,7 +483,7 @@ def test_word_match_option(self): [(1, 0, "This is a test.\n", [(10, 14)])], ) - def test_gzip(self): + def test_gzip(self) -> None: # Test finding matches in a gzip file actually works. # To be identified as a gzip file, it has to be passed in as an actual @@ -495,7 +500,7 @@ def test_gzip(self): ], ) - def test_broken_gzip(self): + def test_broken_gzip(self) -> None: # Make sure do_grep() raises the correct exceptions when passed a broken gzip # file, so that it is caught in grin.main() and retried as a plaintext file. @@ -517,7 +522,7 @@ def test_broken_gzip(self): gzip_file, ) - def test_broken_gzip_plaintext(self): + def test_broken_gzip_plaintext(self) -> None: # Make sure do_grep() can find a plaintext match in a broken gzip file. gt = grin.GrepText(re.compile("ar")) From bef34a92ae3030abc76b89798ea509e5185b0c2e Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 16:20:11 +0200 Subject: [PATCH 18/37] Skip more VCS dirs by default: .git, .jj, .sl, .pijul, _darcs, SCCS --- CHANGELOG.md | 1 + grin/grin.py | 2 +- grin/grind.py | 2 +- tests/test_file_recognizer.py | 46 ++++++++++++++++++++++++++++++++++- 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 262df78..9f6bcd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * add mypy config and tox env * add type hints (mypy strict) * port grinimports example from compiler (py2) to ast (py3) +* skip .git and more VCS dirs by default (.git, .jj, .sl, .pijul, _darcs, SCCS) ## 2.6.1 diff --git a/grin/grin.py b/grin/grin.py index f834a34..06d2e7f 100644 --- a/grin/grin.py +++ b/grin/grin.py @@ -333,7 +333,7 @@ def get_grin_arg_parser(parser: argparse.ArgumentParser | None = None) -> argpar parser.add_argument( "-d", "--skip-dirs", - default="CVS,RCS,.svn,.hg,.bzr,build,dist,target,node_modules", + default="CVS,RCS,SCCS,.svn,.hg,.bzr,.git,.jj,.sl,.pijul,_darcs,build,dist,target,node_modules", help="comma-separated list of directory names to skip [default=%(default)r]", ) parser.add_argument( diff --git a/grin/grind.py b/grin/grind.py index 8340df2..68ba73a 100644 --- a/grin/grind.py +++ b/grin/grind.py @@ -107,7 +107,7 @@ def get_grind_arg_parser(parser: argparse.ArgumentParser | None = None) -> argpa parser.add_argument( "-d", "--skip-dirs", - default="CVS,RCS,.svn,.hg,.bzr,build,dist,target,node_modules", + default="CVS,RCS,SCCS,.svn,.hg,.bzr,.git,.jj,.sl,.pijul,_darcs,build,dist,target,node_modules", help="comma-separated list of directory names to skip [default=%(default)r]", ) parser.add_argument( diff --git a/tests/test_file_recognizer.py b/tests/test_file_recognizer.py index 5ba8b14..ba23097 100644 --- a/tests/test_file_recognizer.py +++ b/tests/test_file_recognizer.py @@ -29,6 +29,7 @@ Test the file recognizer capabilities. """ +import argparse from contextlib import contextmanager import errno import gzip @@ -41,7 +42,7 @@ from typing import Callable, cast, Generator, IO from unittest import TestCase -from grin import FileRecognizer, GZIP_MAGIC +from grin import FileRecognizer, get_grin_arg_parser, get_grind_arg_parser, GZIP_MAGIC from grin.types import DirEntry # The fixture helpers open files for *writing* binary content, unlike grin's @@ -542,3 +543,46 @@ def _recognize_file(self, filename: str, fr: FileRecognizer) -> str: def _recognize_directory(self, filename: str, fr: FileRecognizer) -> str: return self._do_recognize(filename, fr, fr.recognize_directory) + + +class DefaultSkipDirsTestCase(TestCase): + """The default skip-dirs list should cover common VCS metadata dirs.""" + + # Markers for: CVS, RCS, SCCS, Subversion, Mercurial, Bazaar, Git, + # Jujutsu, Sapling, Pijul, Darcs. + VCS_DIRS = [ + "CVS", + "RCS", + "SCCS", + ".svn", + ".hg", + ".bzr", + ".git", + ".jj", + ".sl", + ".pijul", + "_darcs", + ] + + def _default_skip_dirs(self, parser: argparse.ArgumentParser, argv: list[str]) -> set[str]: + args = parser.parse_args(argv) + return {d for d in args.skip_dirs.split(",") if d} + + def test_grin_default_skips_vcs_dirs(self) -> None: + skip_dirs = self._default_skip_dirs(get_grin_arg_parser(), ["regex"]) + for name in self.VCS_DIRS: + with self.subTest(dir=name): + self.assertIn(name, skip_dirs) + + def test_grind_default_skips_vcs_dirs(self) -> None: + skip_dirs = self._default_skip_dirs(get_grind_arg_parser(), []) + for name in self.VCS_DIRS: + with self.subTest(dir=name): + self.assertIn(name, skip_dirs) + + def test_vcs_dirs_skipped_without_hidden_skip(self) -> None: + skip_dirs = self._default_skip_dirs(get_grin_arg_parser(), ["regex"]) + fr = FileRecognizer(skip_dirs=skip_dirs, skip_hidden_dirs=False) + for name in self.VCS_DIRS: + with self.subTest(dir=name): + self.assertEqual(fr.recognize_directory(name, None), "skip") From 6c3c622c4201ecb318cde7ff29f60277dfb74a12 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 22:16:34 +0200 Subject: [PATCH 19/37] Honor .gitignore/.ignore/.hgignore by default --- CHANGELOG.md | 2 + README.md | 31 +++++ grin/grin.py | 8 ++ grin/grind.py | 8 ++ grin/ignore.py | 305 +++++++++++++++++++++++++++++++++++++++++++ grin/options.py | 1 + grin/recognizer.py | 50 +++++-- pyproject.toml | 1 + tests/test_ignore.py | 236 +++++++++++++++++++++++++++++++++ tox.ini | 5 +- 10 files changed, 636 insertions(+), 11 deletions(-) create mode 100644 grin/ignore.py create mode 100644 tests/test_ignore.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f6bcd2..8f3d7c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ * add type hints (mypy strict) * port grinimports example from compiler (py2) to ast (py3) * skip .git and more VCS dirs by default (.git, .jj, .sl, .pijul, _darcs, SCCS) +* honor .gitignore/.ignore/.hgignore by default (--no-ignore to disable) + also honor per-repo and user-global VCS excludes: git .git/info/exclude and core.excludesFile, hg .hg/hgrc and global hgrc [ui] ignore ## 2.6.1 diff --git a/README.md b/README.md index 8f4c6f2..11d8bc7 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,37 @@ $ grin some_regex `find . -newer some_file.txt` $ find . -newer some_file.txt | grin -f - some_regex ``` +## Ignore files + +By default grin (and grind) honor ignore files found while walking a directory: + +* `.gitignore` and `.ignore` (gitwildmatch syntax; `.ignore` takes precedence). + The `.gitignore` syntax also covers Jujutsu and Sapling, which read the same + files. +* `.hgignore` (Mercurial; `syntax: glob`/`syntax: regexp` sections). + +It also honors each VCS's per-repository and user-global excludes, the same way +`git`/`hg` combine them (in-tree files win over per-repo excludes, which win +over global): + +* git — `/.git/info/exclude` and `core.excludesFile` from your git config + (default `$XDG_CONFIG_HOME/git/ignore`). +* hg — the `[ui] ignore`/`ignore.*` files named in `/.hg/hgrc` and in your + global `hgrc`. + +Disable all of this and search everything the other rules allow with +`--no-ignore`: + +```bash +$ grin --no-ignore some_regex +``` + +Ignore files are read only at and below the directories being searched: grin +does not walk *up* to a repository root above the search root, so a repo's +per-repo excludes apply only when its root is at or below where the search +starts. Paths given explicitly on the command line are always searched, never +ignored. + ## Using grind To find files matching the glob "foo\*.py" in this directory or any subdirectory diff --git a/grin/grin.py b/grin/grin.py index 06d2e7f..9f2ea0c 100644 --- a/grin/grin.py +++ b/grin/grin.py @@ -384,6 +384,13 @@ def get_grin_arg_parser(parser: argparse.ArgumentParser | None = None) -> argpar help="filenames specified in --files-from-file are separated by NULs", ) parser.add_argument("--sys-path", action="store_true", help="search the directories on sys.path") + parser.add_argument( + "--no-ignore", + action="store_false", + dest="respect_ignore_files", + default=True, + help="do not honor .gitignore/.ignore/.hgignore files", + ) parser.add_argument( "-x", @@ -436,6 +443,7 @@ def get_grin_config(argv: list[str] | None = None) -> GrinOptions: null_separated=args.null_separated, include=args.include, sys_path=args.sys_path, + respect_ignore_files=args.respect_ignore_files, ) diff --git a/grin/grind.py b/grin/grind.py index 68ba73a..95d9ff1 100644 --- a/grin/grind.py +++ b/grin/grind.py @@ -153,6 +153,13 @@ def get_grind_arg_parser(parser: argparse.ArgumentParser | None = None) -> argpa ) parser.add_argument("--dirs", nargs="+", default=["."], help="the directories to start from") parser.add_argument("--sys-path", action="store_true", help="search the directories on sys.path") + parser.add_argument( + "--no-ignore", + action="store_false", + dest="respect_ignore_files", + default=True, + help="do not honor .gitignore/.ignore/.hgignore files", + ) parser.add_argument( "glob", @@ -186,6 +193,7 @@ def get_grind_config(argv: list[str] | None = None) -> GrindOptions: glob=args.glob, null_separated=args.null_separated, sys_path=args.sys_path, + respect_ignore_files=args.respect_ignore_files, skip_hidden_dirs=args.skip_hidden_dirs, skip_hidden_files=args.skip_hidden_files, skip_backup_files=args.skip_backup_files, diff --git a/grin/ignore.py b/grin/ignore.py new file mode 100644 index 0000000..71b84ec --- /dev/null +++ b/grin/ignore.py @@ -0,0 +1,305 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Honor ignore files (.gitignore, .ignore, .hgignore) while walking a tree. + +`.gitignore` and `.ignore` use the gitwildmatch syntax (via ``pathspec``) and +nest per-directory; `.ignore` is read after `.gitignore` so it takes +precedence. `.gitignore` syntax also covers Jujutsu and Sapling, which read the +same files. `.hgignore` (Mercurial) uses a different, root-level syntax with +``syntax: glob|regexp`` sections (default regexp) and no negation. + +Beyond the tracked ignore files, grin also honors the per-repository and +user-global excludes of both VCSes, mirroring how ``git``/``hg`` themselves +combine them (in-tree files win over per-repo excludes, which win over global): + +* git -- ``/.git/info/exclude`` (per repo) and ``core.excludesFile`` from + the user/XDG git config (global; defaults to ``$XDG_CONFIG_HOME/git/ignore``). +* hg -- the ``[ui] ignore``/``ignore.*`` files named in ``/.hg/hgrc`` (per + repo) and in the user/XDG ``hgrc`` (global). + +Per-repo excludes are discovered while walking, so every repository inside the +search tree contributes its own. Globals are read once per search root. + +Limitations: ignore files are read only at and below each search root -- grin +does not walk *up* to a repository root above the search root (its paths are +relative to the root and cannot express ``..``), so a repo's per-repo excludes +apply only when its root is at or below where the search starts. +""" + +from collections.abc import Iterator +from dataclasses import dataclass +import os +import re + +from pathspec import GitIgnoreSpec +from pathspec.pattern import Pattern + +__all__ = ["IgnoreStack"] + +# gitignore-syntax files, in increasing precedence (later wins). +GITIGNORE_FILES = (".gitignore", ".ignore") +HGIGNORE_FILE = ".hgignore" +GIT_DIR = ".git" +HG_DIR = ".hg" + + +@dataclass +class _Rule: + """A single ignore rule, relative to ``base`` (a path relative to the search root).""" + + base: str + include: bool # True = ignore, False = negation (re-include) + pattern: Pattern | None = None # gitignore/glob rule (pathspec) + regex: re.Pattern[str] | None = None # .hgignore regexp rule + + def _relative(self, rel: str) -> str | None: + """Return ``rel`` made relative to this rule's base, or None if not under it.""" + if not self.base: + return rel + prefix = self.base + "/" + if rel.startswith(prefix): + return rel[len(prefix) :] + return None + + def matches(self, rel: str, is_dir: bool) -> bool: + sub = self._relative(rel) + if not sub: + return False + if self.pattern is not None: + if self.pattern.match_file(sub): + return True + # Directory-only patterns ("build/") match only with a trailing slash. + return is_dir and bool(self.pattern.match_file(sub + "/")) + if self.regex is not None: + return self.regex.search(sub) is not None + return False + + +def _gitignore_rules(lines: list[str], base: str) -> list[_Rule]: + rules: list[_Rule] = [] + try: + patterns = GitIgnoreSpec.from_lines(lines).patterns + except ValueError: + # A single malformed line (e.g. a lone "!") makes pathspec reject the + # whole file; git skips bad patterns, so compile line-by-line and drop + # the offenders instead of aborting the search. + patterns = [] + for line in lines: + try: + patterns.extend(GitIgnoreSpec.from_lines([line]).patterns) + except ValueError: + continue + for pattern in patterns: + # include is None for blank lines and comments. + if pattern.include is not None: + rules.append(_Rule(base=base, include=pattern.include, pattern=pattern)) + return rules + + +def _read_lines(path: str) -> list[str] | None: + try: + with open(path, encoding="utf-8", errors="replace") as fp: + return fp.read().splitlines() + except OSError: + return None + + +def load_dir_rules(dirpath: str, base: str, names: frozenset[str]) -> list[_Rule]: + """Load .gitignore then .ignore rules from ``dirpath`` (relative to ``base``).""" + rules: list[_Rule] = [] + for name in GITIGNORE_FILES: + if name in names: + lines = _read_lines(os.path.join(dirpath, name)) + if lines is not None: + rules.extend(_gitignore_rules(lines, base)) + return rules + + +def _hgignore_rules(lines: list[str], base: str) -> list[_Rule]: + """Parse Mercurial .hgignore-syntax ``lines`` into rules relative to ``base``.""" + rules: list[_Rule] = [] + glob_lines: list[str] = [] + syntax = "regexp" # Mercurial's default + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if stripped.lower().startswith("syntax:"): + value = stripped.split(":", 1)[1].strip().lower() + syntax = "glob" if value.startswith("glob") else "regexp" + continue + if syntax == "glob": + glob_lines.append(stripped) + else: + try: + rules.append(_Rule(base=base, include=True, regex=re.compile(stripped))) + except re.error: + continue + # hg glob patterns are rooted at the repo; approximate with gitwildmatch. + rules.extend(_gitignore_rules(glob_lines, base)) + return rules + + +def load_hgignore(path: str, base: str = "") -> list[_Rule]: + """Parse a Mercurial .hgignore file into rules relative to ``base``.""" + lines = _read_lines(path) + return _hgignore_rules(lines, base) if lines is not None else [] + + +def load_git_exclude(dirpath: str, base: str, names: frozenset[str]) -> list[_Rule]: + """Load ``dirpath/.git/info/exclude`` when ``dirpath`` is a git repo root.""" + if GIT_DIR not in names: + return [] + lines = _read_lines(os.path.join(dirpath, GIT_DIR, "info", "exclude")) + return _gitignore_rules(lines, base) if lines is not None else [] + + +def load_hg_repo_excludes(dirpath: str, base: str, names: frozenset[str]) -> list[_Rule]: + """Load the ignore files named in ``dirpath/.hg/hgrc`` (hg repo-local config).""" + if HG_DIR not in names: + return [] + rules: list[_Rule] = [] + for path in _hg_ignore_paths(os.path.join(dirpath, HG_DIR, "hgrc")): + path = os.path.expanduser(path) + if not os.path.isabs(path): + path = os.path.join(dirpath, path) + rules.extend(load_hgignore(path, base)) + return rules + + +# --- user/XDG VCS config (read once, lowest precedence, applied tree-wide) --- + + +def _config_kv(path: str) -> Iterator[tuple[str, str, str]]: + """Yield ``(section, key, value)`` from a git/hg-style INI config, lowercased + section/key. Subsection headers (``[core "x"]``) collapse to their section.""" + lines = _read_lines(path) + if lines is None: + return + section = "" + for raw in lines: + line = raw.strip() + if not line or line[0] in "#;": + continue + if line[0] == "[": + end = line.find("]") + if end != -1: + head = line[1:end].strip() + section = (head.split()[0] if head else "").strip('"').lower() + continue + if "=" in line: + key, value = line.split("=", 1) + value = value.strip() + if len(value) >= 2 and value[0] == '"' and value[-1] == '"': + value = value[1:-1] + yield section, key.strip().lower(), value + + +def _config_paths(env_var: str, xdg_parts: tuple[str, ...], home_name: str) -> list[str]: + """Config file paths, lowest precedence first, honoring an env override.""" + override = os.environ.get(env_var) + if override is not None: + return [p for p in override.split(os.pathsep) if p] + home = os.path.expanduser("~") + xdg = os.environ.get("XDG_CONFIG_HOME") or os.path.join(home, ".config") + return [os.path.join(xdg, *xdg_parts), os.path.join(home, home_name)] + + +def load_git_global_excludes() -> list[_Rule]: + """Load patterns from git's ``core.excludesFile`` (or its XDG default).""" + excludes: str | None = None + for cfg in _config_paths("GIT_CONFIG_GLOBAL", ("git", "config"), ".gitconfig"): + for section, key, value in _config_kv(cfg): + if section == "core" and key == "excludesfile": + excludes = value # a later config file wins + if excludes is None: + home = os.path.expanduser("~") + xdg = os.environ.get("XDG_CONFIG_HOME") or os.path.join(home, ".config") + excludes = os.path.join(xdg, "git", "ignore") + lines = _read_lines(os.path.expanduser(excludes)) + return _gitignore_rules(lines, "") if lines is not None else [] + + +def _hg_ignore_paths(cfg_path: str) -> list[str]: + """The ``[ui] ignore`` / ``ignore.*`` file paths declared in an hg config.""" + paths: list[str] = [] + for section, key, value in _config_kv(cfg_path): + if section == "ui" and value and (key == "ignore" or key.startswith("ignore.")): + paths.append(value) + return paths + + +def load_hg_global_excludes() -> list[_Rule]: + """Load the ignore files named by ``[ui] ignore`` in the user/XDG hg config.""" + rules: list[_Rule] = [] + for cfg in _config_paths("HGRCPATH", ("hg", "hgrc"), ".hgrc"): + for path in _hg_ignore_paths(cfg): + lines = _read_lines(os.path.expanduser(path)) + if lines is not None: + rules.extend(_hgignore_rules(lines, "")) + return rules + + +@dataclass +class IgnoreStack: + """Accumulated ignore rules from a search root down to the current directory. + + Immutable: :meth:`push` returns a new stack. Rules are stored in order + (root first, deeper directories later) and evaluated with git precedence: + the last matching rule wins, so a deeper or later ``!`` negation re-includes. + User-global excludes sit at the bottom; per-repo and in-tree rules added + while walking override them. + """ + + rules: tuple[_Rule, ...] = () + + @classmethod + def for_root(cls, root: str) -> "IgnoreStack": + # Globals only -- the lowest precedence. Per-directory ignore files and + # per-repo excludes (including the root's own) are added by push(). + return cls(tuple(load_git_global_excludes()) + tuple(load_hg_global_excludes())) + + def push(self, dirpath: str, base: str, names: frozenset[str]) -> "IgnoreStack": + # Lower precedence first: per-repo excludes, then in-tree ignore files, + # so an in-tree .gitignore/.ignore overrides a per-repo exclude here. + new_rules: list[_Rule] = [] + new_rules += load_git_exclude(dirpath, base, names) + new_rules += load_hg_repo_excludes(dirpath, base, names) + if HGIGNORE_FILE in names: + new_rules += load_hgignore(os.path.join(dirpath, HGIGNORE_FILE), base) + new_rules += load_dir_rules(dirpath, base, names) + if not new_rules: + return self + return IgnoreStack(self.rules + tuple(new_rules)) + + def match(self, rel: str, is_dir: bool) -> bool: + ignored = False + for rule in self.rules: + if rule.matches(rel, is_dir): + ignored = rule.include + return ignored diff --git a/grin/options.py b/grin/options.py index c550cf0..298f20b 100644 --- a/grin/options.py +++ b/grin/options.py @@ -57,3 +57,4 @@ class Options: files_from_file: str | None = None null_separated: bool = False include: str = "*" + respect_ignore_files: bool = True diff --git a/grin/recognizer.py b/grin/recognizer.py index c6adba3..e8ffb35 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -32,6 +32,7 @@ import stat from typing import Generator, IO +from .ignore import IgnoreStack from .options import Options from .types import DirEntry from .utils import is_binary_string @@ -78,6 +79,7 @@ class FileRecognizer: skip_symlink_files: bool = True binary_bytes: int = 4096 include: str | None = None + respect_ignore: bool = False skip_exts_simple: set[str] = field(init=False) skip_exts_endswith: list[str] = field(init=False) @@ -275,25 +277,54 @@ def walk(self, startpath: str, direntry: DirEntry = None) -> Generator[tuple[str ---------- startpath : str + When ``respect_ignore`` is set, .gitignore/.ignore/.hgignore files + encountered at and below ``startpath`` are honored, along with each + repo's per-repo excludes (git ``.git/info/exclude``, hg ``.hg/hgrc``) + and the user-global git/hg excludes (the start path itself is never + ignored). See :mod:`grin.ignore`. + Yields ------ filename : str kind : str """ + ignore = IgnoreStack.for_root(startpath) if self.respect_ignore else None + yield from self._walk(startpath, direntry, ignore, startpath) + + def _walk( + self, + startpath: str, + direntry: DirEntry, + ignore: IgnoreStack | None, + root: str, + ) -> Generator[tuple[str, str], None, None]: kind = self.recognize(startpath, direntry) if kind in ("binary", "text", "gzip"): yield startpath, kind # Not a directory, so there is no need to recurse. return - elif kind == "directory": - try: - entries = os.scandir(startpath) - except OSError: - return - for entry in sorted(entries, key=lambda e: e.name): - path = os.path.join(startpath, entry.name) - for fn, k in self.walk(path, entry): - yield fn, k + if kind != "directory": + return + try: + entries = sorted(os.scandir(startpath), key=lambda e: e.name) + except OSError: + return + if ignore is not None: + # Pass the entry names so push() reads only the ignore/config files + # that actually exist here (and detects .git/.hg repo roots) without + # a blind open() per directory. + ignore = ignore.push(startpath, relbase, frozenset(entry.name for entry in entries)) + for entry in entries: + path = os.path.join(startpath, entry.name) + if ignore is not None: + rel = os.path.relpath(path, root).replace(os.sep, "/") + try: + is_dir = entry.is_dir() + except OSError: + is_dir = False + if ignore.match(rel, is_dir): + continue + yield from self._walk(path, entry, ignore, rel) def get_recognizer(options: Options) -> FileRecognizer: @@ -324,4 +355,5 @@ def get_recognizer(options: Options) -> FileRecognizer: skip_symlink_files=options.skip_symlink_files, skip_symlink_dirs=options.skip_symlink_dirs, include=options.include, + respect_ignore=options.respect_ignore_files, ) diff --git a/pyproject.toml b/pyproject.toml index b2ac31f..d9449d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ description = "A grep program configured the way I like it. (python3 port)" readme = "README.md" requires-python = ">=3.10" license = { file = "LICENSE.md" } +dependencies = ["pathspec"] authors = [{ name = "Robert Kern", email = "robert.kern@enthought.com" }] maintainers = [{ name = "Raffaele Salmaso", email = "raffaele@salmaso.org" }] classifiers = [ diff --git a/tests/test_ignore.py b/tests/test_ignore.py new file mode 100644 index 0000000..f868e43 --- /dev/null +++ b/tests/test_ignore.py @@ -0,0 +1,236 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Test that grin honors .gitignore / .ignore / .hgignore files while walking. +""" + +import os +from tempfile import TemporaryDirectory +from unittest import mock, TestCase + +from grin import FileRecognizer + + +def build_tree(root: str, files: dict[str, str]) -> None: + for rel, content in files.items(): + path = os.path.join(root, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fp: + fp.write(content) + + +class _IsolatedEnvTestCase(TestCase): + """Point the global git/hg config env at an empty home so the developer's + real ~/.gitignore / global excludes never leak into the ignore tests.""" + + def setUp(self) -> None: + super().setUp() + tmp = TemporaryDirectory() + self.addCleanup(tmp.cleanup) + home = tmp.name + nowhere = os.path.join(home, "nonexistent") + patcher = mock.patch.dict( + os.environ, + { + "HOME": home, + "XDG_CONFIG_HOME": os.path.join(home, "config"), + "GIT_CONFIG_GLOBAL": nowhere, + "HGRCPATH": nowhere, + }, + ) + patcher.start() + self.addCleanup(patcher.stop) + + +# (name, files-to-create, paths-that-must-be-ignored, paths-that-must-be-kept) +CASES: list[tuple[str, dict[str, str], set[str], set[str]]] = [ + ( + "gitignore glob, negation and directory", + { + ".gitignore": "*.log\n!keep.log\nbuild/\n", + "a.log": "x", + "keep.log": "x", + "a.txt": "x", + "build/x.txt": "x", + }, + {"a.log", "build/x.txt"}, + {"keep.log", "a.txt"}, + ), + ( + "anchored pattern matches only at root", + {".gitignore": "/root.txt\n", "root.txt": "x", "sub/root.txt": "x"}, + {"root.txt"}, + {"sub/root.txt"}, + ), + ( + "nested .gitignore in subdir", + { + ".gitignore": "*.tmp\n", + "a.tmp": "x", + "sub/.gitignore": "local\n", + "sub/local": "x", + "sub/keep.py": "x", + }, + {"a.tmp", "sub/local"}, + {"sub/keep.py"}, + ), + ( + ".ignore overrides .gitignore", + { + ".gitignore": "*.data\n", + ".ignore": "!important.data\n", + "a.data": "x", + "important.data": "x", + }, + {"a.data"}, + {"important.data"}, + ), + ( + "hgignore glob and regexp sections", + { + ".hgignore": "syntax: glob\n*.orig\nsyntax: regexp\n\\.bak$\n", + "f.orig": "x", + "f.bak": "x", + "f.txt": "x", + }, + {"f.orig", "f.bak"}, + {"f.txt"}, + ), + ( + "hgignore applies repo-wide (subdirs)", + {".hgignore": "\\.pyc$\n", "sub/a.pyc": "x", "sub/a.py": "x"}, + {"sub/a.pyc"}, + {"sub/a.py"}, + ), + ( + # A lone "!" is an invalid git pattern; pathspec rejects the whole file + # for it. grin must skip the bad line (like git) instead of aborting the + # walk, and the valid line in the same file must still apply. + "invalid gitignore line is skipped, not fatal", + {".gitignore": "!\n*.log\n", "a.log": "x", "a.txt": "x"}, + {"a.log"}, + {"a.txt"}, + ), +] + + +class IgnoreWalkTestCase(_IsolatedEnvTestCase): + def _walk(self, files: dict[str, str], respect_ignore: bool) -> set[str]: + with TemporaryDirectory() as root: + build_tree(root, files) + fr = FileRecognizer(respect_ignore=respect_ignore) + return {os.path.relpath(path, root).replace(os.sep, "/") for path, _ in fr.walk(root)} + + def test_ignored_paths_are_skipped(self) -> None: + for name, files, ignored, kept in CASES: + with self.subTest(case=name): + walked = self._walk(files, respect_ignore=True) + self.assertEqual(walked & ignored, set(), "should be ignored") + self.assertEqual(kept - walked, set(), "should be kept") + + def test_nothing_ignored_when_disabled(self) -> None: + for name, files, ignored, kept in CASES: + with self.subTest(case=name): + walked = self._walk(files, respect_ignore=False) + self.assertEqual((ignored | kept) - walked, set()) + + +class RepoAndGlobalExcludesTestCase(_IsolatedEnvTestCase): + """Per-repo (.git/info/exclude, .hg/hgrc) and user-global git/hg excludes.""" + + def _walk(self, files: dict[str, str], env: dict[str, str] | None = None) -> set[str]: + with TemporaryDirectory() as root: + build_tree(root, files) + # Skip the VCS dirs while walking (as grin does by default); the + # excludes inside them are still read directly, not by recursing. + with mock.patch.dict(os.environ, env or {}): + fr = FileRecognizer(respect_ignore=True, skip_dirs={".git", ".hg"}) + return {os.path.relpath(path, root).replace(os.sep, "/") for path, _ in fr.walk(root)} + + def test_git_info_exclude_is_per_repo(self) -> None: + walked = self._walk( + { + "repo1/.git/info/exclude": "*.log\n", + "repo1/a.log": "x", + "repo1/keep.py": "x", + "repo2/a.log": "x", # different dir, no .git -> not affected + }, + ) + self.assertNotIn("repo1/a.log", walked) + self.assertIn("repo1/keep.py", walked) + self.assertIn("repo2/a.log", walked) + + def test_git_global_excludesfile(self) -> None: + with TemporaryDirectory() as cfg: + ignore = os.path.join(cfg, "ignore") + with open(ignore, "w", encoding="utf-8") as fp: + fp.write("*.tmp\n") + gitconfig = os.path.join(cfg, "gitconfig") + with open(gitconfig, "w", encoding="utf-8") as fp: + fp.write(f"[core]\n\texcludesfile = {ignore}\n") + walked = self._walk({"a.tmp": "x", "a.py": "x"}, env={"GIT_CONFIG_GLOBAL": gitconfig}) + self.assertNotIn("a.tmp", walked) + self.assertIn("a.py", walked) + + def test_git_global_excludes_xdg_default(self) -> None: + with TemporaryDirectory() as xdg: + gitdir = os.path.join(xdg, "git") + os.makedirs(gitdir) + with open(os.path.join(gitdir, "ignore"), "w", encoding="utf-8") as fp: + fp.write("*.tmp\n") + # No excludesFile set -> git falls back to $XDG_CONFIG_HOME/git/ignore. + walked = self._walk( + {"a.tmp": "x", "a.py": "x"}, + env={"XDG_CONFIG_HOME": xdg, "GIT_CONFIG_GLOBAL": os.path.join(xdg, "none")}, + ) + self.assertNotIn("a.tmp", walked) + self.assertIn("a.py", walked) + + def test_hg_global_excludes(self) -> None: + with TemporaryDirectory() as cfg: + hgignore = os.path.join(cfg, "hgignore") + with open(hgignore, "w", encoding="utf-8") as fp: + fp.write("syntax: glob\n*.bak\n") + hgrc = os.path.join(cfg, "hgrc") + with open(hgrc, "w", encoding="utf-8") as fp: + fp.write(f"[ui]\nignore = {hgignore}\n") + walked = self._walk({"a.bak": "x", "a.py": "x"}, env={"HGRCPATH": hgrc}) + self.assertNotIn("a.bak", walked) + self.assertIn("a.py", walked) + + def test_hg_repo_local_config_ignore(self) -> None: + walked = self._walk( + { + "repo/.hg/hgrc": "[ui]\nignore.extra = extra.hgignore\n", + "repo/extra.hgignore": "syntax: glob\n*.orig\n", + "repo/a.orig": "x", + "repo/a.py": "x", + }, + ) + self.assertNotIn("repo/a.orig", walked) + self.assertIn("repo/a.py", walked) diff --git a/tox.ini b/tox.ini index e44db86..ab923c0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,14 @@ [tox] requires = tox-uv>=1 envlist = py310, py311, py312, py313, py314, py315, pypy3, ruff, mypy -skipsdist = True skip_missing_interpreters = True [testenv] +# editable install so the project's runtime deps (pathspec) are available +package = editable commands = python -m unittest discover tests setenv = PYTHONDONTWRITEBYTECODE = 1 -usedevelop = True [testenv:ruff] changedir = {toxinidir} @@ -24,4 +24,5 @@ changedir = {toxinidir} skip_install = True deps = mypy + pathspec commands = mypy grin/ tests/ examples/ From fc78523ce4af9cb29cdcaaedeae5977c1eb2790f Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 17:43:55 +0200 Subject: [PATCH 20/37] Speed up walk: thread relative path instead of per-entry relpath --- CHANGELOG.md | 1 + grin/recognizer.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f3d7c1..e568f69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ * skip .git and more VCS dirs by default (.git, .jj, .sl, .pijul, _darcs, SCCS) * honor .gitignore/.ignore/.hgignore by default (--no-ignore to disable) also honor per-repo and user-global VCS excludes: git .git/info/exclude and core.excludesFile, hg .hg/hgrc and global hgrc [ui] ignore +* speed up walking: no per-entry path resolution ## 2.6.1 diff --git a/grin/recognizer.py b/grin/recognizer.py index e8ffb35..f2958a4 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -296,8 +296,11 @@ def _walk( startpath: str, direntry: DirEntry, ignore: IgnoreStack | None, - root: str, + relbase: str, ) -> Generator[tuple[str, str], None, None]: + # ``relbase`` is the POSIX path of ``startpath`` relative to the search + # root (""/at the root). It is threaded down the recursion so we never + # call the costly os.path.relpath()/abspath() per entry. kind = self.recognize(startpath, direntry) if kind in ("binary", "text", "gzip"): yield startpath, kind @@ -316,8 +319,8 @@ def _walk( ignore = ignore.push(startpath, relbase, frozenset(entry.name for entry in entries)) for entry in entries: path = os.path.join(startpath, entry.name) + rel = entry.name if not relbase else f"{relbase}/{entry.name}" if ignore is not None: - rel = os.path.relpath(path, root).replace(os.sep, "/") try: is_dir = entry.is_dir() except OSError: From 920f27eadfbee8a2ebaa2e301fa65ce422d81cb6 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Thu, 25 Jun 2026 20:57:58 +0200 Subject: [PATCH 21/37] Single open per file: classify binary/gzip while grepping (grep_path) --- CHANGELOG.md | 2 +- grin/grin.py | 23 +++++++------------ grin/grind.py | 4 +++- grin/main.py | 40 ++++++++++++++++++++++++++++++++- grin/recognizer.py | 17 ++++++++++++-- tests/test_grep.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e568f69..180eb23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ * skip .git and more VCS dirs by default (.git, .jj, .sl, .pijul, _darcs, SCCS) * honor .gitignore/.ignore/.hgignore by default (--no-ignore to disable) also honor per-repo and user-global VCS excludes: git .git/info/exclude and core.excludesFile, hg .hg/hgrc and global hgrc [ui] ignore -* speed up walking: no per-entry path resolution +* speed up walking: one open per file (classify while grepping) and no per-entry path resolution ## 2.6.1 diff --git a/grin/grin.py b/grin/grin.py index 9f2ea0c..e4c6264 100644 --- a/grin/grin.py +++ b/grin/grin.py @@ -28,17 +28,15 @@ import argparse from dataclasses import dataclass import fnmatch -import gzip import os import re import shlex import sys -from typing import cast, Iterator +from typing import Iterator from .main import GrepText from .options import Options from .recognizer import get_recognizer -from .types import Opener from .utils import deprecate_option, get_regex __all__ = ["get_filenames", "get_grin_arg_parser"] @@ -49,7 +47,7 @@ class GrinOptions(Options): pass -def get_filenames(options: Options) -> Iterator[tuple[str, str]]: +def get_filenames(options: Options, sniff_contents: bool = True) -> Iterator[tuple[str, str]]: """Generate the filenames to grep. Parameters @@ -112,7 +110,7 @@ def get_filenames(options: Options) -> Iterator[tuple[str, str]]: # Go over our list of filenames and see if we can recognize each as # something we want to grep. - fr = get_recognizer(options) + fr = get_recognizer(options, sniff_contents=sniff_contents) for fn in files: # Special case text stdin. if fn == "-": @@ -453,16 +451,11 @@ def main(argv: list[str] | None = None) -> None: regex = get_regex(options) g = GrepText(regex, options) - openers: dict[str, Opener] = {"text": cast(Opener, open), "gzip": cast(Opener, gzip.open)} - for filename, kind in get_filenames(options): - if kind in ["text", "gzip"]: - try: - report = g.grep_a_file(filename, opener=openers[kind]) - except (OSError, EOFError): - if kind != "gzip": - raise # probably shouldn't happen; something weird - report = g.grep_a_file(filename, opener=openers["text"]) - sys.stdout.write(report) + # sniff_contents=False: files are classified (binary/gzip) and grepped + # in a single open by grep_path, instead of opening once to classify and + # again to read. + for filename, _kind in get_filenames(options, sniff_contents=False): + sys.stdout.write(g.grep_path(filename)) except KeyboardInterrupt: raise SystemExit(0) except IOError as e: diff --git a/grin/grind.py b/grin/grind.py index 95d9ff1..ca7a346 100644 --- a/grin/grind.py +++ b/grin/grind.py @@ -210,7 +210,9 @@ def main(argv: list[str] | None = None) -> None: options = get_grind_config(argv) # Define the output function. output = print_null if options.null_separated else print - fr = get_recognizer(options) + # grind lists every file (text/binary/gzip) and ignores the kind, so + # there is no need to open files to classify their contents. + fr = get_recognizer(options, sniff_contents=False) for dir in options.dirs: for filename, _ in fr.walk(dir): if fnmatch.fnmatch(os.path.basename(filename), options.glob): diff --git a/grin/main.py b/grin/main.py index 5835c75..eebfd85 100644 --- a/grin/main.py +++ b/grin/main.py @@ -35,12 +35,14 @@ import stat import sys from typing import cast +import zlib from . import types from .colors import STYLES from .datablock import DataBlock from .options import Options -from .utils import get_line_offsets, to_str +from .recognizer import GZIP_MAGIC +from .utils import get_line_offsets, is_binary_string, to_str __all__ = ["GrepText"] @@ -416,3 +418,39 @@ def grep_a_file(self, filename: str, opener: types.Opener = open, encoding: str if filename != "-": f.close() return self.report(unique_context, filename) + + def grep_path(self, filename: str, encoding: str = "utf8") -> str: + """Classify and grep a real file in a single open. + + Unlike :meth:`grep_a_file` (which delegates opening to a caller-supplied + ``opener``), this opens the file once, decides from the first block + whether it is binary (skipped, empty report) or gzip (decompressed), and + greps it -- avoiding the second open that content-classification used to + cost. Used by the grin CLI; ``-`` reads stdin as text. + """ + if filename == "-": + return self.report(self.do_grep(sys.stdin, encoding), "") + try: + with open(filename, "rb") as fp: + head = fp.read(self.options.binary_bytes) + if head[:2] == GZIP_MAGIC: + try: + with gzip.open(filename) as gz: + if is_binary_string(gz.read(self.options.binary_bytes)): + return "" + gz.seek(0) + context = self.do_grep(gz, encoding) + # BadGzipFile is an OSError, but a corrupt deflate stream + # raises zlib.error (not an OSError) -- treat both, plus a + # truncated file (EOFError), as "not searchable" rather than + # letting one bad file abort the whole search. + except (OSError, EOFError, zlib.error): + return "" + elif is_binary_string(head): + return "" + else: + fp.seek(0) + context = self.do_grep(fp, encoding) + except OSError: + return "" + return self.report(context, filename) diff --git a/grin/recognizer.py b/grin/recognizer.py index f2958a4..626d288 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -80,6 +80,7 @@ class FileRecognizer: binary_bytes: int = 4096 include: str | None = None respect_ignore: bool = False + sniff_contents: bool = True skip_exts_simple: set[str] = field(init=False) skip_exts_endswith: list[str] = field(init=False) @@ -255,6 +256,11 @@ def recognize_file(self, filename: str, direntry: DirEntry = None) -> str: for ext in self.skip_exts_endswith: if filename_nc.endswith(ext): return "skip" + if not self.sniff_contents: + # Defer binary/gzip detection to the reader, so the file is opened + # only once (see GrepText.grep_path). The candidate is reported as + # "text"; binary content is skipped at read time. + return "text" try: if self.is_binary(filename): if self.is_gzipped_text(filename): @@ -330,8 +336,14 @@ def _walk( yield from self._walk(path, entry, ignore, rel) -def get_recognizer(options: Options) -> FileRecognizer: - """Get the file recognizer object from the configured options.""" +def get_recognizer(options: Options, sniff_contents: bool = True) -> FileRecognizer: + """Get the file recognizer object from the configured options. + + Pass ``sniff_contents=False`` when the caller reads each file itself and can + classify binary/gzip from the first block (the grin CLI does this for a + single open per file); ``grind`` keeps the default so it can exclude binary + files without reading them twice. + """ # always convert to set skip_dirs = set( @@ -359,4 +371,5 @@ def get_recognizer(options: Options) -> FileRecognizer: skip_symlink_dirs=options.skip_symlink_dirs, include=options.include, respect_ignore=options.respect_ignore_files, + sniff_contents=sniff_contents, ) diff --git a/tests/test_grep.py b/tests/test_grep.py index 549621c..300ac6b 100644 --- a/tests/test_grep.py +++ b/tests/test_grep.py @@ -27,7 +27,9 @@ import gzip from io import BytesIO +import os import re +from tempfile import TemporaryDirectory from unittest import TestCase import grin @@ -527,3 +529,56 @@ def test_broken_gzip_plaintext(self) -> None: gt = grin.GrepText(re.compile("ar")) self.assertEqual(gt.do_grep(BytesIO(gzip_text_compressed_trailing_garbage)), [(2, 0, "arborist\n", [(0, 2)])]) + + +class GrepPathTestCase(TestCase): + """grep_path opens a file once and classifies binary/gzip from the first block.""" + + def setUp(self) -> None: + self._tmp = TemporaryDirectory() + self.root = self._tmp.name + self.gt = grin.GrepText(re.compile("needle")) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _write(self, name: str, data: bytes, *, gzipped: bool = False) -> str: + path = os.path.join(self.root, name) + if gzipped: + with gzip.open(path, "wb") as f: + f.write(data) + else: + with open(path, "wb") as f: + f.write(data) + return path + + def test_text_file_is_grepped(self) -> None: + path = self._write("a.txt", b"foo\nthe needle here\nbar\n") + self.assertIn("needle here", self.gt.grep_path(path)) + + def test_text_file_without_match_is_empty(self) -> None: + path = self._write("a.txt", b"foo\nbar\n") + self.assertEqual(self.gt.grep_path(path), "") + + def test_binary_file_is_skipped(self) -> None: + path = self._write("a.bin", b"junk\x00\x00needle\x00more") + self.assertEqual(self.gt.grep_path(path), "") + + def test_gzip_text_is_grepped(self) -> None: + path = self._write("a.gz", b"a needle in gzipped text\n", gzipped=True) + self.assertIn("needle in gzipped text", self.gt.grep_path(path)) + + def test_gzip_binary_is_skipped(self) -> None: + path = self._write("b.gz", b"prefix\x00\x00needle\x00bytes", gzipped=True) + self.assertEqual(self.gt.grep_path(path), "") + + def test_corrupt_gzip_is_skipped_not_fatal(self) -> None: + # A gzip magic header followed by a corrupt deflate stream raises + # zlib.error ("invalid code lengths set") -- which is neither OSError nor + # EOFError. grep_path must skip the file, not let it abort the search. + payload = bytes([0x1F, 0x8B, 0x08, 0x00, 0, 0, 0, 0, 0, 0xFF]) + bytes([0x05] * 30) + path = self._write("corrupt.gz", payload) + self.assertEqual(self.gt.grep_path(path), "") + + def test_missing_file_is_empty(self) -> None: + self.assertEqual(self.gt.grep_path(os.path.join(self.root, "nope.txt")), "") From 7d9518f68da333141e6b233851259d502b559849 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 18:32:52 +0200 Subject: [PATCH 22/37] Search files in parallel by default --- CHANGELOG.md | 1 + README.md | 11 +++++ grin/grin.py | 17 +++++++- grin/options.py | 1 + grin/parallel.py | 97 +++++++++++++++++++++++++++++++++++++++++ tests/test_parallel.py | 98 ++++++++++++++++++++++++++++++++++++++++++ tox.ini | 2 +- 7 files changed, 224 insertions(+), 3 deletions(-) create mode 100644 grin/parallel.py create mode 100644 tests/test_parallel.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 180eb23..5b47415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ * honor .gitignore/.ignore/.hgignore by default (--no-ignore to disable) also honor per-repo and user-global VCS excludes: git .git/info/exclude and core.excludesFile, hg .hg/hgrc and global hgrc [ui] ignore * speed up walking: one open per file (classify while grepping) and no per-entry path resolution +* search files in parallel by default (-j/--jobs, 1 to disable); processes on standard CPython, threads on free-threaded builds ## 2.6.1 diff --git a/README.md b/README.md index 11d8bc7..a3ac16a 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,17 @@ per-repo excludes apply only when its root is at or below where the search starts. Paths given explicitly on the command line are always searched, never ignored. +## Parallelism + +grin searches files in parallel by default, using one worker per CPU. The output +order is unchanged (still sorted by filename). Control the worker count with +`-j`/`--jobs`: + +```bash +$ grin -j4 some_regex # four workers +$ grin -j1 some_regex # sequential +``` + ## Using grind To find files matching the glob "foo\*.py" in this directory or any subdirectory diff --git a/grin/grin.py b/grin/grin.py index e4c6264..c37689e 100644 --- a/grin/grin.py +++ b/grin/grin.py @@ -36,6 +36,7 @@ from .main import GrepText from .options import Options +from .parallel import parallel_grep from .recognizer import get_recognizer from .utils import deprecate_option, get_regex @@ -389,6 +390,13 @@ def get_grin_arg_parser(parser: argparse.ArgumentParser | None = None) -> argpar default=True, help="do not honor .gitignore/.ignore/.hgignore files", ) + parser.add_argument( + "-J", + "--jobs", + type=int, + default=0, + help="number of files to search in parallel [default: number of CPUs; 1 disables]", + ) parser.add_argument( "-x", @@ -442,6 +450,7 @@ def get_grin_config(argv: list[str] | None = None) -> GrinOptions: include=args.include, sys_path=args.sys_path, respect_ignore_files=args.respect_ignore_files, + jobs=args.jobs, ) @@ -454,8 +463,12 @@ def main(argv: list[str] | None = None) -> None: # sniff_contents=False: files are classified (binary/gzip) and grepped # in a single open by grep_path, instead of opening once to classify and # again to read. - for filename, _kind in get_filenames(options, sniff_contents=False): - sys.stdout.write(g.grep_path(filename)) + filenames = (filename for filename, _kind in get_filenames(options, sniff_contents=False)) + # parallel_grep greps files concurrently (processes on a standard build, + # threads on a free-threaded one) while preserving the sorted-filename + # order, so the output is identical to a sequential run. + for report in parallel_grep(g, options, filenames, options.jobs): + sys.stdout.write(report) except KeyboardInterrupt: raise SystemExit(0) except IOError as e: diff --git a/grin/options.py b/grin/options.py index 298f20b..5b9e6f0 100644 --- a/grin/options.py +++ b/grin/options.py @@ -58,3 +58,4 @@ class Options: null_separated: bool = False include: str = "*" respect_ignore_files: bool = True + jobs: int = 0 diff --git a/grin/parallel.py b/grin/parallel.py new file mode 100644 index 0000000..5f0388f --- /dev/null +++ b/grin/parallel.py @@ -0,0 +1,97 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Grep many files concurrently, yielding reports in the original order. + +The bottleneck is GIL-bound Python (regex + report building), so on a standard +CPython build threads cannot help -- we use *processes* (separate GILs). On a +free-threaded build (``Py_GIL_DISABLED``) there is no GIL, so threads are used +(no pickling, shared memory). Either way results are yielded in input order. +""" + +from collections.abc import Iterable, Iterator +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +import os +import sysconfig + +from .main import GrepText +from .options import Options +from .utils import get_regex + +__all__ = ["parallel_grep", "resolve_jobs"] + +# True on free-threaded CPython (PEP 703), where threads run Python in parallel. +GIL_DISABLED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + +# Below this many files the pool's startup cost outweighs the win. +MIN_PARALLEL_FILES = 16 + +# Per-process GrepText, built once by the pool initializer (process backend). +_worker_grep: GrepText | None = None + + +def _init_worker(options: Options) -> None: + global _worker_grep + _worker_grep = GrepText(get_regex(options), options) + + +def _grep_in_worker(filename: str) -> str: + grep = _worker_grep + if grep is None: # the pool initializer always sets it; guard for type-checkers + raise RuntimeError("worker GrepText is not initialized") + return grep.grep_path(filename, _worker_encoding) + + +def resolve_jobs(jobs: int) -> int: + """Turn a ``--jobs`` value into a worker count: 0 means auto (CPU count).""" + if jobs > 0: + return jobs + return os.cpu_count() or 1 + + +def parallel_grep(grep_text: GrepText, options: Options, filenames: Iterable[str], jobs: int) -> Iterator[str]: + """Yield each file's grep report, in input order, grepping in parallel. + + ``grep_text`` is used directly for the sequential and thread backends; + ``options`` is sent to worker processes to rebuild a GrepText there (bound + methods are not picklable under the spawn/forkserver start methods). + """ + workers = resolve_jobs(jobs) + files = list(filenames) + if workers <= 1 or len(files) < MIN_PARALLEL_FILES: + for filename in files: + yield grep_text.grep_path(filename) + return + + # ~16 chunks per worker balances load while amortizing dispatch overhead. + chunksize = max(1, len(files) // (workers * 16)) + if GIL_DISABLED: + with ThreadPoolExecutor(max_workers=workers) as thread_pool: + yield from thread_pool.map(grep_text.grep_path, files, chunksize=chunksize) + else: + with ProcessPoolExecutor(max_workers=workers, initializer=_init_worker, initargs=(options,)) as process_pool: + yield from process_pool.map(_grep_in_worker, files, chunksize=chunksize) diff --git a/tests/test_parallel.py b/tests/test_parallel.py new file mode 100644 index 0000000..1e16c51 --- /dev/null +++ b/tests/test_parallel.py @@ -0,0 +1,98 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Test parallel search: results match a sequential run and stay in input order. +""" + +import contextlib +import io +import os +import re +from tempfile import TemporaryDirectory +from unittest import TestCase + +import grin +import grin.grin +from grin.parallel import parallel_grep, resolve_jobs + + +def make_tree(root: str, count: int) -> None: + for i in range(count): + with open(os.path.join(root, f"f{i:03d}.txt"), "w", encoding="utf-8") as fp: + fp.write(f"a line\nneedle {i}\nanother line\n") + + +def filenames(root: str) -> list[str]: + return sorted(os.path.join(root, name) for name in os.listdir(root)) + + +def grep(root: str, jobs: int) -> list[str]: + grep_text = grin.GrepText(re.compile("needle"), grin.Options()) + options = grin.Options(regex="needle") # for worker processes to rebuild + return list(parallel_grep(grep_text, options, filenames(root), jobs)) + + +class ParallelGrepTestCase(TestCase): + def test_parallel_matches_sequential(self) -> None: + with TemporaryDirectory() as root: + make_tree(root, 40) + self.assertEqual(grep(root, 4), grep(root, 1)) + + def test_order_preserved(self) -> None: + with TemporaryDirectory() as root: + make_tree(root, 40) + reports = grep(root, 4) + self.assertEqual(len(reports), 40) + for i, report in enumerate(reports): + self.assertIn(f"needle {i}", report) + + def test_small_input_uses_sequential_path(self) -> None: + with TemporaryDirectory() as root: + make_tree(root, 3) # below MIN_PARALLEL_FILES + self.assertEqual(grep(root, 8), grep(root, 1)) + + def test_resolve_jobs(self) -> None: + self.assertEqual(resolve_jobs(3), 3) + self.assertEqual(resolve_jobs(1), 1) + self.assertGreaterEqual(resolve_jobs(0), 1) + + +class GrinParallelOutputTestCase(TestCase): + def _run(self, target: str, jobs: int) -> str: + buf = io.StringIO() + with contextlib.redirect_stdout(buf), contextlib.suppress(SystemExit): + grin.grin.main(["grin", "--no-color", "-J", str(jobs), "needle", target]) + return buf.getvalue() + + def test_parallel_output_matches_sequential(self) -> None: + with TemporaryDirectory() as root: + make_tree(root, 40) + os.mkdir(os.path.join(root, "sub")) + make_tree(os.path.join(root, "sub"), 10) + self.assertEqual(self._run(root, 8), self._run(root, 1)) + self.assertIn("needle 0", self._run(root, 1)) diff --git a/tox.ini b/tox.ini index ab923c0..f7b2889 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox-uv>=1 -envlist = py310, py311, py312, py313, py314, py315, pypy3, ruff, mypy +envlist = py310, py311, py312, py313, py314, py315, py313t, py314t, py315t, pypy3, ruff, mypy skip_missing_interpreters = True [testenv] From e94345d5ff294f5f277df4d70738160b6d77d0a1 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Fri, 26 Jun 2026 00:34:53 +0200 Subject: [PATCH 23/37] Skip paths with an embedded NUL while walking --- CHANGELOG.md | 1 + grin/recognizer.py | 6 +++++- tests/test_file_recognizer.py | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b47415..11fa850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ also honor per-repo and user-global VCS excludes: git .git/info/exclude and core.excludesFile, hg .hg/hgrc and global hgrc [ui] ignore * speed up walking: one open per file (classify while grepping) and no per-entry path resolution * search files in parallel by default (-j/--jobs, 1 to disable); processes on standard CPython, threads on free-threaded builds +* skip (instead of crashing on) paths with an embedded NUL while walking ## 2.6.1 diff --git a/grin/recognizer.py b/grin/recognizer.py index 626d288..8b1fad1 100644 --- a/grin/recognizer.py +++ b/grin/recognizer.py @@ -213,7 +213,11 @@ def recognize(self, filename: str, direntry: DirEntry = None) -> str: return self.recognize_directory(filename, direntry) else: return "skip" - except OSError: + except (OSError, ValueError): + # OSError: the usual "cannot stat/read" case. ValueError: os.stat + # rejects a path with an embedded NUL ("embedded null character in + # path") -- a malformed entry (e.g. from --files-from-file) must be + # skipped, not abort the whole walk. return "unreadable" def recognize_directory(self, filename: str, direntry: DirEntry = None) -> str: diff --git a/tests/test_file_recognizer.py b/tests/test_file_recognizer.py index ba23097..a73d0fe 100644 --- a/tests/test_file_recognizer.py +++ b/tests/test_file_recognizer.py @@ -321,6 +321,11 @@ def test_socket(self) -> None: fr = FileRecognizer() self.assertEqual(self._recognize("socket_test", fr), "skip") + def test_embedded_nul_path_is_unreadable(self) -> None: + # os.stat() raises ValueError (not OSError) on a path with an embedded + # NUL; a malformed entry must be skipped, not abort the walk. + self.assertEqual(FileRecognizer().recognize("bad\0name", None), "unreadable") + def test_dir(self) -> None: fr = FileRecognizer() self.assertEqual(self._recognize_directory("dir", fr), "directory") From 0d8dce111ab68edad6bb42708d61559f71cb2985 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 18:32:17 +0200 Subject: [PATCH 24/37] Expand the test suite and add a coverage tox env --- .gitignore | 2 + .hgignore | 2 + CHANGELOG.md | 1 + pyproject.toml | 9 +- tests/__init__.py | 0 tests/_helpers.py | 99 +++++++++++++++ tests/test_cli.py | 225 ++++++++++++++++++++++++++++++++++ tests/test_colors.py | 85 +++++++++++++ tests/test_datablock.py | 60 +++++++++ tests/test_file_recognizer.py | 75 ++++++++++-- tests/test_grep.py | 127 ++++++++++++++++++- tests/test_grind.py | 141 +++++++++++++++++++++ tests/test_ignore.py | 39 +----- tests/test_parallel.py | 11 +- tests/test_utils.py | 137 +++++++++++++++++++++ tox.ini | 9 ++ 16 files changed, 971 insertions(+), 51 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/_helpers.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_colors.py create mode 100644 tests/test_datablock.py create mode 100644 tests/test_grind.py create mode 100644 tests/test_utils.py diff --git a/.gitignore b/.gitignore index 20ff8f8..f8b80e9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ __pycache__ *.bak *.pyc *.orig +.coverage +.coverage.* diff --git a/.hgignore b/.hgignore index 821c2e7..6186140 100644 --- a/.hgignore +++ b/.hgignore @@ -18,3 +18,5 @@ syntax: glob *.pyc *.kpf *.orig +.coverage +.coverage.* diff --git a/CHANGELOG.md b/CHANGELOG.md index 11fa850..14366e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * speed up walking: one open per file (classify while grepping) and no per-entry path resolution * search files in parallel by default (-j/--jobs, 1 to disable); processes on standard CPython, threads on free-threaded builds * skip (instead of crashing on) paths with an embedded NUL while walking +* expand the test suite (grind, colors, datablock, utils, CLI) and add a coverage tox env ## 2.6.1 diff --git a/pyproject.toml b/pyproject.toml index d9449d1..20cc010 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,14 @@ grin = "grin.grin:main" grind = "grin.grind:main" [dependency-groups] -dev = ["build", "mypy", "ruff", "tox", "tox-uv", "twine"] +dev = ["build", "coverage", "mypy", "ruff", "tox", "tox-uv", "twine"] + +[tool.coverage.run] +branch = true +source = ["grin"] + +[tool.coverage.report] +show_missing = true [tool.hatch.version] path = "grin/__init__.py" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/_helpers.py b/tests/_helpers.py new file mode 100644 index 0000000..21ae220 --- /dev/null +++ b/tests/_helpers.py @@ -0,0 +1,99 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Shared helpers for the grin test suite: a chdir context manager, a fixture-tree +builder, a CLI stdout-capture runner, and an environment-isolation mixin that +keeps the developer's real global git/hg ignore configuration out of tests that +walk directories with ignore handling enabled. +""" + +import contextlib +import io +import os +from tempfile import TemporaryDirectory +from typing import Callable, Generator +from unittest import mock, TestCase + + +@contextlib.contextmanager +def cd(newdir: str) -> Generator[None, None, None]: + """Temporarily change the working directory (restored even on exception).""" + prevdir = os.getcwd() + os.chdir(os.path.expanduser(newdir)) + try: + yield + finally: + os.chdir(prevdir) + + +def build_tree(root: str, files: list[str] | dict[str, str], *, content: str = "x") -> None: + """Create files under ``root`` (parent directories made as needed). + + ``files`` is either a list of ``/``-separated relative paths (each written + with ``content``) or a mapping of relative path -> file contents.""" + entries = list(files.items()) if isinstance(files, dict) else [(rel, content) for rel in files] + for rel, data in entries: + path = os.path.join(root, *rel.split("/")) + os.makedirs(os.path.dirname(path) or root, exist_ok=True) + with open(path, "w", encoding="utf-8") as fp: + fp.write(data) + + +def run_cli(main: Callable[[list[str]], None], *argv: str) -> str: + """Run a grin/grind ``main()`` with stdout captured, swallowing SystemExit.""" + buf = io.StringIO() + with contextlib.redirect_stdout(buf), contextlib.suppress(SystemExit): + main(list(argv)) + return buf.getvalue() + + +class IsolatedVCSEnvMixin(TestCase): + """Point the global git/hg config env at an empty home so the developer's + real ~/.gitignore / global excludes never leak into walking tests. + + Mix this into any TestCase that walks a tree with ignore handling enabled + (FileRecognizer(respect_ignore=True), or the default grin/grind paths where + respect_ignore_files is True). Subclass setUp() must call super().setUp().""" + + def setUp(self) -> None: + super().setUp() + tmp = TemporaryDirectory() + self.addCleanup(tmp.cleanup) + home = tmp.name + nowhere = os.path.join(home, "nonexistent") + patcher = mock.patch.dict( + os.environ, + { + "HOME": home, + "XDG_CONFIG_HOME": os.path.join(home, "config"), + "GIT_CONFIG_GLOBAL": nowhere, + "HGRCPATH": nowhere, + }, + ) + patcher.start() + self.addCleanup(patcher.stop) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..1728937 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,225 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Tests for the grin command-line surface: argument parsing, configuration +(including the GRIN_ARGS environment variable and color handling), the +get_filenames() file-stream generator, and end-to-end runs of main(). +""" + +import argparse +import io +import os +import re +from tempfile import TemporaryDirectory +from unittest import mock, TestCase + +import grin.grin +from grin.grin import get_filenames, get_grin_arg_parser, get_grin_config +from grin.options import Options +from tests._helpers import build_tree, cd, IsolatedVCSEnvMixin, run_cli + +# Files whose second line contains "needle", so grin matches on line 2. +NEEDLE_CONTENT = "a line\nthe needle here\nanother line\n" + + +def run_grin(*argv: str) -> str: + return run_cli(grin.grin.main, "grin", *argv) + + +class GrinArgParserTestCase(TestCase): + def parse(self, argv: list[str]) -> argparse.Namespace: + return get_grin_arg_parser().parse_args(argv) + + def test_regex_flags_accumulate(self) -> None: + args = self.parse(["-a", "-i", "foo"]) + self.assertIn(re.A, args.re_flags) + self.assertIn(re.I, args.re_flags) + + def test_fixed_string_and_word(self) -> None: + args = self.parse(["-F", "-w", "foo"]) + self.assertTrue(args.fixed_string) + self.assertTrue(args.word_regexp) + + def test_context_options(self) -> None: + self.assertEqual(self.parse(["-A", "2", "foo"]).after_context, 2) + self.assertEqual(self.parse(["-B", "3", "foo"]).before_context, 3) + self.assertEqual(self.parse(["-C", "4", "foo"]).context, 4) + self.assertIsNone(self.parse(["foo"]).context) + + def test_output_flags(self) -> None: + self.assertFalse(self.parse(["-N", "foo"]).show_line_numbers) + self.assertFalse(self.parse(["--no-filename", "foo"]).show_filename) + self.assertTrue(self.parse(["--emacs", "foo"]).show_emacs) + self.assertFalse(self.parse(["-l", "foo"]).show_match) + + def test_jobs_and_include(self) -> None: + self.assertEqual(self.parse(["-J", "4", "foo"]).jobs, 4) + self.assertEqual(self.parse(["-I", "*.py", "foo"]).include, "*.py") + + def test_color_choice(self) -> None: + self.assertEqual(self.parse(["--color", "always", "foo"]).color, "always") + + +class GrinConfigTestCase(TestCase): + def test_context_sets_both_sides(self) -> None: + options = get_grin_config(["grin", "-C", "2", "foo"]) + self.assertEqual(options.before_context, 2) + self.assertEqual(options.after_context, 2) + + def test_jobs_and_regex_mapped(self) -> None: + options = get_grin_config(["grin", "-J", "3", "foo"]) + self.assertEqual(options.jobs, 3) + self.assertEqual(options.regex, "foo") + + def test_grin_args_env_is_prepended(self) -> None: + with mock.patch("sys.argv", ["grin", "foo"]), mock.patch.dict(os.environ, {"GRIN_ARGS": "-i"}): + options = get_grin_config(None) + self.assertIn(re.I, options.re_flags) + + def test_force_color_enables_color_without_tty(self) -> None: + options = get_grin_config(["grin", "--force-color", "foo"]) + self.assertTrue(options.use_color) + + def test_no_color_disables_color(self) -> None: + with mock.patch("sys.stdout.isatty", return_value=True): + options = get_grin_config(["grin", "--no-color", "foo"]) + self.assertFalse(options.use_color) + + def test_color_follows_tty_detection(self) -> None: + with mock.patch("sys.stdout.isatty", return_value=True), mock.patch.dict(os.environ, {"TERM": "xterm"}): + self.assertTrue(get_grin_config(["grin", "foo"]).use_color) + with mock.patch("sys.stdout.isatty", return_value=False), mock.patch.dict(os.environ, {"TERM": "xterm"}): + self.assertFalse(get_grin_config(["grin", "foo"]).use_color) + + def test_dumb_terminal_disables_color(self) -> None: + with mock.patch("sys.stdout.isatty", return_value=True), mock.patch.dict(os.environ, {"TERM": "dumb"}): + self.assertFalse(get_grin_config(["grin", "foo"]).use_color) + + +class GetFilenamesTestCase(IsolatedVCSEnvMixin): + def setUp(self) -> None: + super().setUp() + self._tmp = TemporaryDirectory() + self.root = self._tmp.name + build_tree(self.root, ["a.py", "b.txt", "sub/c.py"], content=NEEDLE_CONTENT) + self.addCleanup(self._tmp.cleanup) + + def _names(self, options: Options) -> list[str]: + return sorted(os.path.basename(fn) for fn, _kind in get_filenames(options)) + + def test_directory_is_walked(self) -> None: + self.assertEqual(self._names(Options(files=[self.root])), ["a.py", "b.txt", "c.py"]) + + def test_include_glob_filters(self) -> None: + self.assertEqual(self._names(Options(files=[self.root], include="*.py")), ["a.py", "c.py"]) + + def test_explicit_file(self) -> None: + path = os.path.join(self.root, "a.py") + self.assertEqual(list(get_filenames(Options(files=[path]))), [(path, "text")]) + + def test_stdin_dash_is_text(self) -> None: + self.assertEqual(list(get_filenames(Options(files=["-"]))), [("-", "text")]) + + def test_skips_empty_and_dev_null(self) -> None: + path = os.path.join(self.root, "a.py") + names = [fn for fn, _ in get_filenames(Options(files=["", "/dev/null", path]))] + self.assertEqual(names, [path]) + + def test_empty_defaults_to_cwd(self) -> None: + with cd(self.root): + self.assertEqual(self._names(Options()), ["a.py", "b.txt", "c.py"]) + + def test_files_from_file_path(self) -> None: + # The listing names exactly a.py and c.py, so only those two are yielded + # (the default include="*" does not filter here). + listing = os.path.join(self.root, "list.txt") + targets = [os.path.join(self.root, "a.py"), os.path.join(self.root, "sub", "c.py")] + with open(listing, "w", encoding="utf-8") as fp: + fp.write("\n".join(targets) + "\n") + names = sorted(os.path.basename(fn) for fn, _ in get_filenames(Options(files_from_file=listing))) + self.assertEqual(names, ["a.py", "c.py"]) + + def test_files_from_stdin(self) -> None: + path = os.path.join(self.root, "a.py") + with mock.patch("sys.stdin", io.StringIO(path + "\n")): + names = list(get_filenames(Options(files_from_file="-"))) + self.assertEqual(names, [(path, "text")]) + + def test_files_from_stdin_null_separated(self) -> None: + targets = [os.path.join(self.root, "a.py"), os.path.join(self.root, "b.txt")] + # null mode: paths are NUL-joined (no newlines) and must split on NUL. + on = Options(files_from_file="-", null_separated=True) + with mock.patch("sys.stdin", io.StringIO("\0".join(targets))): + names = sorted(os.path.basename(fn) for fn, _ in get_filenames(on)) + self.assertEqual(names, ["a.py", "b.txt"]) + + # Control: the SAME two paths newline-joined, in line mode, must also be + # read as two files. Together these pin the delimiter to each mode -- an + # implementation hardcoded to one delimiter fails one of the two cases + # (the cross combinations would yield a single bogus path -> no files). + off = Options(files_from_file="-", null_separated=False) + with mock.patch("sys.stdin", io.StringIO("\n".join(targets) + "\n")): + names = sorted(os.path.basename(fn) for fn, _ in get_filenames(off)) + self.assertEqual(names, ["a.py", "b.txt"]) + + def test_missing_files_from_file_raises(self) -> None: + with self.assertRaises(IOError): + list(get_filenames(Options(files_from_file=os.path.join(self.root, "nope")))) + + def test_sys_path_entries_are_searched(self) -> None: + with mock.patch("sys.path", [self.root]): + self.assertEqual(self._names(Options(sys_path=True, include="*.py")), ["a.py", "c.py"]) + + +class GrinMainTestCase(IsolatedVCSEnvMixin): + def setUp(self) -> None: + super().setUp() + self._tmp = TemporaryDirectory() + self.root = self._tmp.name + build_tree(self.root, ["a.py", "sub/c.py"], content=NEEDLE_CONTENT) + self.addCleanup(self._tmp.cleanup) + + def test_matches_with_filename_and_line_number(self) -> None: + output = run_grin("--no-color", "needle", self.root) + self.assertIn("a.py", output) + # default output is ":\n 2 : the needle here\n" -- assert the + # numbered match line, not just the filename header. + self.assertIn(" 2 : the needle here", output) + + def test_emacs_format(self) -> None: + output = run_grin("--no-color", "--emacs", "needle", os.path.join(self.root, "a.py")) + path = os.path.join(self.root, "a.py") + self.assertIn("%s:2: the needle here" % path, output) + + def test_files_with_matches_only(self) -> None: + output = run_grin("--no-color", "-l", "needle", self.root) + self.assertIn("a.py", output) + self.assertNotIn("needle here", output) + + def test_no_match_produces_no_output(self) -> None: + self.assertEqual(run_grin("--no-color", "zzznomatch", self.root), "") diff --git a/tests/test_colors.py b/tests/test_colors.py new file mode 100644 index 0000000..b2e6bae --- /dev/null +++ b/tests/test_colors.py @@ -0,0 +1,85 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Tests for the ANSI color/style helpers in grin.colors. +""" + +from unittest import TestCase + +from grin.colors import Color, Style, STYLES + + +class StyleStartTestCase(TestCase): + def test_no_attributes_is_empty_sgr(self) -> None: + # No fragments -> an (empty) SGR sequence. + self.assertEqual(Style().start, "\x1b[m") + + def test_foreground_color_codes_30_to_39(self) -> None: + self.assertEqual(Style(fg=Color.RED).start, "\x1b[31m") + self.assertEqual(Style(fg=Color.DEFAULT).start, "\x1b[38m") + + def test_background_color_codes_40_to_49(self) -> None: + self.assertEqual(Style(bg=Color.YELLOW).start, "\x1b[43m") + + def test_bold_underline_reverse_flags(self) -> None: + self.assertEqual(Style(bold=True).start, "\x1b[1m") + self.assertEqual(Style(underline=True).start, "\x1b[4m") + self.assertEqual(Style(reverse=True).start, "\x1b[7m") + + def test_combination_preserves_fragment_order(self) -> None: + # Order is fg, bg, bold, underline, reverse. + style = Style(fg=Color.BLACK, bg=Color.YELLOW, bold=True, underline=True, reverse=True) + self.assertEqual(style.start, "\x1b[30;43;1;4;7m") + + +class StyleEndAndApplyTestCase(TestCase): + def test_end_is_reset(self) -> None: + self.assertEqual(Style(fg=Color.RED).end, "\x1b[0m") + + def test_apply_wraps_text(self) -> None: + style = Style(fg=Color.GREEN, bold=True) + self.assertEqual(style.apply("hello"), "\x1b[32;1mhello\x1b[0m") + + def test_apply_empty_text(self) -> None: + self.assertEqual(Style(fg=Color.RED).apply(""), "\x1b[31m\x1b[0m") + + +class StylesRegistryTestCase(TestCase): + def test_known_styles_present(self) -> None: + self.assertIn("filename", STYLES) + self.assertIn("searchterm", STYLES) + + def test_filename_style(self) -> None: + style = STYLES["filename"] + self.assertEqual(style.fg, Color.GREEN) + self.assertTrue(style.bold) + + def test_searchterm_style(self) -> None: + style = STYLES["searchterm"] + self.assertEqual(style.fg, Color.BLACK) + self.assertEqual(style.bg, Color.YELLOW) diff --git a/tests/test_datablock.py b/tests/test_datablock.py new file mode 100644 index 0000000..8fefd6d --- /dev/null +++ b/tests/test_datablock.py @@ -0,0 +1,60 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Tests for the DataBlock container used by the block-reading code in grin.main. +""" + +from unittest import TestCase + +from grin.datablock import DataBlock + + +class DataBlockTestCase(TestCase): + def test_defaults(self) -> None: + block = DataBlock() + self.assertEqual(block.data, "") + self.assertEqual(block.start, 0) + self.assertEqual(block.end, 0) + self.assertEqual(block.before_count, 0) + self.assertFalse(block.is_last) + + def test_default_blocks_compare_equal(self) -> None: + # main.EMPTY_DATABLOCK relies on default instances being equal. + self.assertEqual(DataBlock(), DataBlock()) + + def test_field_assignment(self) -> None: + block = DataBlock(data="abc\ndef\n", start=4, end=8, before_count=1, is_last=True) + self.assertEqual(block.data, "abc\ndef\n") + self.assertEqual(block.start, 4) + self.assertEqual(block.end, 8) + self.assertEqual(block.before_count, 1) + self.assertTrue(block.is_last) + + def test_equality_is_value_based(self) -> None: + self.assertEqual(DataBlock(data="x", start=1), DataBlock(data="x", start=1)) + self.assertNotEqual(DataBlock(data="x"), DataBlock(data="y")) diff --git a/tests/test_file_recognizer.py b/tests/test_file_recognizer.py index a73d0fe..3df8475 100644 --- a/tests/test_file_recognizer.py +++ b/tests/test_file_recognizer.py @@ -30,7 +30,6 @@ """ import argparse -from contextlib import contextmanager import errno import gzip import os @@ -39,11 +38,14 @@ import socket import sys from tempfile import TemporaryDirectory -from typing import Callable, cast, Generator, IO +from typing import Callable, cast, IO from unittest import TestCase from grin import FileRecognizer, get_grin_arg_parser, get_grind_arg_parser, GZIP_MAGIC +from grin.options import Options +from grin.recognizer import get_recognizer from grin.types import DirEntry +from tests._helpers import cd, IsolatedVCSEnvMixin # The fixture helpers open files for *writing* binary content, unlike grin's # read-oriented grin.types.Opener. @@ -52,16 +54,6 @@ GZIP_OPEN = cast(WriteOpener, gzip.open) -@contextmanager -def cd(newdir: str) -> Generator[None, None, None]: - prevdir = os.getcwd() - os.chdir(os.path.expanduser(newdir)) - try: - yield - finally: - os.chdir(prevdir) - - class FilesTextCase(TestCase): oldcwd: Path tempdir: "TemporaryDirectory[str]" @@ -591,3 +583,62 @@ def test_vcs_dirs_skipped_without_hidden_skip(self) -> None: for name in self.VCS_DIRS: with self.subTest(dir=name): self.assertEqual(fr.recognize_directory(name, None), "skip") + + +class RecursiveWalkRegressionTestCase(IsolatedVCSEnvMixin): + """Regression test for the walk recursion: descending into a subdirectory + must thread the relative base correctly (previously raised NameError). + + Inherits env isolation because test_walk_with_ignore_recurses walks with + respect_ignore=True, which would otherwise read the developer's global git/hg + excludes.""" + + def test_deeply_nested_files_are_yielded(self) -> None: + with TemporaryDirectory() as root: + os.makedirs(os.path.join(root, "a", "b", "c")) + for rel in ["top.txt", "a/mid.txt", "a/b/deep.txt", "a/b/c/deepest.txt"]: + with open(os.path.join(root, rel), "w", encoding="utf-8") as fp: + fp.write("content\n") + fr = FileRecognizer() + found = sorted(os.path.relpath(path, root) for path, _kind in fr.walk(root)) + self.assertEqual( + found, + [os.path.join(*p.split("/")) for p in ["a/b/c/deepest.txt", "a/b/deep.txt", "a/mid.txt", "top.txt"]], + ) + + def test_walk_with_ignore_recurses(self) -> None: + # respect_ignore=True takes the IgnoreStack code path through recursion. + with TemporaryDirectory() as root: + os.makedirs(os.path.join(root, "pkg", "sub")) + for rel in ["pkg/a.txt", "pkg/sub/b.txt"]: + with open(os.path.join(root, rel), "w", encoding="utf-8") as fp: + fp.write("content\n") + fr = FileRecognizer(respect_ignore=True) + found = sorted(os.path.relpath(path, root) for path, _kind in fr.walk(root)) + self.assertEqual(found, [os.path.join("pkg", "a.txt"), os.path.join("pkg", "sub", "b.txt")]) + + +class GetRecognizerTestCase(TestCase): + """get_recognizer() builds a FileRecognizer from Options.""" + + def test_converts_comma_separated_strings_to_sets(self) -> None: + fr = get_recognizer(Options(skip_dirs="build,dist", skip_exts=".pyc,.so")) + self.assertEqual(fr.skip_dirs, {"build", "dist"}) + self.assertIn(".pyc", fr.skip_exts) + self.assertIn(".so", fr.skip_exts) + + def test_accepts_sets_directly(self) -> None: + fr = get_recognizer(Options(skip_dirs={"build"}, skip_exts={".pyc"})) + self.assertEqual(fr.skip_dirs, {"build"}) + + def test_skip_backup_files_adds_tilde_ext(self) -> None: + fr = get_recognizer(Options(skip_backup_files=True)) + self.assertIn("~", fr.skip_exts) + + def test_no_skip_backup_files_removes_tilde_ext(self) -> None: + fr = get_recognizer(Options(skip_exts={"~", ".pyc"}, skip_backup_files=False)) + self.assertNotIn("~", fr.skip_exts) + + def test_sniff_contents_flag_forwarded(self) -> None: + self.assertFalse(get_recognizer(Options(), sniff_contents=False).sniff_contents) + self.assertTrue(get_recognizer(Options(), sniff_contents=True).sniff_contents) diff --git a/tests/test_grep.py b/tests/test_grep.py index 300ac6b..b8dd61e 100644 --- a/tests/test_grep.py +++ b/tests/test_grep.py @@ -26,13 +26,15 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import gzip -from io import BytesIO +from io import BytesIO, StringIO import os import re from tempfile import TemporaryDirectory -from unittest import TestCase +from unittest import mock, TestCase import grin +import grin.main +from grin.main import BlockContextType def make_regex( @@ -582,3 +584,124 @@ def test_corrupt_gzip_is_skipped_not_fatal(self) -> None: def test_missing_file_is_empty(self) -> None: self.assertEqual(self.gt.grep_path(os.path.join(self.root, "nope.txt")), "") + + +class MultiBlockTestCase(TestCase): + """Force the slow, multi-block read path and confirm it matches a single read.""" + + # A body large enough to span several blocks once READ_BLOCKSIZE is shrunk, + # with matches surrounded by context that must survive block boundaries. + body = (b"alpha\n" * 4 + b"needle one\n" + b"beta\n" * 6 + b"needle two\n" + b"gamma\n" * 4).decode() + + def _grep(self, options: grin.Options) -> BlockContextType: + return grin.GrepText(re.compile("needle"), options).do_grep(BytesIO(self.body.encode())) + + def test_multiblock_equals_singleblock_no_context(self) -> None: + options = grin.Options() + expected = self._grep(options) + with mock.patch("grin.main.READ_BLOCKSIZE", 16): + self.assertEqual(self._grep(options), expected) + + def test_multiblock_equals_singleblock_with_context(self) -> None: + options = grin.Options(before_context=2, after_context=2) + expected = self._grep(options) + with mock.patch("grin.main.READ_BLOCKSIZE", 16): + self.assertEqual(self._grep(options), expected) + + def test_multiblock_finds_both_matches(self) -> None: + with mock.patch("grin.main.READ_BLOCKSIZE", 16): + result = self._grep(grin.Options()) + self.assertEqual([line for _, _, line, _ in result], ["needle one\n", "needle two\n"]) + + +class ReportTestCase(TestCase): + """Exercise the formatting branches of GrepText.report().""" + + match: BlockContextType = [(0, grin.main.MATCH, "foo\n", [(0, 3)])] + + def _report( + self, + options: grin.Options, + context: BlockContextType | None = None, + filename: str = "f.txt", + ) -> str: + context = self.match if context is None else context + return grin.GrepText(re.compile("foo"), options).report(context, filename) + + def test_empty_context_is_empty_string(self) -> None: + self.assertEqual(self._report(grin.Options(), context=[]), "") + + def test_files_with_matches_shows_only_filename(self) -> None: + self.assertEqual(self._report(grin.Options(show_match=False)), "f.txt\n") + + def test_line_numbers_default(self) -> None: + self.assertEqual( + self._report(grin.Options(show_filename=False)), + " 1 : foo\n", + ) + + def test_no_line_numbers(self) -> None: + self.assertEqual( + self._report(grin.Options(show_filename=False, show_line_numbers=False)), + "foo\n", + ) + + def test_emacs_format(self) -> None: + self.assertEqual(self._report(grin.Options(show_emacs=True)), "f.txt:1: foo\n") + + def test_filename_header_when_shown(self) -> None: + out = self._report(grin.Options()) + self.assertTrue(out.startswith("f.txt:\n")) + + def test_color_wraps_match_span(self) -> None: + out = self._report(grin.Options(use_color=True, show_filename=False, show_line_numbers=False)) + self.assertEqual(out, "\x1b[30;43mfoo\x1b[0m\n") + + def test_color_offset_tracking_two_spans(self) -> None: + # Two spans on one line: the second span's offset must account for the + # color codes inserted around the first. + context: BlockContextType = [(0, grin.main.MATCH, "foo bar foo\n", [(0, 3), (8, 11)])] + out = self._report( + grin.Options(use_color=True, show_filename=False, show_line_numbers=False), + context=context, + ) + self.assertEqual(out, "\x1b[30;43mfoo\x1b[0m bar \x1b[30;43mfoo\x1b[0m\n") + + +class UniquifyContextTestCase(TestCase): + def setUp(self) -> None: + self.gt = grin.GrepText(re.compile("x")) + + def test_match_preferred_over_context_on_same_line(self) -> None: + context: BlockContextType = [ + (5, grin.main.POST, "a\n", None), + (5, grin.main.MATCH, "a\n", [(0, 1)]), + (3, grin.main.PRE, "b\n", None), + ] + self.assertEqual( + self.gt.uniquify_context(context), + [(3, grin.main.PRE, "b\n", None), (5, grin.main.MATCH, "a\n", [(0, 1)])], + ) + + def test_post_preferred_when_no_match(self) -> None: + context: BlockContextType = [ + (2, grin.main.PRE, "p\n", None), + (2, grin.main.POST, "p\n", None), + ] + self.assertEqual(self.gt.uniquify_context(context), [(2, grin.main.POST, "p\n", None)]) + + +class GrepAFileTestCase(TestCase): + """grep_a_file() uses a caller-supplied opener and special-cases stdin.""" + + def test_stdin_dash(self) -> None: + gt = grin.GrepText(re.compile("needle")) + with mock.patch("sys.stdin", StringIO("foo\nthe needle here\nbar\n")): + out = gt.grep_a_file("-") + self.assertIn("needle here", out) + self.assertIn("", out) + + def test_missing_file_raises(self) -> None: + gt = grin.GrepText(re.compile("needle")) + with self.assertRaises(OSError): + gt.grep_a_file("/no/such/file/here.txt") diff --git a/tests/test_grind.py b/tests/test_grind.py new file mode 100644 index 0000000..32eb0ed --- /dev/null +++ b/tests/test_grind.py @@ -0,0 +1,141 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Tests for the grind (find-like) command: argument parsing, configuration, +and end-to-end directory listing. + +Note: grind's positional ``glob`` must come *before* ``--dirs`` on the command +line, because ``--dirs`` uses ``nargs="+"`` and would otherwise swallow it. +""" + +import argparse +import contextlib +import io +import os +from tempfile import TemporaryDirectory +from unittest import mock, TestCase + +import grin.grind +from grin.grind import get_grind_arg_parser, get_grind_config, print_null +from tests._helpers import build_tree, IsolatedVCSEnvMixin, run_cli + + +def run_grind(*argv: str) -> str: + return run_cli(grin.grind.main, "grind", *argv) + + +def basenames(output: str, sep: str = "\n") -> list[str]: + return sorted(os.path.basename(line) for line in output.split(sep) if line) + + +class GrindArgParserTestCase(TestCase): + def parse(self, argv: list[str]) -> argparse.Namespace: + return get_grind_arg_parser().parse_args(argv) + + def test_defaults(self) -> None: + args = self.parse([]) + self.assertEqual(args.glob, "*") + self.assertEqual(args.dirs, ["."]) + self.assertFalse(args.null_separated) + self.assertTrue(args.respect_ignore_files) + self.assertFalse(args.follow_symlinks) + + def test_glob_positional(self) -> None: + self.assertEqual(self.parse(["*.py"]).glob, "*.py") + + def test_null_separated_flag(self) -> None: + self.assertTrue(self.parse(["-0"]).null_separated) + self.assertTrue(self.parse(["--null-separated"]).null_separated) + + def test_follow_flag(self) -> None: + self.assertTrue(self.parse(["--follow"]).follow_symlinks) + + def test_no_skip_dirs_and_exts(self) -> None: + self.assertEqual(self.parse(["-D"]).skip_dirs, "") + self.assertEqual(self.parse(["-E"]).skip_exts, "") + + def test_no_ignore_flag(self) -> None: + self.assertFalse(self.parse(["--no-ignore"]).respect_ignore_files) + + def test_dirs_accepts_multiple(self) -> None: + self.assertEqual(self.parse(["--dirs", "a", "b"]).dirs, ["a", "b"]) + + +class GrindConfigTestCase(TestCase): + def test_config_maps_args(self) -> None: + options = get_grind_config(["grind", "*.py", "--dirs", "x", "-0", "--no-ignore"]) + self.assertEqual(options.glob, "*.py") + self.assertEqual(options.dirs, ["x"]) + self.assertTrue(options.null_separated) + self.assertFalse(options.respect_ignore_files) + + def test_grind_args_env_is_prepended(self) -> None: + # GRIND_ARGS is consumed only when argv is None (read from sys.argv). + with mock.patch("sys.argv", ["grind", "*.py"]), mock.patch.dict(os.environ, {"GRIND_ARGS": "-0"}): + options = get_grind_config(None) + self.assertTrue(options.null_separated) + self.assertEqual(options.glob, "*.py") + + +class PrintNullTestCase(TestCase): + def test_writes_trailing_nul(self) -> None: + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + print_null("some/file.txt") + self.assertEqual(buf.getvalue(), "some/file.txt\0") + + +class GrindMainTestCase(IsolatedVCSEnvMixin): + def setUp(self) -> None: + super().setUp() + self._tmp = TemporaryDirectory() + self.root = self._tmp.name + build_tree(self.root, ["a.py", "b.txt", "sub/c.py", "sub/d.txt"]) + self.addCleanup(self._tmp.cleanup) + + def test_lists_all_files_recursively(self) -> None: + self.assertEqual(basenames(run_grind("*", "--dirs", self.root)), ["a.py", "b.txt", "c.py", "d.txt"]) + + def test_glob_filters_by_basename(self) -> None: + self.assertEqual(basenames(run_grind("*.py", "--dirs", self.root)), ["a.py", "c.py"]) + + def test_null_separated_output(self) -> None: + output = run_grind("-0", "*.py", "--dirs", self.root) + self.assertTrue(output.endswith("\0")) + self.assertNotIn("\n", output) + self.assertEqual(basenames(output, sep="\0"), ["a.py", "c.py"]) + + def test_respects_gitignore_by_default(self) -> None: + with open(os.path.join(self.root, ".gitignore"), "w", encoding="utf-8") as fp: + fp.write("b.txt\n") + self.assertNotIn("b.txt", basenames(run_grind("*", "--dirs", self.root))) + + def test_no_ignore_disables_gitignore(self) -> None: + with open(os.path.join(self.root, ".gitignore"), "w", encoding="utf-8") as fp: + fp.write("b.txt\n") + self.assertIn("b.txt", basenames(run_grind("--no-ignore", "*", "--dirs", self.root))) diff --git a/tests/test_ignore.py b/tests/test_ignore.py index f868e43..1d51208 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -31,41 +31,10 @@ import os from tempfile import TemporaryDirectory -from unittest import mock, TestCase +from unittest import mock from grin import FileRecognizer - - -def build_tree(root: str, files: dict[str, str]) -> None: - for rel, content in files.items(): - path = os.path.join(root, rel) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w", encoding="utf-8") as fp: - fp.write(content) - - -class _IsolatedEnvTestCase(TestCase): - """Point the global git/hg config env at an empty home so the developer's - real ~/.gitignore / global excludes never leak into the ignore tests.""" - - def setUp(self) -> None: - super().setUp() - tmp = TemporaryDirectory() - self.addCleanup(tmp.cleanup) - home = tmp.name - nowhere = os.path.join(home, "nonexistent") - patcher = mock.patch.dict( - os.environ, - { - "HOME": home, - "XDG_CONFIG_HOME": os.path.join(home, "config"), - "GIT_CONFIG_GLOBAL": nowhere, - "HGRCPATH": nowhere, - }, - ) - patcher.start() - self.addCleanup(patcher.stop) - +from tests._helpers import build_tree, IsolatedVCSEnvMixin # (name, files-to-create, paths-that-must-be-ignored, paths-that-must-be-kept) CASES: list[tuple[str, dict[str, str], set[str], set[str]]] = [ @@ -139,7 +108,7 @@ def setUp(self) -> None: ] -class IgnoreWalkTestCase(_IsolatedEnvTestCase): +class IgnoreWalkTestCase(IsolatedVCSEnvMixin): def _walk(self, files: dict[str, str], respect_ignore: bool) -> set[str]: with TemporaryDirectory() as root: build_tree(root, files) @@ -160,7 +129,7 @@ def test_nothing_ignored_when_disabled(self) -> None: self.assertEqual((ignored | kept) - walked, set()) -class RepoAndGlobalExcludesTestCase(_IsolatedEnvTestCase): +class RepoAndGlobalExcludesTestCase(IsolatedVCSEnvMixin): """Per-repo (.git/info/exclude, .hg/hgrc) and user-global git/hg excludes.""" def _walk(self, files: dict[str, str], env: dict[str, str] | None = None) -> set[str]: diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 1e16c51..eeb688c 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -34,7 +34,7 @@ import os import re from tempfile import TemporaryDirectory -from unittest import TestCase +from unittest import mock, TestCase import grin import grin.grin @@ -81,6 +81,15 @@ def test_resolve_jobs(self) -> None: self.assertEqual(resolve_jobs(1), 1) self.assertGreaterEqual(resolve_jobs(0), 1) + def test_resolve_jobs_auto_uses_cpu_count(self) -> None: + with mock.patch("grin.parallel.os.cpu_count", return_value=7): + self.assertEqual(resolve_jobs(0), 7) + + def test_resolve_jobs_auto_falls_back_to_one(self) -> None: + # os.cpu_count() can return None; resolve_jobs must not return None. + with mock.patch("grin.parallel.os.cpu_count", return_value=None): + self.assertEqual(resolve_jobs(0), 1) + class GrinParallelOutputTestCase(TestCase): def _run(self, target: str, jobs: int) -> str: diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..e2a3452 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,137 @@ +# Copyright (C) 2017-2021, Raffaele Salmaso +# Copyright (C) 2007, Enthought, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Enthought, Inc. nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Unit tests for the small helpers in grin.utils. +""" + +import argparse +import re +from unittest import mock, TestCase + +from grin.datablock import DataBlock +from grin.options import Options +from grin.utils import deprecate_option, get_line_offsets, get_regex, is_binary_string, to_str + + +class ToStrTestCase(TestCase): + def test_str_passthrough(self) -> None: + self.assertEqual(to_str("already a str"), "already a str") + + def test_decode_utf8(self) -> None: + self.assertEqual(to_str("Rémy".encode("utf8")), "Rémy") + + def test_latin1_fallback_on_invalid_utf8(self) -> None: + # 0xe9 is a valid latin1 'é' but not valid standalone UTF-8. + self.assertEqual(to_str(b"R\xe9my"), "R\xe9my") + + def test_explicit_encoding(self) -> None: + self.assertEqual(to_str("Rémy".encode("latin1"), encoding="latin1"), "Rémy") + + +class IsBinaryStringTestCase(TestCase): + def test_empty_is_text(self) -> None: + self.assertFalse(is_binary_string(b"")) + + def test_plain_ascii_is_text(self) -> None: + self.assertFalse(is_binary_string(b"hello world\n")) + + def test_allowed_control_chars_are_text(self) -> None: + # bell, backspace, tab, lf, ff, cr, esc are all considered text. + self.assertFalse(is_binary_string(bytes([7, 8, 9, 10, 12, 13, 27]))) + + def test_utf8_high_bytes_are_text(self) -> None: + self.assertFalse(is_binary_string("Rémy\n".encode("utf8"))) + + def test_nul_byte_is_binary(self) -> None: + self.assertTrue(is_binary_string(b"abc\x00def")) + + def test_other_control_byte_is_binary(self) -> None: + # 0x00..0x06 (except the allowed set) classify as binary. + self.assertTrue(is_binary_string(b"abc\x01def")) + + +class GetLineOffsetsTestCase(TestCase): + def test_single_line_no_newline(self) -> None: + block = DataBlock(data="abc", start=0, end=3) + self.assertEqual(get_line_offsets(block), ([0, 3], 0)) + + def test_multiple_lines_with_trailing_newline(self) -> None: + block = DataBlock(data="abc\ndef\nghi\n", start=0, end=12) + self.assertEqual(get_line_offsets(block), ([0, 4, 8, 12], 3)) + + def test_multiple_lines_without_trailing_newline(self) -> None: + # A trailing EOF "line start" is appended so successive offsets bound + # each line, but it does not count as a newline-terminated line. + block = DataBlock(data="abc\ndef\nghi", start=0, end=11) + self.assertEqual(get_line_offsets(block), ([0, 4, 8, 11], 2)) + + def test_empty_block(self) -> None: + block = DataBlock(data="", start=0, end=0) + self.assertEqual(get_line_offsets(block), ([0], 0)) + + def test_only_lines_within_window_are_counted(self) -> None: + # Newlines sit at offsets 2, 5, 8, 11. Only those in [start, end) -> 5, 8. + block = DataBlock(data="aa\nbb\ncc\ndd\n", start=3, end=9) + self.assertEqual(get_line_offsets(block), ([0, 3, 6, 9, 12], 2)) + + +class GetRegexTestCase(TestCase): + def test_plain_compile(self) -> None: + regex = get_regex(Options(regex="foo")) + self.assertTrue(regex.search("a foo b")) + self.assertEqual(regex.pattern, "foo") + + def test_fixed_string_escapes_metachars(self) -> None: + regex = get_regex(Options(regex="foo(", fixed_string=True)) + self.assertTrue(regex.search("def foo(")) + self.assertFalse(regex.search("fooX")) + + def test_word_regexp_wraps_boundaries(self) -> None: + regex = get_regex(Options(regex="test", word_regexp=True)) + self.assertTrue(regex.search("a test.")) + self.assertFalse(regex.search("testing")) + + def test_re_flags_are_combined(self) -> None: + regex = get_regex(Options(regex="foo", re_flags=[re.I, re.M])) + self.assertTrue(regex.flags & re.I) + self.assertTrue(regex.flags & re.M) + self.assertTrue(regex.search("FOO")) + + def test_invalid_pattern_raises(self) -> None: + with self.assertRaises(re.error): + get_regex(Options(regex="(unclosed")) + + +class DeprecateOptionTestCase(TestCase): + def test_suppressed_without_help_verbose(self) -> None: + with mock.patch("sys.argv", ["grin", "foo"]): + self.assertEqual(deprecate_option("some help"), argparse.SUPPRESS) + + def test_shown_with_help_verbose(self) -> None: + with mock.patch("sys.argv", ["grin", "--help-verbose"]): + self.assertEqual(deprecate_option("some help"), "some help") diff --git a/tox.ini b/tox.ini index f7b2889..84bdc2c 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,15 @@ commands = python -m unittest discover tests setenv = PYTHONDONTWRITEBYTECODE = 1 +[testenv:coverage] +# editable install so the project's runtime deps (pathspec) are available +package = editable +deps = + coverage +commands = + coverage run -m unittest discover tests + coverage report -m + [testenv:ruff] changedir = {toxinidir} skip_install = True From 8db98d32d40c299f9503df5bead93d52660f4100 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Fri, 26 Jun 2026 11:02:37 +0200 Subject: [PATCH 25/37] Fix force_color option management --- grin/grin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/grin/grin.py b/grin/grin.py index c37689e..99315be 100644 --- a/grin/grin.py +++ b/grin/grin.py @@ -418,7 +418,6 @@ def get_grin_config(argv: list[str] | None = None) -> GrinOptions: argv = [sys.argv[0]] + env_args + sys.argv[1:] parser = get_grin_arg_parser() args = parser.parse_args(argv[1:]) - use_term_color = not args.no_color and sys.stdout.isatty() and (os.environ.get("TERM") != "dumb") # handle deprecated --{no,use,force}-color options isatty = sys.stdout.isatty() and (os.environ.get("TERM") != "dumb") @@ -430,7 +429,7 @@ def get_grin_config(argv: list[str] | None = None) -> GrinOptions: return GrinOptions( before_context=args.context if args.context is not None else args.before_context, after_context=args.context if args.context is not None else args.after_context, - use_color=args.force_color or use_term_color, + use_color=args.use_color, show_line_numbers=args.show_line_numbers, show_match=args.show_match, show_filename=args.show_filename, From 83be7637028890e8f5aadd293a549f0f00d07183 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Fri, 26 Jun 2026 11:05:55 +0200 Subject: [PATCH 26/37] Fix grind --sys-path: search the directories on sys.path again (the expansion had been dropped) --- grin/grind.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/grin/grind.py b/grin/grind.py index ca7a346..4ecb0de 100644 --- a/grin/grind.py +++ b/grin/grind.py @@ -188,8 +188,11 @@ def get_grind_config(argv: list[str] | None = None) -> GrindOptions: argv = [sys.argv[0]] + env_args + sys.argv[1:] parser = get_grind_arg_parser() args = parser.parse_args(argv[1:]) + dirs = list(args.dirs) + if args.sys_path: + dirs.extend(sys.path) options = GrindOptions( - dirs=args.dirs, + dirs=dirs, glob=args.glob, null_separated=args.null_separated, sys_path=args.sys_path, From 7af9591d28a62fd244aa241247414acda7e61993 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Fri, 26 Jun 2026 11:30:49 +0200 Subject: [PATCH 27/37] Fix grin -x/--encoding: thread the requested encoding through to the grep workers --- grin/grin.py | 3 ++- grin/options.py | 1 + grin/parallel.py | 10 +++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/grin/grin.py b/grin/grin.py index 99315be..8127eb4 100644 --- a/grin/grin.py +++ b/grin/grin.py @@ -401,7 +401,7 @@ def get_grin_arg_parser(parser: argparse.ArgumentParser | None = None) -> argpar parser.add_argument( "-x", "--encoding", - default=sys.stdout.encoding, + default=sys.stdout.encoding or "utf8", help="Encoding from which to open the included files from in order for the regex" " and the stdout output to work properly. Default to your terminal output encoding.", ) @@ -430,6 +430,7 @@ def get_grin_config(argv: list[str] | None = None) -> GrinOptions: before_context=args.context if args.context is not None else args.before_context, after_context=args.context if args.context is not None else args.after_context, use_color=args.use_color, + encoding=args.encoding, show_line_numbers=args.show_line_numbers, show_match=args.show_match, show_filename=args.show_filename, diff --git a/grin/options.py b/grin/options.py index 5b9e6f0..ae2f6b2 100644 --- a/grin/options.py +++ b/grin/options.py @@ -51,6 +51,7 @@ class Options: fixed_string: bool = False word_regexp: bool = False use_color: bool = False + encoding: str = "utf8" regex: str = "" files: list[str] = field(default_factory=list) sys_path: bool = False diff --git a/grin/parallel.py b/grin/parallel.py index 5f0388f..a206ae9 100644 --- a/grin/parallel.py +++ b/grin/parallel.py @@ -35,6 +35,7 @@ from collections.abc import Iterable, Iterator from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +import functools import os import sysconfig @@ -52,11 +53,13 @@ # Per-process GrepText, built once by the pool initializer (process backend). _worker_grep: GrepText | None = None +_worker_encoding: str = "utf8" def _init_worker(options: Options) -> None: - global _worker_grep + global _worker_grep, _worker_encoding _worker_grep = GrepText(get_regex(options), options) + _worker_encoding = options.encoding def _grep_in_worker(filename: str) -> str: @@ -84,14 +87,15 @@ def parallel_grep(grep_text: GrepText, options: Options, filenames: Iterable[str files = list(filenames) if workers <= 1 or len(files) < MIN_PARALLEL_FILES: for filename in files: - yield grep_text.grep_path(filename) + yield grep_text.grep_path(filename, options.encoding) return # ~16 chunks per worker balances load while amortizing dispatch overhead. chunksize = max(1, len(files) // (workers * 16)) if GIL_DISABLED: + grep_one = functools.partial(grep_text.grep_path, encoding=options.encoding) with ThreadPoolExecutor(max_workers=workers) as thread_pool: - yield from thread_pool.map(grep_text.grep_path, files, chunksize=chunksize) + yield from thread_pool.map(grep_one, files, chunksize=chunksize) else: with ProcessPoolExecutor(max_workers=workers, initializer=_init_worker, initargs=(options,)) as process_pool: yield from process_pool.map(_grep_in_worker, files, chunksize=chunksize) From 57368d1398310feb6a7fec57fab08dc79f354a48 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Fri, 26 Jun 2026 11:35:18 +0200 Subject: [PATCH 28/37] Fix grinpython example: forward the regex, files, and file-discovery options --- examples/grinpython.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/examples/grinpython.py b/examples/grinpython.py index f423097..22a91d1 100755 --- a/examples/grinpython.py +++ b/examples/grinpython.py @@ -54,13 +54,19 @@ def replace_with_spaces(self, s: str) -> str: """ return s.translate(self.space_table) - def __call__(self, filename: str, mode: str = "rt") -> IO[str]: + def __call__(self, filename: str, mode: str = "rb") -> IO[str]: """ Open a file and convert it to a filelike object with transformed contents. + + ``mode`` is ignored: GrepText opens files in binary ("rb"), but + tokenize.generate_tokens() needs text, so we always open with + tokenize.open() (which honors the source's PEP 263 encoding cookie). + The resulting StringIO of text flows through GrepText unchanged + (utils.to_str passes str through). """ g: StringIO = StringIO() - f: IO[str] = open(filename, mode) + f: IO[str] = tokenize.open(filename) try: gen = tokenize.generate_tokens(f.readline) old_end = (1, 0) @@ -127,6 +133,27 @@ def get_grinpython_config(argv: list[str] | None = None) -> GrinpythonOptions: before_context=args.context if args.context is not None else 0, after_context=args.context if args.context is not None else 0, use_color=args.force_color or _isatty, + encoding=args.encoding, + show_line_numbers=args.show_line_numbers, + show_match=args.show_match, + show_filename=args.show_filename, + show_emacs=args.show_emacs, + skip_hidden_dirs=args.skip_hidden_dirs, + skip_hidden_files=args.skip_hidden_files, + skip_backup_files=args.skip_backup_files, + skip_dirs=set([x for x in args.skip_dirs.split(",") if x]), + skip_exts=set([x for x in args.skip_exts.split(",") if x]), + skip_symlink_files=not args.follow_symlinks, + skip_symlink_dirs=not args.follow_symlinks, + re_flags=args.re_flags, + regex=args.regex, + files=args.files, + files_from_file=args.files_from_file, + null_separated=args.null_separated, + include=args.include, + sys_path=args.sys_path, + respect_ignore_files=args.respect_ignore_files, + jobs=args.jobs, python_code=args.python_code, comments=args.comments, strings=args.strings, From 7e5022ca75fca5dae2cdc4797ad842fffc730672 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Fri, 26 Jun 2026 13:16:04 +0200 Subject: [PATCH 29/37] Fix GrepText.grep_a_file: stop closing sys.stdin when grepping "-" The guard checked the reassigned filename and never matched. --- grin/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grin/main.py b/grin/main.py index eebfd85..846ed34 100644 --- a/grin/main.py +++ b/grin/main.py @@ -415,7 +415,7 @@ def grep_a_file(self, filename: str, opener: types.Opener = open, encoding: str try: unique_context = self.do_grep(f, encoding) finally: - if filename != "-": + if f is not sys.stdin: f.close() return self.report(unique_context, filename) From 4fc39e73ab85f065db6bc1bff02faecd79ba931c Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 09:11:51 +0200 Subject: [PATCH 30/37] Speed up matching Derive match spans from the match object instead of re-scanning each matched line with the regex, tighten the get_line_offsets newline scan, and render colorized lines in a single pass --- grin/main.py | 73 +++++++++++++++++++++++++++++++++------------------ grin/utils.py | 27 +++++++++++-------- 2 files changed, 64 insertions(+), 36 deletions(-) diff --git a/grin/main.py b/grin/main.py index 846ed34..4c01f35 100644 --- a/grin/main.py +++ b/grin/main.py @@ -75,19 +75,24 @@ class BuildMatchContext: line_count: LineCountType = None def _call(self, match: re.Match[str]) -> BlockContextType: - match_line_num: int = bisect.bisect(cast(list[int], self.line_offsets), match.start() + self.block.start) - 1 + offsets = cast(list[int], self.line_offsets) + abs_start = match.start() + self.block.start + match_line_num: int = bisect.bisect(offsets, abs_start) - 1 before_count: int = min(self.before, match_line_num) - after_count: int = min(self.after, (len(cast(list[int], self.line_offsets)) - 1) - match_line_num - 1) - match_line: str = self.block.data[ - cast(list[int], self.line_offsets)[match_line_num] : cast(list[int], self.line_offsets)[match_line_num + 1] - ] - spans = [m.span() for m in self.regex.finditer(match_line)] + after_count: int = min(self.after, (len(offsets) - 1) - match_line_num - 1) + line_start = offsets[match_line_num] + match_line: str = self.block.data[line_start : offsets[match_line_num + 1]] + # The span of this match within its line. We already have the match + # object, so derive the span directly instead of re-scanning the line + # with the regex. A line may contain several matches; each produces its + # own MATCH entry here and uniquify_context() merges their spans. + spans: list[tuple[int, int]] | None = [(abs_start - line_start, match.end() + self.block.start - line_start)] before_ctx: BlockContextType = [ ( i + self.line_num_offset, PRE, - self.block.data[cast(list[int], self.line_offsets)[i] : cast(list[int], self.line_offsets)[i + 1]], + self.block.data[offsets[i] : offsets[i + 1]], None, ) for i in range(match_line_num - before_count, match_line_num) @@ -96,7 +101,7 @@ def _call(self, match: re.Match[str]) -> BlockContextType: ( i + self.line_num_offset, POST, - self.block.data[cast(list[int], self.line_offsets)[i] : cast(list[int], self.line_offsets)[i + 1]], + self.block.data[offsets[i] : offsets[i + 1]], None, ) for i in range(match_line_num + 1, match_line_num + after_count + 1) @@ -314,19 +319,35 @@ def do_grep_block(self, block: DataBlock, line_num_offset: int) -> tuple[int | N return (build_match_context.line_count, build_match_context.block_context) def uniquify_context(self, context: BlockContextType) -> BlockContextType: - """Remove duplicate lines from the list of context lines.""" + """Remove duplicate lines from the list of context lines. + + A line may produce several MATCH entries (one per match on it); their + spans are merged into a single MATCH entry so every occurrence can be + colored. + """ context.sort() unique_context = [] - for group in itertools.groupby(context, lambda ikl: ikl[0]): - for i, kind, line, matches in group[1]: - if kind == MATCH: - # Always use a match. - unique_context.append((i, kind, line, matches)) - break + for _lineno, group in itertools.groupby(context, lambda ikl: ikl[0]): + match_entry = None + merged_spans: list[tuple[int, int]] = [] + last_entry = None + for entry in group: + last_entry = entry + if entry[1] == MATCH: + # Always prefer a match over PRE/POST context on the same line. + if match_entry is None: + match_entry = entry + if entry[3]: + merged_spans.extend(entry[3]) + if match_entry is not None: + i, kind, line, _spans = match_entry + unique_context.append((i, kind, line, merged_spans if merged_spans else None)) else: # No match, only PRE and/or POST lines. Use the last one, which - # should be a POST since we've sorted it that way. - unique_context.append((i, kind, line, matches)) + # should be a POST since we've sorted it that way. (groupby never + # yields an empty group, so last_entry is always set here.) + assert last_entry is not None + unique_context.append(last_entry) return unique_context @@ -369,15 +390,17 @@ def report(self, context_lines: BlockContextType, filename: str | None = None) - for i, kind, line, spans in context_lines: if self.options.use_color and kind == MATCH and spans is not None and "searchterm" in STYLES: style = STYLES["searchterm"] - orig_line = line[:] - total_offset = 0 + # Build the colorized line in a single left-to-right pass: + # interleave the unmatched chunks with the colored matches and + # join once, instead of rebuilding the whole line per span. + parts = [] + prev = 0 for start, end in spans: - old_substring = orig_line[start:end] - start += total_offset - end += total_offset - color_substring = style.apply(old_substring) - line = line[:start] + color_substring + line[end:] - total_offset += len(color_substring) - len(old_substring) + parts.append(line[prev:start]) + parts.append(style.apply(line[start:end])) + prev = end + parts.append(line[prev:]) + line = "".join(parts) ns = dict( lineno=i + 1, diff --git a/grin/utils.py b/grin/utils.py index 970eca4..9bbcfa7 100644 --- a/grin/utils.py +++ b/grin/utils.py @@ -76,23 +76,28 @@ def get_line_offsets(block: DataBlock) -> tuple[list[int], int]: """ # Note: this implementation based on string.find() benchmarks about twice as # fast as a list comprehension using re.finditer(). - line_offsets = [0] - line_count = 0 # Count of lines inside range [block.start, block.end) *only* s = block.data + line_offsets = [0] + # Bind the hot methods to locals and track the scan position directly instead + # of re-reading line_offsets[-1] on every iteration. + find = s.find + append = line_offsets.append + pos = 0 while True: - next_newline = s.find("\n", line_offsets[-1]) + next_newline = find("\n", pos) if next_newline < 0: # Tack on a final "line start" corresponding to EOF, if not done already. # This makes it possible to determine the length of each line by computing # a difference between successive elements. - if line_offsets[-1] < len(s): - line_offsets.append(len(s)) - return (line_offsets, line_count) - else: - line_offsets.append(next_newline + 1) - # Keep track of the count of lines within the "current block" - if next_newline >= block.start and next_newline < block.end: - line_count += 1 + if pos < len(s): + append(len(s)) + break + pos = next_newline + 1 + append(pos) + # Count of newlines inside [block.start, block.end) -- one C-level pass instead + # of a per-iteration range check in the loop above. + line_count = s.count("\n", block.start, block.end) + return (line_offsets, line_count) def get_regex(options: Options) -> re.Pattern[str]: From 2daac63644faf591c44430405c3645164334a562 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 09:25:46 +0200 Subject: [PATCH 31/37] Speed up large (multi-block, >16 MB) files Search each block with finditer pos/endpos instead of copying the block slice (~9% faster on such files; no change for files that fit in a single block). As a side effect, lookbehind/\A-anchored patterns now also match correctly across block boundaries. --- grin/main.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/grin/main.py b/grin/main.py index 4c01f35..54d41a6 100644 --- a/grin/main.py +++ b/grin/main.py @@ -76,7 +76,9 @@ class BuildMatchContext: def _call(self, match: re.Match[str]) -> BlockContextType: offsets = cast(list[int], self.line_offsets) - abs_start = match.start() + self.block.start + # match offsets are absolute into block.data (finditer is called with + # pos/endpos, not on a slice). + abs_start = match.start() match_line_num: int = bisect.bisect(offsets, abs_start) - 1 before_count: int = min(self.before, match_line_num) after_count: int = min(self.after, (len(offsets) - 1) - match_line_num - 1) @@ -86,7 +88,7 @@ def _call(self, match: re.Match[str]) -> BlockContextType: # object, so derive the span directly instead of re-scanning the line # with the regex. A line may contain several matches; each produces its # own MATCH entry here and uniquify_context() merges their spans. - spans: list[tuple[int, int]] | None = [(abs_start - line_start, match.end() + self.block.start - line_start)] + spans: list[tuple[int, int]] | None = [(abs_start - line_start, match.end() - line_start)] before_ctx: BlockContextType = [ ( @@ -312,8 +314,10 @@ def do_grep_block(self, block: DataBlock, line_num_offset: int) -> tuple[int | N regex=self.regex, ) - # Using re.MULTILINE here, so ^ and $ will work as expected. - for match in self.regex_m.finditer(block.data[block.start : block.end]): + # Using re.MULTILINE here, so ^ and $ will work as expected. Pass + # pos/endpos instead of slicing block.data to avoid copying the (up to + # 16 MB) current region; match offsets are then absolute into block.data. + for match in self.regex_m.finditer(block.data, block.start, block.end): build_match_context(match) return (build_match_context.line_count, build_match_context.block_context) From 5bbdbebfbd3522979022a489a3bee3c9a3fbcaa1 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 09:51:59 +0200 Subject: [PATCH 32/37] Lazily import where possible for faster startup --- grin/ignore.py | 9 +++++++-- grin/parallel.py | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/grin/ignore.py b/grin/ignore.py index 71b84ec..c50e500 100644 --- a/grin/ignore.py +++ b/grin/ignore.py @@ -51,13 +51,16 @@ apply only when its root is at or below where the search starts. """ +from __future__ import annotations + from collections.abc import Iterator from dataclasses import dataclass import os import re +from typing import TYPE_CHECKING -from pathspec import GitIgnoreSpec -from pathspec.pattern import Pattern +if TYPE_CHECKING: + from pathspec.pattern import Pattern __all__ = ["IgnoreStack"] @@ -101,6 +104,8 @@ def matches(self, rel: str, is_dir: bool) -> bool: def _gitignore_rules(lines: list[str], base: str) -> list[_Rule]: + from pathspec import GitIgnoreSpec + rules: list[_Rule] = [] try: patterns = GitIgnoreSpec.from_lines(lines).patterns diff --git a/grin/parallel.py b/grin/parallel.py index a206ae9..875f593 100644 --- a/grin/parallel.py +++ b/grin/parallel.py @@ -34,7 +34,6 @@ """ from collections.abc import Iterable, Iterator -from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor import functools import os import sysconfig @@ -93,9 +92,13 @@ def parallel_grep(grep_text: GrepText, options: Options, filenames: Iterable[str # ~16 chunks per worker balances load while amortizing dispatch overhead. chunksize = max(1, len(files) // (workers * 16)) if GIL_DISABLED: + from concurrent.futures import ThreadPoolExecutor + grep_one = functools.partial(grep_text.grep_path, encoding=options.encoding) with ThreadPoolExecutor(max_workers=workers) as thread_pool: yield from thread_pool.map(grep_one, files, chunksize=chunksize) else: + from concurrent.futures import ProcessPoolExecutor + with ProcessPoolExecutor(max_workers=workers, initializer=_init_worker, initargs=(options,)) as process_pool: yield from process_pool.map(_grep_in_worker, files, chunksize=chunksize) From 12a073c4fc61a367aa6c957a2c0effc07343a6b7 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 10:48:23 +0200 Subject: [PATCH 33/37] Much faster -l/--files-with-matches Stop at the first match per file and skip line-offset and context computation entirely (only the presence of a match matters), which is dramatically faster on match-dense files --- grin/main.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/grin/main.py b/grin/main.py index 54d41a6..7a283bc 100644 --- a/grin/main.py +++ b/grin/main.py @@ -264,10 +264,19 @@ def do_grep(self, fp: types.ReadableFile, encoding: str = "utf8") -> BlockContex else: fp_size = None + show_match = self.options.show_match block = self.read_block_with_context(None, fp, fp_size, encoding) while block.end > block.start: (block_line_count, block_context) = self.do_grep_block(block, line_count - block.before_count) context += block_context + if not show_match: + # Files-with-matches mode: a single match anywhere is enough, and + # line numbering is irrelevant, so stop on the first hit and skip + # the per-block line-count bookkeeping below. + if context or block.is_last: + break + block = self.read_block_with_context(block, fp, fp_size, encoding) + continue if block.is_last: break @@ -283,8 +292,9 @@ def do_grep(self, fp: types.ReadableFile, encoding: str = "utf8") -> BlockContex line_count += block_line_count block = next_block - unique_context = self.uniquify_context(context) - return unique_context + if not show_match: + return context + return self.uniquify_context(context) def do_grep_block(self, block: DataBlock, line_num_offset: int) -> tuple[int | None, BlockContextType]: """Grep a single block of file content. @@ -306,6 +316,15 @@ def do_grep_block(self, block: DataBlock, line_num_offset: int) -> tuple[int | N **spans** is a list of (start,end) positions of substrings that matched the pattern. """ + # Files-with-matches mode (-l) only needs to know whether the block + # contains a match, not where. Stop at the first one and skip the line + # offsets and per-match context entirely; the returned marker just tells + # report() there is something to report. + if not self.options.show_match: + if self.regex_m.search(block.data, block.start, block.end) is not None: + return (None, [(0, MATCH, "", None)]) + return (None, []) + build_match_context = BuildMatchContext( block=block, before=self.options.before_context, From 1807128f6d8ae173615e61805de36fb9c00ec366 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 10:57:44 +0200 Subject: [PATCH 34/37] Faster --no-line-number searches (no context, no --emacs) Extract each matched line directly around the match instead of building the whole block's line-offset table, ~3-6x faster for that output mode --- grin/main.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/grin/main.py b/grin/main.py index 7a283bc..1294e43 100644 --- a/grin/main.py +++ b/grin/main.py @@ -264,7 +264,9 @@ def do_grep(self, fp: types.ReadableFile, encoding: str = "utf8") -> BlockContex else: fp_size = None - show_match = self.options.show_match + opts = self.options + show_match = opts.show_match + need_line_info = bool(opts.show_line_numbers or opts.show_emacs or opts.before_context or opts.after_context) block = self.read_block_with_context(None, fp, fp_size, encoding) while block.end > block.start: (block_line_count, block_context) = self.do_grep_block(block, line_count - block.before_count) @@ -281,7 +283,7 @@ def do_grep(self, fp: types.ReadableFile, encoding: str = "utf8") -> BlockContex break next_block = self.read_block_with_context(block, fp, fp_size, encoding) - if next_block.end > next_block.start: + if need_line_info and next_block.end > next_block.start: if block_line_count is None: # If the file contains N blocks, then in the best case we # will need to compute line offsets for the first N-1 blocks. @@ -292,7 +294,7 @@ def do_grep(self, fp: types.ReadableFile, encoding: str = "utf8") -> BlockContex line_count += block_line_count block = next_block - if not show_match: + if not show_match or not need_line_info: return context return self.uniquify_context(context) @@ -325,6 +327,37 @@ def do_grep_block(self, block: DataBlock, line_num_offset: int) -> tuple[int | N return (None, [(0, MATCH, "", None)]) return (None, []) + opts = self.options + if not (opts.show_line_numbers or opts.show_emacs or opts.before_context or opts.after_context): + # Line-text-only mode: no line numbers, no context. Extract each + # matched line locally by finding the newlines around the match + # instead of building the whole block's line-offset table. Several + # matches on one line collapse to a single entry (merging spans); + # lines never span blocks, so deduping within the block suffices. + # The line number is irrelevant here, so a placeholder 0 is used. + data = block.data + compute_spans = opts.use_color + results: BlockContextType = [] + seen: dict[int, int] = {} + for match in self.regex_m.finditer(data, block.start, block.end): + ms = match.start() + line_start = data.rfind("\n", 0, ms) + 1 + idx = seen.get(line_start) + if idx is not None: + if compute_spans: + old = results[idx] + old_spans = old[3] + assert old_spans is not None + new_span = (ms - line_start, match.end() - line_start) + results[idx] = (old[0], old[1], old[2], [*old_spans, new_span]) + continue + nl = data.find("\n", match.end()) + line_end = len(data) if nl < 0 else nl + 1 + spans = [(ms - line_start, match.end() - line_start)] if compute_spans else None + seen[line_start] = len(results) + results.append((0, MATCH, data[line_start:line_end], spans)) + return (None, results) + build_match_context = BuildMatchContext( block=block, before=self.options.before_context, From 914ad89dda7867739e84a9b5a8075ae78b242d13 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 11:08:24 +0200 Subject: [PATCH 35/37] Faster default (line-numbered, no-context) searches Track the line number with an incrementally advanced newline cursor instead of building the whole block's line-offset table and bisecting it per match (~1.1x typical, up to ~1.8x on match-dense files); the full line-offset path is now used only when context (-A/-B/-C) is requested --- grin/main.py | 62 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/grin/main.py b/grin/main.py index 1294e43..c4781e0 100644 --- a/grin/main.py +++ b/grin/main.py @@ -266,6 +266,8 @@ def do_grep(self, fp: types.ReadableFile, encoding: str = "utf8") -> BlockContex opts = self.options show_match = opts.show_match + # Line numbers/context need a line count carried across blocks (and the + # full per-match line bookkeeping); without them we can skip all of it. need_line_info = bool(opts.show_line_numbers or opts.show_emacs or opts.before_context or opts.after_context) block = self.read_block_with_context(None, fp, fp_size, encoding) while block.end > block.start: @@ -294,7 +296,11 @@ def do_grep(self, fp: types.ReadableFile, encoding: str = "utf8") -> BlockContex line_count += block_line_count block = next_block - if not show_match or not need_line_info: + # uniquify_context (sort + cross-line dedup) is only needed for the + # context path, which emits overlapping PRE/POST lines. -l and the + # no-context path already return at most one entry per line, in order. + needs_uniquify = show_match and bool(opts.before_context or opts.after_context) + if not needs_uniquify: return context return self.uniquify_context(context) @@ -328,35 +334,48 @@ def do_grep_block(self, block: DataBlock, line_num_offset: int) -> tuple[int | N return (None, []) opts = self.options - if not (opts.show_line_numbers or opts.show_emacs or opts.before_context or opts.after_context): - # Line-text-only mode: no line numbers, no context. Extract each - # matched line locally by finding the newlines around the match - # instead of building the whole block's line-offset table. Several + if not (opts.before_context or opts.after_context): + # No-context path: surrounding lines are never needed, so skip + # building the whole block's line-offset table. Each matched line is + # extracted locally (find the newlines around the match), and several # matches on one line collapse to a single entry (merging spans); # lines never span blocks, so deduping within the block suffices. - # The line number is irrelevant here, so a placeholder 0 is used. + # + # When line numbers are shown, the line number is tracked with a + # newline cursor advanced from match to match (matches arrive in + # ascending order) instead of bisecting a full offsets array. When + # they are not, a placeholder 0 is used. data = block.data - compute_spans = opts.use_color + numbered = opts.show_line_numbers or opts.show_emacs results: BlockContextType = [] seen: dict[int, int] = {} + cursor_pos = 0 + cursor_line = 0 for match in self.regex_m.finditer(data, block.start, block.end): ms = match.start() line_start = data.rfind("\n", 0, ms) + 1 + new_span = (ms - line_start, match.end() - line_start) idx = seen.get(line_start) if idx is not None: - if compute_spans: - old = results[idx] - old_spans = old[3] - assert old_spans is not None - new_span = (ms - line_start, match.end() - line_start) - results[idx] = (old[0], old[1], old[2], [*old_spans, new_span]) + # Another match on a line already recorded: merge the span so + # every occurrence can be colored (matches the full path, + # which always populates spans regardless of --color). + old = results[idx] + results[idx] = (old[0], old[1], old[2], [*(old[3] or []), new_span]) continue + if numbered: + cursor_line += data.count("\n", cursor_pos, ms) + cursor_pos = ms + lineno = cursor_line + line_num_offset + else: + lineno = 0 nl = data.find("\n", match.end()) line_end = len(data) if nl < 0 else nl + 1 - spans = [(ms - line_start, match.end() - line_start)] if compute_spans else None seen[line_start] = len(results) - results.append((0, MATCH, data[line_start:line_end], spans)) - return (None, results) + results.append((lineno, MATCH, data[line_start:line_end], [new_span])) + # The next block needs this block's line count to offset its numbers. + block_line_count = data.count("\n", block.start, block.end) if numbered else None + return (block_line_count, results) build_match_context = BuildMatchContext( block=block, @@ -384,11 +403,10 @@ def uniquify_context(self, context: BlockContextType) -> BlockContextType: context.sort() unique_context = [] for _lineno, group in itertools.groupby(context, lambda ikl: ikl[0]): + entries = list(group) # groupby never yields an empty group match_entry = None merged_spans: list[tuple[int, int]] = [] - last_entry = None - for entry in group: - last_entry = entry + for entry in entries: if entry[1] == MATCH: # Always prefer a match over PRE/POST context on the same line. if match_entry is None: @@ -400,10 +418,8 @@ def uniquify_context(self, context: BlockContextType) -> BlockContextType: unique_context.append((i, kind, line, merged_spans if merged_spans else None)) else: # No match, only PRE and/or POST lines. Use the last one, which - # should be a POST since we've sorted it that way. (groupby never - # yields an empty group, so last_entry is always set here.) - assert last_entry is not None - unique_context.append(last_entry) + # should be a POST since we've sorted it that way. + unique_context.append(entries[-1]) return unique_context From 2cacc983471561d0f7ee0d5cd6b3126b7e4002a2 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sat, 27 Jun 2026 23:27:18 +0200 Subject: [PATCH 36/37] Faster ignore matching while walking Each ignore source is matched as one pathspec spec instead of rule-by-rule (~1.4x faster walking of rule-heavy trees). If the optional `google-re2` package is installed, pathspec uses it automatically for a further speedup; otherwise the pure-Python matcher is used --- grin/ignore.py | 194 ++++++++++++++++++++++++++++++------------------- 1 file changed, 118 insertions(+), 76 deletions(-) diff --git a/grin/ignore.py b/grin/ignore.py index c50e500..ca52e2f 100644 --- a/grin/ignore.py +++ b/grin/ignore.py @@ -60,7 +60,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from pathspec.pattern import Pattern + from pathspec import GitIgnoreSpec __all__ = ["IgnoreStack"] @@ -71,44 +71,67 @@ HG_DIR = ".hg" +def _relative(base: str, rel: str) -> str | None: + """Return ``rel`` made relative to ``base``, or None if not under it.""" + if not base: + return rel + prefix = base + "/" + if rel.startswith(prefix): + return rel[len(prefix) :] + return None + + @dataclass -class _Rule: - """A single ignore rule, relative to ``base`` (a path relative to the search root).""" +class _GitignoreMatcher: + """One gitignore-syntax source (a single file/origin), relative to ``base``. + + Matching is delegated to pathspec's ``GitIgnoreSpec`` so the whole source is + evaluated in one batched pass (and accelerated by the re2/hyperscan backend + when installed). :meth:`match` returns True (ignore), False (re-included by a + ``!`` negation) or None (no pattern here matched -- defer to lower precedence). + """ base: str - include: bool # True = ignore, False = negation (re-include) - pattern: Pattern | None = None # gitignore/glob rule (pathspec) - regex: re.Pattern[str] | None = None # .hgignore regexp rule - - def _relative(self, rel: str) -> str | None: - """Return ``rel`` made relative to this rule's base, or None if not under it.""" - if not self.base: - return rel - prefix = self.base + "/" - if rel.startswith(prefix): - return rel[len(prefix) :] - return None + spec: GitIgnoreSpec - def matches(self, rel: str, is_dir: bool) -> bool: - sub = self._relative(rel) + def match(self, rel: str, is_dir: bool) -> bool | None: + sub = _relative(self.base, rel) if not sub: - return False - if self.pattern is not None: - if self.pattern.match_file(sub): + return None + # A trailing slash lets directory-only patterns ("build/") match a dir. + # include is True (ignore), False (negated) or None (no pattern matched). + verdict: bool | None = self.spec.check_file(sub + "/" if is_dir else sub).include + return verdict + + +@dataclass +class _RegexMatcher: + """One Mercurial .hgignore regexp source. hg has no negation, so :meth:`match` + returns True (ignore) or None (no match).""" + + base: str + regexes: list[re.Pattern[str]] + + def match(self, rel: str, is_dir: bool) -> bool | None: + sub = _relative(self.base, rel) + if not sub: + return None + for regex in self.regexes: + if regex.search(sub) is not None: return True - # Directory-only patterns ("build/") match only with a trailing slash. - return is_dir and bool(self.pattern.match_file(sub + "/")) - if self.regex is not None: - return self.regex.search(sub) is not None - return False + return None + +_Matcher = _GitignoreMatcher | _RegexMatcher -def _gitignore_rules(lines: list[str], base: str) -> list[_Rule]: + +def _gitignore_matcher(lines: list[str], base: str) -> _GitignoreMatcher | None: + """Build a matcher for one gitignore-syntax source, or None if it has no + actionable patterns (blank/comment-only file).""" from pathspec import GitIgnoreSpec - rules: list[_Rule] = [] try: - patterns = GitIgnoreSpec.from_lines(lines).patterns + spec = GitIgnoreSpec.from_lines(lines) except ValueError: # A single malformed line (e.g. a lone "!") makes pathspec reject the # whole file; git skips bad patterns, so compile line-by-line and drop @@ -119,11 +142,11 @@ def _gitignore_rules(lines: list[str], base: str) -> list[_Rule]: patterns.extend(GitIgnoreSpec.from_lines([line]).patterns) except ValueError: continue - for pattern in patterns: - # include is None for blank lines and comments. - if pattern.include is not None: - rules.append(_Rule(base=base, include=pattern.include, pattern=pattern)) - return rules + spec = GitIgnoreSpec(patterns) + # include is None for blank lines and comments; skip sources with none useful. + if not any(pattern.include is not None for pattern in spec.patterns): + return None + return _GitignoreMatcher(base, spec) def _read_lines(path: str) -> list[str] | None: @@ -134,20 +157,27 @@ def _read_lines(path: str) -> list[str] | None: return None -def load_dir_rules(dirpath: str, base: str, names: frozenset[str]) -> list[_Rule]: - """Load .gitignore then .ignore rules from ``dirpath`` (relative to ``base``).""" - rules: list[_Rule] = [] +def load_dir_rules(dirpath: str, base: str, names: frozenset[str]) -> list[_Matcher]: + """Load .gitignore then .ignore matchers from ``dirpath`` (relative to ``base``).""" + matchers: list[_Matcher] = [] for name in GITIGNORE_FILES: if name in names: lines = _read_lines(os.path.join(dirpath, name)) if lines is not None: - rules.extend(_gitignore_rules(lines, base)) - return rules + matcher = _gitignore_matcher(lines, base) + if matcher is not None: + matchers.append(matcher) + return matchers + +def _hgignore_matchers(lines: list[str], base: str) -> list[_Matcher]: + """Parse Mercurial .hgignore-syntax ``lines`` into matchers relative to ``base``. -def _hgignore_rules(lines: list[str], base: str) -> list[_Rule]: - """Parse Mercurial .hgignore-syntax ``lines`` into rules relative to ``base``.""" - rules: list[_Rule] = [] + Returns the regexp rules (as one regex matcher) followed by the glob rules + (as one gitignore matcher), preserving the original rule precedence order. + """ + matchers: list[_Matcher] = [] + regexes: list[re.Pattern[str]] = [] glob_lines: list[str] = [] syntax = "regexp" # Mercurial's default for line in lines: @@ -162,39 +192,47 @@ def _hgignore_rules(lines: list[str], base: str) -> list[_Rule]: glob_lines.append(stripped) else: try: - rules.append(_Rule(base=base, include=True, regex=re.compile(stripped))) + regexes.append(re.compile(stripped)) except re.error: continue + if regexes: + matchers.append(_RegexMatcher(base, regexes)) # hg glob patterns are rooted at the repo; approximate with gitwildmatch. - rules.extend(_gitignore_rules(glob_lines, base)) - return rules + if glob_lines: + matcher = _gitignore_matcher(glob_lines, base) + if matcher is not None: + matchers.append(matcher) + return matchers -def load_hgignore(path: str, base: str = "") -> list[_Rule]: - """Parse a Mercurial .hgignore file into rules relative to ``base``.""" +def load_hgignore(path: str, base: str = "") -> list[_Matcher]: + """Parse a Mercurial .hgignore file into matchers relative to ``base``.""" lines = _read_lines(path) - return _hgignore_rules(lines, base) if lines is not None else [] + return _hgignore_matchers(lines, base) if lines is not None else [] -def load_git_exclude(dirpath: str, base: str, names: frozenset[str]) -> list[_Rule]: +def load_git_exclude(dirpath: str, base: str, names: frozenset[str]) -> list[_Matcher]: """Load ``dirpath/.git/info/exclude`` when ``dirpath`` is a git repo root.""" if GIT_DIR not in names: return [] lines = _read_lines(os.path.join(dirpath, GIT_DIR, "info", "exclude")) - return _gitignore_rules(lines, base) if lines is not None else [] + if lines is None: + return [] + matcher = _gitignore_matcher(lines, base) + return [matcher] if matcher is not None else [] -def load_hg_repo_excludes(dirpath: str, base: str, names: frozenset[str]) -> list[_Rule]: +def load_hg_repo_excludes(dirpath: str, base: str, names: frozenset[str]) -> list[_Matcher]: """Load the ignore files named in ``dirpath/.hg/hgrc`` (hg repo-local config).""" if HG_DIR not in names: return [] - rules: list[_Rule] = [] + matchers: list[_Matcher] = [] for path in _hg_ignore_paths(os.path.join(dirpath, HG_DIR, "hgrc")): path = os.path.expanduser(path) if not os.path.isabs(path): path = os.path.join(dirpath, path) - rules.extend(load_hgignore(path, base)) - return rules + matchers.extend(load_hgignore(path, base)) + return matchers # --- user/XDG VCS config (read once, lowest precedence, applied tree-wide) --- @@ -235,7 +273,7 @@ def _config_paths(env_var: str, xdg_parts: tuple[str, ...], home_name: str) -> l return [os.path.join(xdg, *xdg_parts), os.path.join(home, home_name)] -def load_git_global_excludes() -> list[_Rule]: +def load_git_global_excludes() -> list[_Matcher]: """Load patterns from git's ``core.excludesFile`` (or its XDG default).""" excludes: str | None = None for cfg in _config_paths("GIT_CONFIG_GLOBAL", ("git", "config"), ".gitconfig"): @@ -247,7 +285,10 @@ def load_git_global_excludes() -> list[_Rule]: xdg = os.environ.get("XDG_CONFIG_HOME") or os.path.join(home, ".config") excludes = os.path.join(xdg, "git", "ignore") lines = _read_lines(os.path.expanduser(excludes)) - return _gitignore_rules(lines, "") if lines is not None else [] + if lines is None: + return [] + matcher = _gitignore_matcher(lines, "") + return [matcher] if matcher is not None else [] def _hg_ignore_paths(cfg_path: str) -> list[str]: @@ -259,29 +300,29 @@ def _hg_ignore_paths(cfg_path: str) -> list[str]: return paths -def load_hg_global_excludes() -> list[_Rule]: +def load_hg_global_excludes() -> list[_Matcher]: """Load the ignore files named by ``[ui] ignore`` in the user/XDG hg config.""" - rules: list[_Rule] = [] + matchers: list[_Matcher] = [] for cfg in _config_paths("HGRCPATH", ("hg", "hgrc"), ".hgrc"): for path in _hg_ignore_paths(cfg): lines = _read_lines(os.path.expanduser(path)) if lines is not None: - rules.extend(_hgignore_rules(lines, "")) - return rules + matchers.extend(_hgignore_matchers(lines, "")) + return matchers @dataclass class IgnoreStack: - """Accumulated ignore rules from a search root down to the current directory. + """Accumulated ignore matchers from a search root down to the current directory. - Immutable: :meth:`push` returns a new stack. Rules are stored in order - (root first, deeper directories later) and evaluated with git precedence: - the last matching rule wins, so a deeper or later ``!`` negation re-includes. - User-global excludes sit at the bottom; per-repo and in-tree rules added - while walking override them. + Immutable: :meth:`push` returns a new stack. Matchers are stored in order + (root first, deeper directories later) and evaluated with git precedence: the + last matcher with a definite verdict wins, so a deeper or later ``!`` negation + re-includes. User-global excludes sit at the bottom; per-repo and in-tree + sources added while walking override them. """ - rules: tuple[_Rule, ...] = () + matchers: tuple[_Matcher, ...] = () @classmethod def for_root(cls, root: str) -> "IgnoreStack": @@ -292,19 +333,20 @@ def for_root(cls, root: str) -> "IgnoreStack": def push(self, dirpath: str, base: str, names: frozenset[str]) -> "IgnoreStack": # Lower precedence first: per-repo excludes, then in-tree ignore files, # so an in-tree .gitignore/.ignore overrides a per-repo exclude here. - new_rules: list[_Rule] = [] - new_rules += load_git_exclude(dirpath, base, names) - new_rules += load_hg_repo_excludes(dirpath, base, names) + new_matchers: list[_Matcher] = [] + new_matchers += load_git_exclude(dirpath, base, names) + new_matchers += load_hg_repo_excludes(dirpath, base, names) if HGIGNORE_FILE in names: - new_rules += load_hgignore(os.path.join(dirpath, HGIGNORE_FILE), base) - new_rules += load_dir_rules(dirpath, base, names) - if not new_rules: + new_matchers += load_hgignore(os.path.join(dirpath, HGIGNORE_FILE), base) + new_matchers += load_dir_rules(dirpath, base, names) + if not new_matchers: return self - return IgnoreStack(self.rules + tuple(new_rules)) + return IgnoreStack(self.matchers + tuple(new_matchers)) def match(self, rel: str, is_dir: bool) -> bool: ignored = False - for rule in self.rules: - if rule.matches(rel, is_dir): - ignored = rule.include + for matcher in self.matchers: + verdict = matcher.match(rel, is_dir) + if verdict is not None: + ignored = verdict return ignored From 3e0ef36bef5ff15d9ae8c90de7791025894dea08 Mon Sep 17 00:00:00 2001 From: Raffaele Salmaso Date: Sun, 28 Jun 2026 09:37:26 +0200 Subject: [PATCH 37/37] Add an optional `re2` extra Installs google-re2, which pathspec then uses automatically to accelerate ignore-file matching further. The default install stays pure-Python (pathspec's built-in matcher is the fallback) --- README.md | 17 +++++++++++++++++ pyproject.toml | 3 +++ 2 files changed, 20 insertions(+) diff --git a/README.md b/README.md index a3ac16a..76db4ed 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,21 @@ per-repo excludes apply only when its root is at or below where the search starts. Paths given explicitly on the command line are always searched, never ignored. +### Faster ignore matching (optional) + +Ignore-file matching is pure Python by default. For an extra speedup on large +trees with many ignore rules, install the optional `re2` extra, which pulls in +[`google-re2`]; [`pathspec`] then uses it automatically: + +```bash +$ python3 -m pip install "grin3[re2]" +``` + +This is entirely optional — without it grin uses pathspec's built-in pure-Python +matcher. Note that `google-re2` is a compiled package and currently ships no +free-threaded wheels, so the extra may not install on a free-threaded interpreter; +the pure-Python matcher is used there. + ## Parallelism grin searches files in parallel by default, using one worker per CPU. The output @@ -363,3 +378,5 @@ $ uv run --python 3.15 -m unittest discover tests [find]: http://www.gnu.org/software/findutils/ [pip]: https://pip.pypa.io/en/stable/ [ruff]: https://docs.astral.sh/ruff/ +[pathspec]: https://pypi.org/project/pathspec/ +[google-re2]: https://pypi.org/project/google-re2/ diff --git a/pyproject.toml b/pyproject.toml index 20cc010..6f7df94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,9 @@ classifiers = [ "Topic :: Utilities", ] +[project.optional-dependencies] +re2 = ["pathspec[re2]"] + [project.urls] Github = "https://github.com/rsalmaso/grin3" Gitlab = "https://gitlab.com/rsalmaso/grin3"