Finish re2

Author: cpburnz

.github/workflows/ci.yaml

@@ -15,9 +15,24 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu, macos, windows]
-        python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "pypy-3.9", "pypy-3.10", "pypy-3.11"]
-        backend: [base, hyperscan]
+        os:
+          - ubuntu
+          - macos
+          - windows
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13"
+          - "3.14"
+          - "pypy-3.9"
+          - "pypy-3.10"
+          - "pypy-3.11"
+        backend:
+          - base
+          - hyperscan
+          - re2
         exclude:
           # Hyperscan does hot have wheels for Python 3.9 on Windows.
           - os: windows

CHANGES.rst

@@ -10,9 +10,17 @@ Major changes:
 
 - `Issue #91`_: Dropped support of EoL Python 3.8.
 
+- Added concept of backends to allow for faster regex matching. The backend can be controlled using the `backend` argument to `PathSpec()`, `PathSpec.from_lines()`, `GitIgnoreSpec()`, and `GitIgnoreSpec.from_lines()`.
+
+API changes:
+
+- Protected method `pathspec.pathspec.PathSpec._match_file()` (with a leading underscore) has been removed and replaced by backends. This does not affect normal usage of `PathSpec()` and `GitIgnoreSpec()`. Only custom subclasses will be affected. If this breaks your usage, `open an issue <https://github.com/cpburnz/python-pathspec/issues>`_.
+
 New features:
 
-- Added optional `hyperscan`_ backend for faster regex matching. It will automatically be used when installed. The backend can be controlled using the `backend` argument to `PathSpec()`, `PathSpec.from_lines()`, `GitIgnoreSpec()`, or `GitIgnoreSpec.from_lines()`.
+- Added optional `hyperscan`_ backend. It will automatically be used when installed. This dependency can be installed with ``pip install 'pathspec[hyperscan]'``.
+
+- Added optional "re2" backend using the `google-re2`_ library. It will automatically be used when installed. This dependency can be installed with ``pip install 'pathspec[google-re2]'``.
 
 Bug fixes:
 
@@ -23,9 +31,12 @@ Improvements:
 - Mark Python 3.13 and 3.14 as supported.
 - No-op patterns are now filtered out when matching files, slightly improving performance.
 
-.. _`hyperscan`: https://pypi.org/project/hyperscan/
+
+.. _`Issue #38`: https://github.com/cpburnz/python-pathspec/issues/38
 .. _`Issue #91`: https://github.com/cpburnz/python-pathspec/issues/91
 .. _`Issue #98`: https://github.com/cpburnz/python-pathspec/issues/98
+.. _`google-re2`: https://pypi.org/project/google-re2/
+.. _`hyperscan`: https://pypi.org/project/hyperscan/
 
 
 0.12.1 (2023-12-10)

benchmarks/bench_gitignore_many_to_many.py

@@ -26,6 +26,8 @@
 GROUP = "GitIgnore.match_files(): 180 lines, 6.5k files"
 
 
+# Hyperscan backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_hs_r1_block_closure(
 	benchmark: BenchmarkFixture,
@@ -138,6 +140,23 @@ def bench_hs_v1(
 	benchmark(run_match, spec, cpython_files)
 
 
+# Re2 backend.
+
+@pytest.mark.benchmark(group=GROUP)
+def bench_re2_v1(
+	benchmark: BenchmarkFixture,
+	cpython_files: set[str],
+	cpython_gi_lines_all: list[str],
+):
+	spec = GitIgnoreSpec.from_lines(
+		cpython_gi_lines_all,
+		backend='re2',
+	)
+	benchmark(run_match, spec, cpython_files)
+
+
+# Simple backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_sm_filtered(
 	benchmark: BenchmarkFixture,

benchmarks/bench_gitignore_many_to_one_end.py

@@ -25,6 +25,8 @@
 GROUP = "GitIgnore.match_file(): 180 lines, one file (end)"
 
 
+# Hyperscan backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_hs_r1_block_closure(
 	benchmark: BenchmarkFixture,
@@ -122,6 +124,23 @@ def bench_hs_v1(
 	benchmark(run_match, spec, cpython_file_match_end)
 
 
+# Re2 backend.
+
+@pytest.mark.benchmark(group=GROUP)
+def bench_re2_v1(
+	benchmark: BenchmarkFixture,
+	cpython_file_match_end: str,
+	cpython_gi_lines_all: list[str],
+):
+	spec = GitIgnoreSpec.from_lines(
+		cpython_gi_lines_all,
+		backend='re2',
+	)
+	benchmark(run_match, spec, cpython_file_match_end)
+
+
+# Simple backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_sm_filtered(
 	benchmark: BenchmarkFixture,

benchmarks/bench_gitignore_many_to_one_middle.py

@@ -25,6 +25,8 @@
 GROUP = "GitIgnore.match_file(): 180 lines, one file (middle)"
 
 
+# Hyperscan backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_hs_r1_block_closure(
 	benchmark: BenchmarkFixture,
@@ -122,6 +124,23 @@ def bench_hs_v1(
 	benchmark(run_match, spec, cpython_file_match_middle)
 
 
+# Re2 backend.
+
+@pytest.mark.benchmark(group=GROUP)
+def bench_re2_v1(
+	benchmark: BenchmarkFixture,
+	cpython_file_match_middle: str,
+	cpython_gi_lines_all: list[str],
+):
+	spec = GitIgnoreSpec.from_lines(
+		cpython_gi_lines_all,
+		backend='re2',
+	)
+	benchmark(run_match, spec, cpython_file_match_middle)
+
+
+# Simple backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_sm_filtered(
 	benchmark: BenchmarkFixture,

benchmarks/bench_gitignore_many_to_one_none.py

@@ -24,6 +24,8 @@
 GROUP = "GitIgnore.match_file(): 180 lines, one file (none)"
 
 
+# Hyperscan backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_hs_r1_block_closure(
 	benchmark: BenchmarkFixture,
@@ -121,6 +123,23 @@ def bench_hs_v1(
 	benchmark(run_match, spec, cpython_file_match_none)
 
 
+# Re2 backend.
+
+@pytest.mark.benchmark(group=GROUP)
+def bench_re2_v1(
+	benchmark: BenchmarkFixture,
+	cpython_file_match_none: str,
+	cpython_gi_lines_all: list[str],
+):
+	spec = GitIgnoreSpec.from_lines(
+		cpython_gi_lines_all,
+		backend='re2',
+	)
+	benchmark(run_match, spec, cpython_file_match_none)
+
+
+# Simple backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_sm_filtered(
 	benchmark: BenchmarkFixture,

benchmarks/bench_gitignore_many_to_one_start.py

@@ -25,6 +25,8 @@
 GROUP = "GitIgnore.match_file(): 180 lines, one file (start)"
 
 
+# Hyperscan backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_hs_r1_block_closure(
 	benchmark: BenchmarkFixture,
@@ -122,6 +124,23 @@ def bench_hs_v1(
 	benchmark(run_match, spec, cpython_file_match_start)
 
 
+# Re2 backend.
+
+@pytest.mark.benchmark(group=GROUP)
+def bench_re2_v1(
+	benchmark: BenchmarkFixture,
+	cpython_file_match_start: str,
+	cpython_gi_lines_all: list[str],
+):
+	spec = GitIgnoreSpec.from_lines(
+		cpython_gi_lines_all,
+		backend='re2',
+	)
+	benchmark(run_match, spec, cpython_file_match_start)
+
+
+# Simple backend.
+
 @pytest.mark.benchmark(group=GROUP)
 def bench_sm_filtered(
 	benchmark: BenchmarkFixture,

benchmarks/match_gitignore.py

@@ -109,7 +109,7 @@ def on_match(
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 
 class HyperscanGiR1BlockStateBackend(_HyperscanGiR1BlockBaseBackend):
@@ -136,7 +136,7 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]:
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 	@override
 	def __on_match(
@@ -242,7 +242,7 @@ def on_match(
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 
 # WARNING: This segfaults.
@@ -269,7 +269,7 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]:
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 	@override
 	def __on_match(
@@ -452,7 +452,7 @@ def on_match(
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 
 class HyperscanGiR2BlockStateBackend(_HyperscanGiR2BlockBaseBackend):
@@ -477,7 +477,7 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]:
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 	@override
 	def __on_match(
@@ -571,4 +571,4 @@ def on_match(
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)

benchmarks/match_pathspec.py

@@ -124,7 +124,7 @@ def on_match(
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 
 class HyperscanPsR1BlockStateBackend(_HyperscanPsR1BlockBaseBackend):
@@ -146,7 +146,7 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]:
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 	@override
 	def __on_match(
@@ -205,7 +205,7 @@ def on_match(
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 
 # WARNING: This segfaults.
@@ -230,7 +230,7 @@ def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]:
 		if out_index == -1:
 			out_index = None
 
-		return out_include, out_index
+		return (out_include, out_index)
 
 	@override
 	def __on_match(

pathspec/_backends/agg.py

@@ -26,6 +26,8 @@
 	HyperscanPsBackend)
 from .re2.base import (
 	re2_error)
+from .re2.gitignore import (
+	Re2GiBackend)
 from .re2.pathspec import (
 	Re2PsBackend)
 from .simple.gitignore import (
@@ -64,7 +66,9 @@ def make_gitignore_backend(
 	if name == 'best':
 		name = _BEST_BACKEND
 
-	if name == 'hyperscan':
+	if name == 're2':
+		return Re2GiBackend(cast(Sequence[RegexPattern], patterns))
+	elif name == 'hyperscan':
 		return HyperscanGiBackend(cast(Sequence[RegexPattern], patterns))
 	elif name == 'simple':
 		return SimpleGiBackend(cast(Sequence[RegexPattern], patterns))
@@ -90,10 +94,10 @@ def make_pathspec_backend(
 	if name == 'best':
 		name = _BEST_BACKEND
 
-	if name == 'hyperscan':
-		return HyperscanPsBackend(cast(Sequence[RegexPattern], patterns))
-	elif name == 're2':
+	if name == 're2':
 		return Re2PsBackend(cast(Sequence[RegexPattern], patterns))
+	elif name == 'hyperscan':
+		return HyperscanPsBackend(cast(Sequence[RegexPattern], patterns))
 	elif name == 'simple':
 		return SimplePsBackend(patterns)
 	else: