diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index b642403834a..6cbb882d9ba 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit b642403834a67c8ef14a7109dcd1bb5e5f3cb68a +Subproject commit 6cbb882d9baac25c88b8ef38b338123bd2c35dbc diff --git a/pyproject.toml b/pyproject.toml index eb10f570993..2bbcd4d82f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ dependencies=[ "pytest-xdist", "pytest-rerunfailures==15.1", "pytest-json-report", - "pytorch-tokenizers", + "pytorch-tokenizers>=1.2.0", "pyyaml", "ruamel.yaml", "sympy",