changed tokenizers to tokenizer_options due to naming conflict

mmoffatt2 · Dec 9, 2024 · 20656cb · 20656cb
1 parent 82ebac1
commit 20656cb
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 349 deletions.
diff --git a/data/create_new_dataset.sh b/data/create_new_dataset.sh
@@ -7,6 +7,8 @@ pushd "$new_dataset"
 # Use softlinks so we can use template/prepare.py for development
 ln -s ../template/prepare.py prepare.py
 ln -s ../template/utils ./utils
+ln -s ../template/tests.py tests.py
+ln -s ../template/tokenizer_options.py tokenizer_options.py
 
 # Different datasets may have different phoneme sets
 cp ../template/get_dataset.sh get_dataset.sh

diff --git a/data/template/prepare.py b/data/template/prepare.py
@@ -2,7 +2,7 @@
 import pickle
 import argparse
 import numpy as np
-from tokenizers import (
+from tokenizer_options import (
     NumericRangeTokenizer,
     SentencePieceTokenizer,
     TiktokenTokenizer,

diff --git a/data/template/tests.py b/data/template/tests.py
@@ -3,7 +3,7 @@
 import unittest
 import os
 import sys  # Import sys to exit with error codes
-from tokenizers import (
+from tokenizer_options import (
     NumericRangeTokenizer,
     SentencePieceTokenizer,
     TiktokenTokenizer,

diff --git a/data/template/tokenizers.py b/data/template/tokenizers.py