Skip to content

Commit

Permalink
Remove dependency on ftfy
Browse files Browse the repository at this point in the history
  • Loading branch information
honnibal committed Mar 28, 2018
1 parent 6d2c85f commit 070b6c6
Showing 1 changed file with 9 additions and 1 deletion.
10 changes: 9 additions & 1 deletion spacy/cli/init_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
from ..vectors import Vectors
from ..util import prints, ensure_path, get_lang_class

try:
import ftfy
except ImportError:
ftfy = None


@plac.annotations(
lang=("model language", "positional", None, str),
Expand Down Expand Up @@ -140,11 +145,14 @@ def read_freqs(freqs_loc, max_length=100, min_doc_freq=5, min_freq=50):
def read_clusters(clusters_loc):
print("Reading clusters...")
clusters = {}
if ftfy is None:
print("Warning: No text fixing. Run pip install ftfy if necessary")
with clusters_loc.open() as f:
for line in tqdm(f):
try:
cluster, word, freq = line.split()
word = fix_text(word)
if ftfy is not None:
word = fix_text(word)
except ValueError:
continue
# If the clusterer has only seen the word a few times, its
Expand Down

0 comments on commit 070b6c6

Please sign in to comment.