Fixes bug reported in JuliaText/TextAnalysis.jl#149

zgornel · May 7, 2019 · db5bed3 · db5bed3
1 parent b250d15
commit db5bed3
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/src/stemmer.jl b/src/stemmer.jl
@@ -155,7 +155,7 @@ end
 
 function stem!(stemmer::Stemmer, d::NGramDocument)
     for token in keys(d.ngrams)
-        new_token = stem(stemmer, token)
+        new_token = join(stem(stemmer, split(token)), " ")
         if new_token != token
             if haskey(d.ngrams, new_token)
                 d.ngrams[new_token] = d.ngrams[new_token] + d.ngrams[token]

diff --git a/test/preprocessing.jl b/test/preprocessing.jl
@@ -21,7 +21,7 @@
     map(x->prepare!(x, strip_everything_stem), [sdoc, ndoc, tdoc, crps])
     @test prepare(poem_no_1, strip_everything_stem) == "pin"
     @test text(sdoc) == "pin"
-    @test ngrams(ndoc) == Dict("tag"=>2,"pin "=>1,"hold"=>1,"thrill "=>1)
+    @test ngrams(ndoc) == Dict("tag"=>2,"pin"=>1,"hold"=>1,"thrill"=>1)
     @test tokens(tdoc) == ["pin ", "tag", "hold", "tag", "thrill "]
     @test text(crps[1]) == "pin"
     # Flag generation

diff --git a/test/stemmer.jl b/test/stemmer.jl
@@ -75,4 +75,15 @@
     @test try stem!(fdoc); false
               catch; true end
 
+    # Test 3-gram document
+    doc = NGramDocument("parts of language", DocumentMetadata(), 3)
+    stem!(doc)
+    @test begin
+        "of" in keys(doc.ngrams)
+        "languag" in keys(doc.ngrams)
+        "part of languag" in keys(doc.ngrams)
+        "part of" in keys(doc.ngrams)
+        "part" in keys(doc.ngrams)
+        "of languag" in keys(doc.ngrams)
+    end
 end