Skip to content

Commit a095488

Browse files
mawiesnemawiesne
committed
OPENNLP-1525 Improve TokenizerME to make use of abbreviations provided in TokenizerModel
- enhances TokenizerME impl to make use of abbreviations if available in TokenizerModel - adjusts expectation in TokenizerFactoryTest#testCustomPatternForTokenizerMEDeu to 14 tokens
1 parent 5deae24 commit a095488

File tree

2 files changed

+74
-32
lines changed

2 files changed

+74
-32
lines changed

‎opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java

+37-5
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import opennlp.tools.util.DownloadUtil;
3636
import opennlp.tools.util.ObjectStream;
3737
import opennlp.tools.util.Span;
38+
import opennlp.tools.util.StringList;
3839
import opennlp.tools.util.TrainingParameters;
3940

4041
/**
@@ -110,6 +111,11 @@ public class TokenizerME extends AbstractTokenizer {
110111

111112
private final List<Span> newTokens;
112113

114+
/*
115+
* The {@link Dictionary abbreviation dictionary} if available (may be {@code null}).
116+
*/
117+
private final Dictionary abbDict;
118+
113119
/**
114120
* Initializes a {@link TokenizerME} by downloading a default model.
115121
* @param language The language of the tokenizer.
@@ -132,6 +138,7 @@ public TokenizerME(TokenizerModel model) {
132138
this.model = model.getMaxentModel();
133139
this.useAlphaNumericOptimization = factory.isUseAlphaNumericOptimization();
134140

141+
abbDict = model.getAbbreviations();
135142
newTokens = new ArrayList<>();
136143
tokProbs = new ArrayList<>(50);
137144
}
@@ -151,6 +158,7 @@ public TokenizerME(TokenizerModel model, Factory factory) {
151158
this.model = model.getMaxentModel();
152159
useAlphaNumericOptimization = model.useAlphaNumericOptimization();
153160

161+
abbDict = model.getAbbreviations();
154162
newTokens = new ArrayList<>();
155163
tokProbs = new ArrayList<>(50);
156164
}
@@ -182,6 +190,7 @@ public double[] getTokenProbabilities() {
182190
*
183191
* @return A {@link Span} array containing individual tokens as elements.
184192
*/
193+
@Override
185194
public Span[] tokenizePos(String d) {
186195
WhitespaceTokenizer whitespaceTokenizer = WhitespaceTokenizer.INSTANCE;
187196
whitespaceTokenizer.setKeepNewLines(keepNewLines);
@@ -208,14 +217,22 @@ public Span[] tokenizePos(String d) {
208217
String best = model.getBestOutcome(probs);
209218
tokenProb *= probs[model.getIndex(best)];
210219
if (best.equals(TokenizerME.SPLIT)) {
211-
newTokens.add(new Span(start, j));
212-
tokProbs.add(tokenProb);
213-
start = j;
220+
if (isAcceptableAbbreviation(tok)) {
221+
newTokens.add(new Span(start, end));
222+
tokProbs.add(tokenProb);
223+
start = j + 1; // To compensate for the abbreviation dot
224+
} else {
225+
newTokens.add(new Span(start, j));
226+
tokProbs.add(tokenProb);
227+
start = j;
228+
}
214229
tokenProb = 1.0;
215230
}
216231
}
217-
newTokens.add(new Span(start, end));
218-
tokProbs.add(tokenProb);
232+
if (start < end) {
233+
newTokens.add(new Span(start, end));
234+
tokProbs.add(tokenProb);
235+
}
219236
}
220237
}
221238

@@ -258,4 +275,19 @@ public boolean useAlphaNumericOptimization() {
258275
return useAlphaNumericOptimization;
259276
}
260277

278+
/**
279+
* Allows checking a token abbreviation candidate for acceptability.
280+
*
281+
* <p>Note: The implementation always returns {@code false} if no
282+
* abbreviation dictionary is available for the underlying model.</p>
283+
*
284+
* @param s the {@link CharSequence token} to check for.
285+
* @return {@code true} if the candidate is acceptable, {@code false} otherwise.
286+
*/
287+
protected boolean isAcceptableAbbreviation(CharSequence s) {
288+
if (abbDict == null)
289+
return false;
290+
291+
return abbDict.contains(new StringList(s.toString()));
292+
}
261293
}

‎opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java

+37-27
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
import java.io.ByteArrayInputStream;
2121
import java.io.ByteArrayOutputStream;
2222
import java.io.IOException;
23-
import java.io.InputStream;
2423
import java.nio.charset.StandardCharsets;
24+
import java.util.Locale;
2525
import java.util.regex.Pattern;
2626

2727
import org.junit.jupiter.api.Assertions;
@@ -54,24 +54,28 @@ private static TokenizerModel train(TokenizerFactory factory)
5454
return TokenizerME.train(createSampleStream(), factory, TrainingParameters.defaultParams());
5555
}
5656

57-
private static Dictionary loadAbbDictionary() throws IOException {
58-
InputStream in = TokenizerFactoryTest.class.getClassLoader()
59-
.getResourceAsStream("opennlp/tools/sentdetect/abb.xml");
60-
61-
return new Dictionary(in);
57+
private static Dictionary loadAbbDictionary(Locale loc) throws IOException {
58+
final String abbrevDict;
59+
if (loc.equals(Locale.GERMAN)) {
60+
abbrevDict = "opennlp/tools/sentdetect/abb_DE.xml";
61+
} else {
62+
abbrevDict = "opennlp/tools/sentdetect/abb.xml";
63+
}
64+
return new Dictionary(TokenizerFactoryTest.class.getClassLoader()
65+
.getResourceAsStream(abbrevDict));
6266
}
6367

6468
@Test
6569
void testDefault() throws IOException {
6670

67-
Dictionary dic = loadAbbDictionary();
71+
Dictionary dic = loadAbbDictionary(Locale.ENGLISH);
6872
final String lang = "eng";
6973

7074
TokenizerModel model = train(new TokenizerFactory(lang, dic, false, null));
7175

7276
TokenizerFactory factory = model.getFactory();
7377
Assertions.assertNotNull(factory.getAbbreviationDictionary());
74-
Assertions.assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
78+
Assertions.assertInstanceOf(DefaultTokenContextGenerator.class, factory.getContextGenerator());
7579

7680
String defaultPattern = Factory.DEFAULT_ALPHANUMERIC.pattern();
7781
Assertions.assertEquals(defaultPattern, factory.getAlphaNumericPattern().pattern());
@@ -87,7 +91,7 @@ void testDefault() throws IOException {
8791

8892
factory = fromSerialized.getFactory();
8993
Assertions.assertNotNull(factory.getAbbreviationDictionary());
90-
Assertions.assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
94+
Assertions.assertInstanceOf(DefaultTokenContextGenerator.class, factory.getContextGenerator());
9195

9296
Assertions.assertEquals(defaultPattern, factory.getAlphaNumericPattern().pattern());
9397
Assertions.assertEquals(lang, factory.getLanguageCode());
@@ -105,7 +109,7 @@ void testNullDict() throws IOException {
105109

106110
TokenizerFactory factory = model.getFactory();
107111
Assertions.assertNull(factory.getAbbreviationDictionary());
108-
Assertions.assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
112+
Assertions.assertInstanceOf(DefaultTokenContextGenerator.class, factory.getContextGenerator());
109113

110114
String defaultPattern = Factory.DEFAULT_ALPHANUMERIC.pattern();
111115
Assertions.assertEquals(defaultPattern, factory.getAlphaNumericPattern().pattern());
@@ -121,7 +125,7 @@ void testNullDict() throws IOException {
121125

122126
factory = fromSerialized.getFactory();
123127
Assertions.assertNull(factory.getAbbreviationDictionary());
124-
Assertions.assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
128+
Assertions.assertInstanceOf(DefaultTokenContextGenerator.class, factory.getContextGenerator());
125129

126130
Assertions.assertEquals(defaultPattern, factory.getAlphaNumericPattern().pattern());
127131
Assertions.assertEquals(lang, factory.getLanguageCode());
@@ -141,7 +145,7 @@ void testCustomPatternAndAlphaOpt() throws IOException {
141145

142146
TokenizerFactory factory = model.getFactory();
143147
Assertions.assertNull(factory.getAbbreviationDictionary());
144-
Assertions.assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
148+
Assertions.assertInstanceOf(DefaultTokenContextGenerator.class, factory.getContextGenerator());
145149

146150
Assertions.assertEquals(pattern, factory.getAlphaNumericPattern().pattern());
147151
Assertions.assertEquals(lang, factory.getLanguageCode());
@@ -156,7 +160,7 @@ void testCustomPatternAndAlphaOpt() throws IOException {
156160

157161
factory = fromSerialized.getFactory();
158162
Assertions.assertNull(factory.getAbbreviationDictionary());
159-
Assertions.assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
163+
Assertions.assertInstanceOf(DefaultTokenContextGenerator.class, factory.getContextGenerator());
160164
Assertions.assertEquals(pattern, factory.getAlphaNumericPattern().pattern());
161165
Assertions.assertEquals(lang, factory.getLanguageCode());
162166
Assertions.assertEquals(lang, model.getLanguage());
@@ -165,18 +169,24 @@ void testCustomPatternAndAlphaOpt() throws IOException {
165169

166170
void checkCustomPatternForTokenizerME(String lang, String pattern, String sentence,
167171
int expectedNumTokens) throws IOException {
168-
169-
TokenizerModel model = train(new TokenizerFactory(lang, null, true,
172+
Locale loc = Locale.ENGLISH;
173+
if ("deu".equals(lang)) {
174+
loc = Locale.GERMAN;
175+
}
176+
TokenizerModel model = train(new TokenizerFactory(lang, loadAbbDictionary(loc), true,
170177
Pattern.compile(pattern)));
171178

172179
TokenizerME tokenizer = new TokenizerME(model);
173180
String[] tokens = tokenizer.tokenize(sentence);
174181

175182
Assertions.assertEquals(expectedNumTokens, tokens.length);
176-
String[] sentSplit = sentence.replaceAll("\\.", " .")
177-
.replaceAll("'", " '").split(" ");
183+
String[] sentSplit = sentence.replaceAll("'", " '").split(" ");
178184
for (int i = 0; i < sentSplit.length; i++) {
179-
Assertions.assertEquals(sentSplit[i], tokens[i]);
185+
String sElement = sentSplit[i];
186+
if (i == sentSplit.length - 1) {
187+
sElement = sElement.replace(".", ""); // compensate for sentence ending
188+
}
189+
Assertions.assertEquals(sElement, tokens[i]);
180190
}
181191
}
182192

@@ -185,7 +195,7 @@ void testCustomPatternForTokenizerMEDeu() throws IOException {
185195
String lang = "deu";
186196
String pattern = "^[A-Za-z0-9äéöüÄÉÖÜß]+$";
187197
String sentence = "Ich wähle den auf S. 183 ff. mitgeteilten Traum von der botanischen Monographie.";
188-
checkCustomPatternForTokenizerME(lang, pattern, sentence, 16);
198+
checkCustomPatternForTokenizerME(lang, pattern, sentence, 14);
189199
}
190200

191201
@Test
@@ -267,16 +277,16 @@ void testContractionsEng() throws IOException {
267277
@Test
268278
void testDummyFactory() throws IOException {
269279

270-
Dictionary dic = loadAbbDictionary();
280+
Dictionary dic = loadAbbDictionary(Locale.ENGLISH);
271281
final String lang = "eng";
272282
String pattern = "^[0-9A-Za-z]+$";
273283

274284
TokenizerModel model = train(new DummyTokenizerFactory(lang, dic, true,
275285
Pattern.compile(pattern)));
276286

277287
TokenizerFactory factory = model.getFactory();
278-
Assertions.assertTrue(factory.getAbbreviationDictionary() instanceof DummyDictionary);
279-
Assertions.assertTrue(factory.getContextGenerator() instanceof DummyContextGenerator);
288+
Assertions.assertInstanceOf(DummyDictionary.class, factory.getAbbreviationDictionary());
289+
Assertions.assertInstanceOf(DummyContextGenerator.class, factory.getContextGenerator());
280290
Assertions.assertEquals(pattern, factory.getAlphaNumericPattern().pattern());
281291
Assertions.assertEquals(lang, factory.getLanguageCode());
282292
Assertions.assertEquals(lang, model.getLanguage());
@@ -289,8 +299,8 @@ void testDummyFactory() throws IOException {
289299
TokenizerModel fromSerialized = new TokenizerModel(in);
290300

291301
factory = fromSerialized.getFactory();
292-
Assertions.assertTrue(factory.getAbbreviationDictionary() instanceof DummyDictionary);
293-
Assertions.assertTrue(factory.getContextGenerator() instanceof DummyContextGenerator);
302+
Assertions.assertInstanceOf(DummyDictionary.class, factory.getAbbreviationDictionary());
303+
Assertions.assertInstanceOf(DummyContextGenerator.class, factory.getContextGenerator());
294304
Assertions.assertEquals(pattern, factory.getAlphaNumericPattern().pattern());
295305
Assertions.assertEquals(lang, factory.getLanguageCode());
296306
Assertions.assertEquals(lang, model.getLanguage());
@@ -299,16 +309,16 @@ void testDummyFactory() throws IOException {
299309

300310
@Test
301311
void testCreateDummyFactory() throws IOException {
302-
Dictionary dic = loadAbbDictionary();
312+
Dictionary dic = loadAbbDictionary(Locale.ENGLISH);
303313
final String lang = "eng";
304314
String pattern = "^[0-9A-Za-z]+$";
305315

306316
TokenizerFactory factory = TokenizerFactory.create(
307317
DummyTokenizerFactory.class.getCanonicalName(), lang, dic, true,
308318
Pattern.compile(pattern));
309319

310-
Assertions.assertTrue(factory.getAbbreviationDictionary() instanceof DummyDictionary);
311-
Assertions.assertTrue(factory.getContextGenerator() instanceof DummyContextGenerator);
320+
Assertions.assertInstanceOf(DummyDictionary.class, factory.getAbbreviationDictionary());
321+
Assertions.assertInstanceOf(DummyContextGenerator.class, factory.getContextGenerator());
312322
Assertions.assertEquals(pattern, factory.getAlphaNumericPattern().pattern());
313323
Assertions.assertEquals(lang, factory.getLanguageCode());
314324
Assertions.assertTrue(factory.isUseAlphaNumericOptimization());

0 commit comments

Comments
 (0)