Fix crash when arpa line is not properly formatted

git-svn-id: svn+ssh://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase@13273 94700074-3cef-4d97-a70e-9c8c206c02f5

Author: nshmyrev

src/libsphinxbase/lm/ngram_model_trie.c

@@ -119,12 +119,12 @@ read_1grams_arpa(lineiter_t ** li, uint32 count, ngram_model_t * base,
         *li = lineiter_next(*li);
         if (*li == NULL) {
             E_ERROR
-                ("Unexpected end of ARPA file. Failed to read %dth unigram\n",
+                ("Unexpected end of ARPA file. Failed to read unigram %d\n",
                  i + 1);
             return -1;
         }
         if ((n = str2words((*li)->buf, wptr, 3)) < n_parts) {
-            E_ERROR("Format error at line %s, Failed to read unigrams\n", (*li)->buf);
+            E_ERROR("Format error at line %d, Failed to read unigrams\n", (*li)->lineno);
             return -1;
         }
 

src/libsphinxbase/lm/ngrams_raw.c

@@ -68,28 +68,20 @@ ngram_ord_comparator(const void *a_raw, const void *b_raw)
 }
 
 static int
-read_ngram_instance(lineiter_t ** li, hash_table_t * wid,
-                    logmath_t * lmath, int order, int order_max,
-                    ngram_raw_t * raw_ngram)
+ngrams_raw_read_line(lineiter_t *li, hash_table_t *wid,
+                    logmath_t *lmath, int order, int order_max,
+                    ngram_raw_t *raw_ngram)
 {
-    int n;
+    int n, i;
     int words_expected;
-    int i;
     char *wptr[NGRAM_MAX_ORDER + 1];
     uint32 *word_out;
 
-    if (*li) 
-        *li = lineiter_next(*li);
-    if (*li == NULL) {
-        E_ERROR("Unexpected end of ARPA file. Failed to read %d-gram\n",
-                order);
-        return -1;
-    }
     words_expected = order + 1;
     if ((n =
-         str2words((*li)->buf, wptr,
+         str2words(li->buf, wptr,
                    NGRAM_MAX_ORDER + 1)) < words_expected) {
-        E_ERROR("Format error; %d-gram ignored: %s\n", order, (*li)->buf);
+        E_ERROR("Format error; %d-gram ignored at line %d\n", order, li->lineno);
         return -1;
     }
 
@@ -136,12 +128,12 @@ read_ngram_instance(lineiter_t ** li, hash_table_t * wid,
 }
 
 static int
-ngrams_raw_read_order(ngram_raw_t ** raw_ngrams, lineiter_t ** li,
-                      hash_table_t * wid, logmath_t * lmath, uint32 count,
+ngrams_raw_read_section(ngram_raw_t ** raw_ngrams, lineiter_t ** li,
+                      hash_table_t * wid, logmath_t * lmath, uint32 *count,
                       int order, int order_max)
 {
     char expected_header[20];
-    uint32 i;
+    uint32 i, cur;
 
     sprintf(expected_header, "\\%d-grams:", order);
     while (*li && strcmp((*li)->buf, expected_header) != 0) {
@@ -153,14 +145,21 @@ ngrams_raw_read_order(ngram_raw_t ** raw_ngrams, lineiter_t ** li,
 	return -1;
     }
 
-    *raw_ngrams = (ngram_raw_t *) ckd_calloc(count, sizeof(ngram_raw_t));
-    for (i = 0; i < count; i++) {
-        if (read_ngram_instance(li, wid, lmath, order, order_max,
-                            &((*raw_ngrams)[i])) < 0)
-            break;
+    *raw_ngrams = (ngram_raw_t *) ckd_calloc(*count, sizeof(ngram_raw_t));
+    for (i = 0, cur = 0; i < *count && *li != NULL; i++) {
+	*li = lineiter_next(*li);
+        if (*li == NULL) {
+	    E_ERROR("Unexpected end of ARPA file. Failed to read %d-gram\n",
+                order);
+    	    return -1;
+	}
+        if (ngrams_raw_read_line(*li, wid, lmath, order, order_max,
+                                *raw_ngrams + cur) == 0) {
+            cur++;
+        }
     }
-
-    qsort(*raw_ngrams, count, sizeof(ngram_raw_t), &ngram_ord_comparator);
+    *count = cur;
+    qsort(*raw_ngrams, *count, sizeof(ngram_raw_t), &ngram_ord_comparator);
     return 0;
 }
 
@@ -175,8 +174,8 @@ ngrams_raw_read_arpa(lineiter_t ** li, logmath_t * lmath, uint32 * counts,
         (ngram_raw_t **) ckd_calloc(order - 1, sizeof(*raw_ngrams));
 
     for (order_it = 2; order_it <= order; order_it++) {
-        if (ngrams_raw_read_order(&raw_ngrams[order_it - 2], li, wid, lmath,
-                              counts[order_it - 1], order_it, order) < 0)
+        if (ngrams_raw_read_section(&raw_ngrams[order_it - 2], li, wid, lmath,
+                              counts + order_it - 1, order_it, order) < 0)
         break;
     }
 

test/unit/test_ngram/107.lm.gz

@@ -31,6 +31,7 @@ EXTRA_DIST = \
 	104.lm.gz \
 	105.lm.gz \
 	106.lm.gz \
+	107.lm.gz \
 	turtle.lm \
 	turtle.lm.dmp \
 	turtle.ug.lm \

test/unit/test_ngram/Makefile.am

@@ -73,6 +73,10 @@ main(int argc, char *argv[])
 	model = ngram_model_read(NULL, LMDIR "/106.lm.gz", NGRAM_ARPA, lmath);
 	TEST_EQUAL(NULL, model);
 
+	/* Read corrupted language model with wrong format line, error expected */
+	model = ngram_model_read(NULL, LMDIR "/107.lm.gz", NGRAM_ARPA, lmath);
+	TEST_EQUAL(0, ngram_model_free(model));
+
 	/* Read a language model */
 	model = ngram_model_read(NULL, LMDIR "/100.lm.bz2", NGRAM_ARPA, lmath);
 	test_lm_vals(model);