Allow Hiragana and Katakana in the text segmenter

Author: Alex Amies

dicttypes/dicttypes.go

@@ -2,7 +2,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//    http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -59,7 +59,7 @@ func CloneWord(w Word) Word {
 // Only looks at the first charater in the string
 func IsCJKChar(character string) bool {
 	r := []rune(character)
-	return unicode.Is(unicode.Han, r[0]) && !unicode.IsPunct(r[0])
+	return (unicode.Is(unicode.Han, r[0]) || unicode.Is(unicode.Hiragana, r[0]) || unicode.Is(unicode.Katakana, r[0])) && !unicode.IsPunct(r[0])
 }
 
 // Tests whether the word is a function word

dicttypes/dicttypes_test.go

@@ -51,7 +51,7 @@ func makeHW2() Word {
 	}
 }
 
-// TestAddWordSense2Map does a query expecting empty list
+// TestCloneWord does a query expecting empty list
 func TestCloneWord(t *testing.T) {
 	w1 := Word{
 		Simplified:  "你好",
@@ -65,7 +65,7 @@ func TestCloneWord(t *testing.T) {
 	}
 }
 
-// TestAddWordSense2Map does a query expecting empty list
+// TestIsProperNoun does a query expecting empty list
 func TestIsProperNoun(t *testing.T) {
 	s := WordSense{
 		Simplified:  "王",
@@ -273,3 +273,40 @@ func TestIsQuote(t *testing.T) {
 		}
 	}
 }
+
+// TestIsCJKChar tests IsCJKChar
+func TestIsCJKChar(t *testing.T) {
+	type test struct {
+		name   string
+		input  string
+		expect bool
+	}
+	tests := []test{
+		{
+			name:   "Chinese",
+			input:  "中国",
+			expect: true,
+		},
+		{
+			name:   "English",
+			input:  "USA",
+			expect: false,
+		},
+		{
+			name:   "Hiragana",
+			input:  "あ",
+			expect: true,
+		},
+		{
+			name:   "Katakana",
+			input:  "ア",
+			expect: true,
+		},
+	}
+	for _, tc := range tests {
+		got := IsCJKChar(tc.input)
+		if got != tc.expect {
+			t.Errorf("%s: IsCJKChar(%s) got %t but expected %t ", tc.name, tc.input, got, tc.expect)
+		}
+	}
+}