xapian_wrap.cpp: query verbatim for terms using fullwidth form

This works around an issue with Xapian 1.5 when indexing and
querying for text containing codepoints in the Halfwidth and
Fullwidth Forms Unicode block.

It does so by querying for both the terms as generated by the
query parser and the form as generated by the term generator
during indexing. This only is applied if the query actually
contains at least on codepoint in that Unicode block.

The issue has been reported to the Xapian mailing list at
https://lists.xapian.org/pipermail/xapian-devel/2024-January/003410.html

Signed-off-by: Robert Stepanek <[email protected]>

Author: rsto

Diff for: cassandane/tiny-tests/JMAPEmail/email_query_cjk_fullwidth

@@ -0,0 +1,70 @@
+#!perl
+use Cassandane::Tiny;
+
+sub test_email_query_cjk_fullwidth
+    :min_version_3_9 :needs_component_jmap :JMAPExtensions
+{
+    my ($self) = @_;
+    my $jmap = $self->{jmap};
+
+use utf8;
+    my $res = $jmap->CallMethods([
+        ['Email/set', {
+            create => {
+                email1 => {
+                    mailboxIds => {
+                        '$inbox' => JSON::true
+                    },
+                    from => [{ email => 'foo@local' }],
+                    to => [{ email => 'bar@local' }],
+                    subject => $_->{id},
+                    bodyStructure => {
+                        type => 'text/plain',
+                        partId => 'part1',
+                    },
+                    bodyValues => {
+                        part1 => {
+                            value =>  <<'EOF'
+三菱ＵＦＪファクター株式会社
+EOF
+                        },
+                    },
+                },
+            },
+        }, 'R1'],
+    ]);
+    my $email1Id = $res->[0][1]{created}{email1}{id};
+    $self->assert_not_null($email1Id);
+
+    xlog $self, "run squatter";
+    $self->{instance}->run_command({cyrus => 1}, 'squatter');
+
+    my $res = $jmap->CallMethods([
+        ['Email/query', {
+            filter => {
+                body => "ＵＦＪ",
+            },
+        }, 'R1'],
+        ['Email/query', {
+            filter => {
+                body => "三菱ＵＦＪファクター株式会社",
+            },
+        }, 'R2'],
+        ['Email/query', {
+            filter => {
+                body => "三菱ＵＦＪ",
+            },
+        }, 'R3'],
+        ['Email/query', {
+            filter => {
+                body => "三菱",
+            },
+        }, 'R4'],
+    ]);
+    $self->assert_deep_equals([$email1Id], $res->[0][1]{ids});
+    $self->assert_deep_equals([$email1Id], $res->[1][1]{ids});
+    $self->assert_deep_equals([$email1Id], $res->[2][1]{ids});
+    $self->assert_deep_equals([$email1Id], $res->[3][1]{ids});
+
+no utf8;
+}

Diff for: imap/xapian_wrap.cpp

@@ -720,21 +720,30 @@ struct xapian_dbw
     std::vector<std::string> *subjects;
 };
 
-
-static int xapian_dbw_init(xapian_dbw_t *dbw)
+static Xapian::TermGenerator *new_term_generator(void)
 {
-    dbw->default_stemmer = new Xapian::Stem(new CyrusSearchStemmer);
-    dbw->default_stopper = get_stopper("en");
-    dbw->term_generator = new Xapian::TermGenerator;
-    dbw->term_generator->set_max_word_length(XAPIAN_MAX_TERM_LENGTH);
+    auto termgen = new Xapian::TermGenerator;
+    termgen->set_max_word_length(XAPIAN_MAX_TERM_LENGTH);
     /* Always enable CJK word tokenization */
-#ifdef USE_XAPIAN_CJK_WORDS
-    dbw->term_generator->set_flags(Xapian::TermGenerator::FLAG_CJK_WORDS,
+#if defined(USE_XAPIAN_WORD_BREAKS)
+    termgen->set_flags(Xapian::TermGenerator::FLAG_WORD_BREAKS,
+            ~Xapian::TermGenerator::FLAG_WORD_BREAKS);
+#elif defined(USE_XAPIAN_CJK_WORDS)
+    termgen->set_flags(Xapian::TermGenerator::FLAG_CJK_WORDS,
             ~Xapian::TermGenerator::FLAG_CJK_WORDS);
 #else
-    dbw->term_generator->set_flags(Xapian::TermGenerator::FLAG_CJK_NGRAM,
+    termgen->set_flags(Xapian::TermGenerator::FLAG_CJK_NGRAM,
             ~Xapian::TermGenerator::FLAG_CJK_NGRAM);
 #endif
+    return termgen;
+}
+
+
+static int xapian_dbw_init(xapian_dbw_t *dbw)
+{
+    dbw->default_stemmer = new Xapian::Stem(new CyrusSearchStemmer);
+    dbw->default_stopper = get_stopper("en");
+    dbw->term_generator = new_term_generator();
     dbw->doclangs = new std::set<std::string>;
     dbw->subjects = new std::vector<std::string>;
     return 0;
@@ -1865,6 +1874,52 @@ static Xapian::Query *query_new_type(const xapian_db_t *db __attribute__((unused
     return new Xapian::Query(q);
 }
 
+static Xapian::Query* xapian_query_new_match_cjk(const xapian_db_t *db, const char *str, const char *prefix)
+{
+    Xapian::Query *q = new Xapian::Query {db->parser->parse_query(
+            str,
+#if defined(USE_XAPIAN_WORD_BREAKS)
+            Xapian::QueryParser::FLAG_WORD_BREAKS,
+#elif defined(USE_XAPIAN_CJK_WORDS)
+            Xapian::QueryParser::FLAG_CJK_WORDS,
+#else
+            Xapian::QueryParser::FLAG_CJK_NGRAM,
+#endif
+            prefix)};
+
+#if defined(USE_XAPIAN_WORD_BREAKS) || defined(USE_XAPIAN_CJK_WORDS)
+    // There is a bug in Xapian v1.5 and CJK word segmentation, in which
+    // a term starting with a fullwidth Latin character such as U+FF21
+    // is indexed with capitalization, but queried in small letter form
+    // when parsed with parse_query.
+    //
+    // As a workaround, the following code checks if the query string
+    // contains any characters in the Halfwidth and Fullwidth Forms
+    // Unicode block. If so, it queries for either the query as
+    // generated by the query parser, or the terms as generated during
+    // indexing.
+    //
+    // XXX this should better be handled by normalizing characters in
+    // that Unicode block during indexing.
+    for (auto it = Xapian::Utf8Iterator(str); it != Xapian::Utf8Iterator(); ++it) {
+        if (*it >= 0xff00 && *it <= 0xffef) {
+            Xapian::TermGenerator *termgen = new_term_generator();
+            Xapian::Document doc;
+            termgen->set_document(doc);
+            termgen->index_text(str, 1, prefix);
+            Xapian::Query qq{Xapian::Query::OP_AND, doc.termlist_begin(), doc.termlist_end()};
+            if (qq.get_length()) {
+                *q |= qq;
+            }
+            delete termgen;
+            break; // stop at first occurence of full/half-width character
+        }
+    }
+#endif
+
+    return q;
+}
+
 EXPORTED Xapian::Query *
 xapian_query_new_match_internal(const xapian_db_t *db, int partnum, const char *str)
 {
@@ -1895,14 +1950,7 @@ xapian_query_new_match_internal(const xapian_db_t *db, int partnum, const char *
         // Don't stem queries for Thaana codepage (0780) or higher.
         for (const unsigned char *p = (const unsigned char *)str; *p; p++) {
             if (*p > 221) //has highbit
-                return new Xapian::Query {db->parser->parse_query(
-                    str,
-#ifdef USE_XAPIAN_CJK_WORDS
-                    Xapian::QueryParser::FLAG_CJK_WORDS,
-#else
-                    Xapian::QueryParser::FLAG_CJK_NGRAM,
-#endif
-                    prefix)};
+                return xapian_query_new_match_cjk(db, str, prefix);
         }
 
         // Stemable codepage.
@@ -2182,7 +2230,10 @@ static Xapian::Query xapian_snipgen_build_query(xapian_snipgen_t *snipgen, Xapia
     if (snipgen->loose_terms) {
         /* Add loose query terms */
         term_generator.set_stemmer(stemmer);
-#ifdef USE_XAPIAN_CJK_WORDS
+#if defined(USE_XAPIAN_WORD_BREAKS)
+        term_generator.set_flags(Xapian::TermGenerator::FLAG_WORD_BREAKS,
+                ~Xapian::TermGenerator::FLAG_WORD_BREAKS);
+#elif defined(USE_XAPIAN_CJK_WORDS)
         term_generator.set_flags(Xapian::TermGenerator::FLAG_CJK_WORDS,
                 ~Xapian::TermGenerator::FLAG_CJK_WORDS);
 #else
@@ -2203,7 +2254,9 @@ static Xapian::Query xapian_snipgen_build_query(xapian_snipgen_t *snipgen, Xapia
         /* Add phrase queries */
         unsigned flags = Xapian::QueryParser::FLAG_PHRASE|
                          Xapian::QueryParser::FLAG_WILDCARD|
-#ifdef USE_XAPIAN_CJK_WORDS
+#if defined(USE_XAPIAN_WORD_BREAKS)
+                         Xapian::QueryParser::FLAG_WORD_BREAKS;
+#elif defined(USE_XAPIAN_CJK_WORDS)
                          Xapian::QueryParser::FLAG_CJK_WORDS;
 #else
                          Xapian::QueryParser::FLAG_CJK_NGRAM;
@@ -2267,7 +2320,9 @@ EXPORTED int xapian_snipgen_make_snippet(xapian_snipgen_t *snipgen,
 
         unsigned flags = Xapian::MSet::SNIPPET_EXHAUSTIVE |
                          Xapian::MSet::SNIPPET_EMPTY_WITHOUT_MATCH;
-#ifdef USE_XAPIAN_CJK_WORDS
+#if defined(USE_XAPIAN_WORD_BREAKS)
+        flags |= Xapian::MSet::SNIPPET_WORD_BREAKS;
+#elif defined(USE_XAPIAN_CJK_WORDS)
         flags |= Xapian::MSet::SNIPPET_CJK_WORDS;
 #endif