Merge pull request #4977 from rsto/xapian_add_messageid

Index Message-ID, References and In-Reply-To headers in Xapian
cyrusimap · Jul 17, 2024 · c4e15bf · c4e15bf
2 parents e8de054 + 904479a
commit c4e15bf
Show file tree

Hide file tree

Showing 22 changed files with 938 additions and 409 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -642,8 +642,8 @@ cunit_TESTS = \
 	cunit/mboxname.testc \
 	cunit/md5.testc \
 	cunit/message.testc \
+	cunit/message_iter_msgid.testc \
 	cunit/message_guid.testc \
-	cunit/msgid.testc \
 	cunit/parseaddr.testc \
 	cunit/parse.testc \
 	cunit/proc.testc \

diff --git a/cassandane/Cassandane/Cyrus/SearchFuzzy.pm b/cassandane/Cassandane/Cyrus/SearchFuzzy.pm
@@ -198,7 +198,7 @@ sub get_snippets
 sub run_delve {
     my ($self, $dir, @args) = @_;
     my $basedir = $self->{instance}->{basedir};
-    my @myargs = ('delve');
+    my @myargs = ('xapian-delve');
     push(@myargs, @args);
     push(@myargs, $dir);
     $self->{instance}->run_command({redirects => {stdout => "$basedir/delve.out"}}, @myargs);

diff --git a/cassandane/Cassandane/Instance.pm b/cassandane/Cassandane/Instance.pm
@@ -533,10 +533,10 @@ sub _find_binary
 
     my $base = $self->{cyrus_destdir} . $self->{cyrus_prefix};
 
-    if ($name eq 'delve') {
+    if ($name =~ m/xapian-.*$/) {
         my $lib = `ldd $base/libexec/imapd` || die "can't ldd imapd";
         $lib =~ m{(/\S+)/lib/libxapian-([0-9.]+)\.so};
-        return "$1/bin/xapian-delve-$2";
+        return "$1/bin/$name-$2";
     }
 
     foreach (qw( bin sbin libexec libexec/cyrus-imapd lib cyrus/bin ))
@@ -571,7 +571,7 @@ sub _binary
     my $cassini = Cassandane::Cassini->instance();
 
     if ($cassini->bool_val('valgrind', 'enabled') &&
-        !($name =~ m/delve$/) &&
+        !($name =~ m/xapian.*$/) &&
         !($name =~ m/\.pl$/) &&
         !($name =~ m/^\//))
     {
@@ -1950,7 +1950,7 @@ sub _fork_command
     {
         push(@cmd, $self->_binary($binary), '-C', $self->_imapd_conf());
     }
-    elsif ($binary eq 'delve') {
+    elsif ($binary =~ m/xapian.*$/) {
         push(@cmd, $self->_binary($binary));
     }
     else {

diff --git a/cassandane/tiny-tests/JMAPEmail/email_query_messageid b/cassandane/tiny-tests/JMAPEmail/email_query_messageid
@@ -0,0 +1,124 @@
+#!perl
+use Cassandane::Tiny;
+
+sub test_email_query_messageid
+  : needs_component_jmap : JMAPExtensions : needs_component_sieve {
+    my ($self) = @_;
+    my $jmap   = $self->{jmap};
+    my $imap   = $self->{store}->get_client();
+
+    $jmap->AddUsing('https://cyrusimap.org/ns/jmap/debug');
+    $jmap->AddUsing('https://cyrusimap.org/ns/jmap/performance');
+
+    my $mime = <<'EOF';
+From: from@local
+To: to@local
+Message-ID: <[email protected]>
+Subject: test
+Date: Mon, 13 Apr 2020 15:34:03 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+test
+EOF
+    $mime =~ s/\r?\n/\r\n/gs;
+    $imap->append('INBOX', $mime) || die $@;
+
+    xlog $self, "run squatter";
+    $self->{instance}->run_command({ cyrus => 1 }, 'squatter');
+
+    xlog $self, "Assert 'messageId' filter condition";
+
+    my $res = $jmap->CallMethods([
+        [ 'Email/query', {}, 'R1' ],
+    ]);
+    my $emailId = $res->[0][1]{ids}[0];
+    $self->assert_not_null($emailId);
+
+    my $res = $jmap->CallMethods([
+        [
+            'Email/query',
+            {
+                filter => {
+                    messageId => '[email protected]',
+                },
+            },
+            'R1'
+        ],
+        [
+            'Email/query',
+            {
+                filter => {
+                    header => [ 'message-id', '[email protected]' ],
+                },
+            },
+            'R2'
+        ],
+    ]);
+    $self->assert_deep_equals([$emailId], $res->[0][1]{ids});
+    $self->assert_deep_equals(['xapian'], $res->[0][1]{performance}{details}{filters});
+    $self->assert_equals(JSON::true, $res->[0][1]{performance}{details}{isGuidSearch});
+    $self->assert_deep_equals([$emailId], $res->[1][1]{ids});
+    $self->assert_deep_equals(['cache'],  $res->[1][1]{performance}{details}{filters});
+    $self->assert_equals(JSON::false, $res->[1][1]{performance}{details}{isGuidSearch});
+
+    xlog $self, "Assert 'messageId' filter in Sieve";
+
+    $imap->create("matches") or die;
+    $self->{instance}->install_sieve_script(
+        <<'EOF'
+require ["x-cyrus-jmapquery", "x-cyrus-log", "variables", "fileinto"];
+if
+  allof( not string :is "${stop}" "Y",
+    jmapquery text:
+  {
+    "messageId" : "[email protected]"
+  }
+.
+  )
+{
+  fileinto "matches";
+}
+EOF
+    );
+
+    $mime = <<'EOF';
+From: from2@local
+To: to2@local
+Message-ID: <[email protected]>
+Subject: test2
+Date: Mon, 13 Apr 2020 15:34:03 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+test
+EOF
+    $mime =~ s/\r?\n/\r\n/gs;
+    my $msg = Cassandane::Message->new();
+    $msg->set_lines(split /\n/, $mime);
+    $self->{instance}->deliver($msg);
+    $self->assert_num_equals(1, $imap->message_count('matches'));
+
+    xlog $self, "Assert 'messageId' filter on legacy index version falls back to cache";
+
+    my $xapdirs = ($self->{instance}->run_mbpath(-u => 'cassandane'))->{xapian};
+    my $xdbpath = $xapdirs->{t1} . "/xapian";
+    $self->{instance}->run_command(
+        {},
+        'xapian-metadata', 'set', $xdbpath, 'cyrus.db_version', '16,17'
+    );
+    $res = $jmap->CallMethods([
+        [
+            'Email/query',
+            {
+                filter => {
+                    messageId => '[email protected]',
+                },
+            },
+            'R1'
+        ],
+    ]);
+    $self->assert_deep_equals([$emailId], $res->[0][1]{ids});
+    $self->assert_deep_equals(['cache'],  $res->[0][1]{performance}{details}{filters});
+    $self->assert_equals(JSON::false, $res->[0][1]{performance}{details}{isGuidSearch});
+}
diff --git a/cassandane/tiny-tests/JMAPEmail/email_query_references_inreplyto b/cassandane/tiny-tests/JMAPEmail/email_query_references_inreplyto
@@ -0,0 +1,177 @@
+#!perl
+use Cassandane::Tiny;
+
+sub test_email_query_references_inreplyto
+  : needs_component_jmap : JMAPExtensions : needs_component_sieve {
+    my ($self) = @_;
+    my $jmap   = $self->{jmap};
+    my $imap   = $self->{store}->get_client();
+
+    $jmap->AddUsing('https://cyrusimap.org/ns/jmap/debug');
+    $jmap->AddUsing('https://cyrusimap.org/ns/jmap/performance');
+
+    xlog $self, "Assert 'inReplyTo' and 'references' filter conditions";
+
+    my $res = $jmap->CallMethods([ [
+        'Email/set',
+        {
+            create => {
+                email1 => {
+                    'header:references' => '<refA@local> <refB@local>',
+                    mailboxIds          => { '$inbox' => JSON::true },
+                    from                => [ { email => 'foo@local' } ],
+                    to                  => [ { email => 'bar@local' } ],
+                    subject             => 'test1',
+                    bodyStructure       => {
+                        type   => 'text/plain',
+                        partId => 'part1',
+                    },
+                    bodyValues => {
+                        part1 => {
+                            value => 'test',
+                        }
+                    }
+                },
+                email2 => {
+                    'header:in-reply-to' => '<replytoA@local>',
+                    mailboxIds           => { '$inbox' => JSON::true },
+                    from                 => [ { email => 'foo@local' } ],
+                    to                   => [ { email => 'bar@local' } ],
+                    subject              => 'test2',
+                    bodyStructure        => {
+                        type   => 'text/plain',
+                        partId => 'part1',
+                    },
+                    bodyValues => {
+                        part1 => {
+                            value => 'test',
+                        }
+                    }
+                }
+            }
+        },
+        'createEmail'
+    ] ]);
+    my $email1Id = $res->[0][1]{created}{email1}{id};
+    $self->assert_not_null($email1Id);
+    my $email2Id = $res->[0][1]{created}{email2}{id};
+    $self->assert_not_null($email2Id);
+
+    xlog $self, "run squatter";
+    $self->{instance}->run_command({ cyrus => 1 }, 'squatter');
+
+    my $res = $jmap->CallMethods([
+        [
+            'Email/query',
+            {
+                filter => {
+                    references => 'refA@local',
+                },
+            },
+            'R1'
+        ],
+        [
+            'Email/query',
+            {
+                filter => {
+                    header => [ 'references', 'refA@local' ],
+                },
+            },
+            'R2'
+        ],
+        [
+            'Email/query',
+            {
+                filter => {
+                    references => 'refB@local',
+                },
+            },
+            'R3'
+        ],
+        [
+            'Email/query',
+            {
+                filter => {
+                    inReplyTo => 'replytoA@local',
+                },
+            },
+            'R4'
+        ],
+    ]);
+    $self->assert_deep_equals([$email1Id], $res->[0][1]{ids});
+    $self->assert_deep_equals(['xapian'],  $res->[0][1]{performance}{details}{filters});
+    $self->assert_equals(JSON::true, $res->[0][1]{performance}{details}{isGuidSearch});
+
+    $self->assert_deep_equals([$email1Id], $res->[1][1]{ids});
+    $self->assert_deep_equals(['cache'],   $res->[1][1]{performance}{details}{filters});
+    $self->assert_equals(JSON::false, $res->[1][1]{performance}{details}{isGuidSearch});
+
+    $self->assert_deep_equals([$email1Id], $res->[2][1]{ids});
+    $self->assert_deep_equals(['xapian'],  $res->[2][1]{performance}{details}{filters});
+    $self->assert_equals(JSON::true, $res->[2][1]{performance}{details}{isGuidSearch});
+
+    $self->assert_deep_equals([$email2Id], $res->[3][1]{ids});
+    $self->assert_deep_equals(['xapian'],  $res->[3][1]{performance}{details}{filters});
+    $self->assert_equals(JSON::true, $res->[3][1]{performance}{details}{isGuidSearch});
+
+    xlog $self, "Assert 'inReplyTo' and 'references' filters in Sieve";
+
+    $imap->create("matches") or die;
+    $self->{instance}->install_sieve_script(
+        <<'EOF'
+require ["x-cyrus-jmapquery", "x-cyrus-log", "variables", "fileinto"];
+if
+  allof( not string :is "${stop}" "Y",
+    jmapquery text:
+  {
+    "operator" : "OR",
+    "conditions": [{
+      "references": "refC@local"
+    }, {
+      "inReplyTo": "replyToC@local"
+    }]
+  }
+.
+  )
+{
+  fileinto "matches";
+}
+EOF
+    );
+
+    $mime = <<'EOF';
+From: foo@local
+To: bar@local
+Message-Id: <[email protected]>
+References: <refC@local>
+Subject: sievetest1
+Date: Mon, 13 Apr 2020 15:34:03 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+test
+EOF
+    $mime =~ s/\r?\n/\r\n/gs;
+    my $msg = Cassandane::Message->new();
+    $msg->set_lines(split /\n/, $mime);
+    $self->{instance}->deliver($msg);
+    $self->assert_num_equals(1, $imap->message_count('matches'));
+
+    $mime = <<'EOF';
+From: foo@local
+To: bar@local
+Message-Id: <[email protected]>
+In-Reply-To: <replyToC@local>
+Subject: sievetest2
+Date: Mon, 13 Apr 2020 15:34:03 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+test
+EOF
+    $mime =~ s/\r?\n/\r\n/gs;
+    $msg = Cassandane::Message->new();
+    $msg->set_lines(split /\n/, $mime);
+    $self->{instance}->deliver($msg);
+    $self->assert_num_equals(2, $imap->message_count('matches'));
+}
diff --git a/changes/next/xapian_add_messageid b/changes/next/xapian_add_messageid
@@ -0,0 +1,19 @@
+Description:
+
+Adds JMAP Email/query filter conditions `messageId`, `references` and `inReplyTo`.
+
+Config changes:
+
+None.
+
+
+Upgrade instructions:
+
+It is recommended to rebuild the Xapian index to make use of these filter
+conditions. Otherwise, email queries having these filter fall back to
+reading the MIME headers from disk, resulting in slower search.
+
+
+GitHub issue:
+
+None.