Merge remote-tracking branch 'origin/search'

* origin/search:
  view: deduplicate common code for loading search results
  SearchMsg: ensure metadata for ghost messages mid
  implement /s/$SUBJECT_PATH.html lookups
  search: remove unnecessary xpfx export
  www: /t/$MESSAGE_ID.html for threads
  view: hoist out index_walk function
  view: reply threading adjustment
  thread: common sorting code
  view: display replies in per-message view
  search: make search results more OO
  extract redundant Message-ID handling code
  search: implement index_sync to fixup indexer
  initial search backend implementation

1 files changed, 239 insertions, 0 deletions

diff --git a/t/search.t b/t/search.t
new file mode 100644
index 00000000..9bdd3cee
--- /dev/null
+++ b/t/search.t
@@ -0,0 +1,239 @@
+# Copyright (C) 2015, all contributors <meta@public-inbox.org>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+use strict;
+use warnings;
+use Test::More;
+use File::Temp qw/tempdir/;
+use PublicInbox::Search;
+use Email::MIME;
+use Data::Dumper;
+my $tmpdir = tempdir(CLEANUP => 1);
+my $git_dir = "$tmpdir/a.git";
+my ($root_id, $last_id);
+
+is(0, system(qw(git init -q --bare), $git_dir), "git init (main)");
+eval { PublicInbox::Search->new($git_dir) };
+ok($@, "exception raised on non-existent DB");
+
+my $rw = PublicInbox::Search->new($git_dir, 1);
+my $ro = PublicInbox::Search->new($git_dir);
+
+{
+        my $root = Email::MIME->create(
+                header_str => [
+                        Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
+                        Subject => 'Hello world',
+                        'Message-ID' => '<root@s>',
+                        From => 'John Smith <js@example.com>',
+                        To => 'list@example.com',
+                ],
+                body => "\\m/\n");
+        my $last = Email::MIME->create(
+                header_str => [
+                        Date => 'Sat, 02 Oct 2010 00:00:00 +0000',
+                        Subject => 'Re: Hello world',
+                        'In-Reply-To' => '<root@s>',
+                        'Message-ID' => '<last@s>',
+                        From => 'John Smith <js@example.com>',
+                        To => 'list@example.com',
+                ],
+                body => "goodbye forever :<\n");
+
+        my $rv;
+        $root_id = $rw->add_message($root);
+        is($root_id, int($root_id), "root_id is an integer: $root_id");
+        $last_id = $rw->add_message($last);
+        is($last_id, int($last_id), "last_id is an integer: $last_id");
+}
+
+sub filter_mids {
+        my ($res) = @_;
+        sort(map { $_->mid } @{$res->{msgs}});
+}
+
+{
+        $ro->reopen;
+        my $found = $ro->lookup_message('<root@s>');
+        ok($found, "message found");
+        is($root_id, $found->{doc_id}, 'doc_id set correctly');
+        $found->ensure_metadata;
+        is($found->mid, 'root@s', 'mid set correctly');
+        ok(int($found->thread_id) > 0, 'thread_id is an integer');
+
+        my @exp = sort qw(root@s last@s);
+        my $res = $ro->query("path:hello_world");
+        my @res = filter_mids($res);
+        is_deeply(\@res, \@exp, 'got expected results for path: match');
+
+        foreach my $p (qw(hello hello_ hello_world2 hello_world_)) {
+                $res = $ro->query("path:$p");
+                is($res->{count}, 0, "path variant `$p' does not match");
+        }
+
+        $res = $ro->query('subject:(Hello world)');
+        @res = filter_mids($res);
+        is_deeply(\@res, \@exp, 'got expected results for subject:() match');
+
+        $res = $ro->query('subject:"Hello world"');
+        @res = filter_mids($res);
+        is_deeply(\@res, \@exp, 'got expected results for subject:"" match');
+
+        $res = $ro->query('subject:"Hello world"', {limit => 1});
+        is(scalar @{$res->{msgs}}, 1, "limit works");
+        my $first = $res->{msgs}->[0];
+
+        $res = $ro->query('subject:"Hello world"', {offset => 1});
+        is(scalar @{$res->{msgs}}, 1, "offset works");
+        my $second = $res->{msgs}->[0];
+
+        isnt($first, $second, "offset returned different result from limit");
+
+        foreach my $f (qw(inreplyto references)) {
+                $res = $ro->query($f . ':root@s');
+                @res = filter_mids($res);
+                is_deeply(\@res, [ 'last@s' ],
+                          "got expected results for $f: match");
+                $res = $ro->query($f . ':root');
+                is($res->{count}, 0, "no partial mid match");
+        }
+}
+
+# ghost vivication
+{
+        $rw->reopen;
+        my $rmid = '<ghost-message@s>';
+        my $reply_to_ghost = Email::MIME->create(
+                header_str => [
+                        Date => 'Sat, 02 Oct 2010 00:00:00 +0000',
+                        Subject => 'Re: ghosts',
+                        'Message-ID' => '<ghost-reply@s>',
+                        'In-Reply-To' => $rmid,
+                        From => 'Time Traveler <tt@example.com>',
+                        To => 'list@example.com',
+                ],
+                body => "-_-\n");
+
+        my $rv;
+        my $reply_id = $rw->add_message($reply_to_ghost);
+        is($reply_id, int($reply_id), "reply_id is an integer: $reply_id");
+
+        my $was_ghost = Email::MIME->create(
+                header_str => [
+                        Date => 'Sat, 02 Oct 2010 00:00:01 +0000',
+                        Subject => 'ghosts',
+                        'Message-ID' => $rmid,
+                        From => 'Laggy Sender <lag@example.com>',
+                        To => 'list@example.com',
+                ],
+                body => "are real\n");
+
+        my $ghost_id = $rw->add_message($was_ghost);
+        is($ghost_id, int($ghost_id), "ghost_id is an integer: $ghost_id");
+        ok($ghost_id < $reply_id, "ghost vivified from earlier message");
+}
+
+# search thread on ghost
+{
+        $ro->reopen;
+
+        # Subject:
+        my $res = $ro->query('ghost');
+        my @exp = sort qw(ghost-message@s ghost-reply@s);
+        my @res = filter_mids($res);
+        is_deeply(\@res, \@exp, 'got expected results for Subject match');
+
+        # body
+        $res = $ro->query('goodbye');
+        is($res->{msgs}->[0]->mid, 'last@s', 'got goodbye message body');
+}
+
+# long message-id
+{
+        $rw->reopen;
+        $ro->reopen;
+        my $long_mid = 'last' . ('x' x 60). '@s';
+        my $long_midc = Digest::SHA::sha1_hex($long_mid);
+
+        my $long = Email::MIME->create(
+                header_str => [
+                        Date => 'Sat, 02 Oct 2010 00:00:00 +0000',
+                        Subject => 'long message ID',
+                        'References' => '<root@s> <last@s>',
+                        'In-Reply-To' => '<last@s>',
+                        'Message-ID' => "<$long_mid>",
+                        From => '"Long I.D." <long-id@example.com>',
+                        To => 'list@example.com',
+                ],
+                body => "wut\n");
+        my $long_id = $rw->add_message($long);
+        is($long_id, int($long_id), "long_id is an integer: $long_id");
+
+        $ro->reopen;
+        my $res = $ro->query('references:root@s');
+        my @res = filter_mids($res);
+        is_deeply(\@res, [ sort('last@s', $long_midc) ],
+                  "got expected results for references: match");
+
+        my $replies = $ro->get_replies('root@s');
+        $replies = [ filter_mids($replies) ];
+        is_deeply($replies, [ filter_mids($res) ], "get_replies matches");
+
+        my $long_reply_mid = 'reply-to-long@1';
+        my $long_reply = Email::MIME->create(
+                header_str => [
+                        Subject => 'I break references',
+                        Date => 'Sat, 02 Oct 2010 00:00:00 +0000',
+                        'Message-ID' => "<$long_reply_mid>",
+                        # No References:
+                        # 'References' => '<root@s> <last@s> <'.$long_mid.'>',
+                        'In-Reply-To' => "<$long_mid>",
+                        From => '"no1 <no1@example.com>',
+                        To => 'list@example.com',
+                ],
+                body => "no References\n");
+        ok($rw->add_message($long_reply) > $long_id, "inserted long reply");
+
+        $ro->reopen;
+        my $t = $ro->get_thread('root@s');
+        is($t->{count}, 4, "got all 4 mesages in thread");
+        my @exp = sort($long_reply_mid, 'root@s', 'last@s', $long_midc);
+        @res = filter_mids($t);
+        is_deeply(\@res, \@exp, "get_thread works");
+}
+
+# quote prioritization
+{
+        $rw->reopen;
+        $rw->add_message(Email::MIME->create(
+                header_str => [
+                        Date => 'Sat, 02 Oct 2010 00:00:01 +0000',
+                        Subject => 'Hello',
+                        'Message-ID' => '<quote@a>',
+                        From => 'Quoter <quoter@example.com>',
+                        To => 'list@example.com',
+                ],
+                body => "> theatre illusions\nfade\n"));
+
+        $rw->add_message(Email::MIME->create(
+                header_str => [
+                        Date => 'Sat, 02 Oct 2010 00:00:02 +0000',
+                        Subject => 'Hello',
+                        'Message-ID' => '<nquote@a>',
+                        From => 'Non-Quoter<non-quoter@example.com>',
+                        To => 'list@example.com',
+                ],
+                body => "theatre\nfade\n"));
+        my $res = $rw->query("theatre");
+        is($res->{count}, 2, "got both matches");
+        is($res->{msgs}->[0]->mid, 'nquote@a', "non-quoted scores higher");
+        is($res->{msgs}->[1]->mid, 'quote@a', "quoted result still returned");
+
+        $res = $rw->query("illusions");
+        is($res->{count}, 1, "got a match for quoted text");
+        is($res->{msgs}->[0]->mid, 'quote@a',
+                "quoted result returned if nothing else");
+}
+
+done_testing();
+
+1;