Merge remote-tracking branch 'origin/master' into repobrowse

* origin/master: (25 commits)
  evcleanup: ensure deferred close from timers are handled ASAP
  httpd/async: improve variable naming
  githttpbackend: minor cleanups to improve readability
  githttpbackend: simplify compatibility code
  githttpbackend: minor readability improvement
  http: fix clobbering of $null_io
  linkify: modify argument in place
  view: do not modify array during iteration
  view: stop chomping off whitespace at ends of messages
  view: remove unused parameter
  search: lookup_mail handles modified DBs
  doc: various comments on async handling
  searchthread: simplify API and remove needless OO
  searchthread: update comment about loop prevention
  searchmsg: remove ensure_metadata
  tests: add thread-all testing for benchmarking
  searchmsg: do not memoize {date} field
  searchmsg: remove locale-dependency for ->date
  t/config.t: fix feedmax default
  wwwtext: link to RFC4685 (Atom Threading)
  ...

5 files changed, 48 insertions, 16 deletions

diff --git a/t/config.t b/t/config.t
index 073d1d03..7271351b 100644
--- a/t/config.t
+++ b/t/config.t
@@ -30,6 +30,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
                 'url' => 'http://example.com/meta',
                 -primary_address => 'meta@public-inbox.org',
                 'name' => 'meta',
+                feedmax => 25,
                 -pi_config => $cfg,
         }, "lookup matches expected output");
 
@@ -45,6 +46,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
                 'mainrepo' => '/home/pi/test-main.git',
                 'domain' => 'public-inbox.org',
                 'name' => 'test',
+                feedmax => 25,
                 'url' => 'http://example.com/test',
                 -pi_config => $cfg,
         }, "lookup matches expected output for test");
diff --git a/t/feed.t b/t/feed.t
index 19a9ba09..b60273ed 100644
--- a/t/feed.t
+++ b/t/feed.t
@@ -46,6 +46,7 @@ my $ibx = PublicInbox::Inbox->new({
         name => 'testbox',
         mainrepo => $git_dir,
         url => 'http://example.com/test',
+        feedmax => 3,
 });
 my $git = $ibx->git;
 my $im = PublicInbox::Import->new($git, $ibx->{name}, 'test@example');
@@ -101,10 +102,7 @@ EOF
 {
         # check initial feed
         {
-                my $feed = string_feed({
-                        -inbox => $ibx,
-                        max => 3
-                });
+                my $feed = string_feed({ -inbox => $ibx });
                 SKIP: {
                         skip 'XML::Feed missing', 2 unless $have_xml_feed;
                         my $p = XML::Feed->parse(\$feed);
@@ -142,10 +140,7 @@ EOF
 
         # check spam shows up
         {
-                my $spammy_feed = string_feed({
-                        -inbox => $ibx,
-                        max => 3
-                });
+                my $spammy_feed = string_feed({ -inbox => $ibx });
                 SKIP: {
                         skip 'XML::Feed missing', 2 unless $have_xml_feed;
                         my $p = XML::Feed->parse(\$spammy_feed);
@@ -167,7 +162,7 @@ EOF
 
         # spam no longer shows up
         {
-                my $feed = string_feed({ -inbox => $ibx, max => 3 });
+                my $feed = string_feed({ -inbox => $ibx });
                 SKIP: {
                         skip 'XML::Feed missing', 2 unless $have_xml_feed;
                         my $p = XML::Feed->parse(\$feed);
diff --git a/t/search.t b/t/search.t
index eed9c9b6..c16811d8 100644
--- a/t/search.t
+++ b/t/search.t
@@ -109,7 +109,6 @@ sub filter_mids {
         my $found = $ro->lookup_message('<root@s>');
         ok($found, "message found");
         is($root_id, $found->{doc_id}, 'doc_id set correctly');
-        $found->ensure_metadata;
         is($found->mid, 'root@s', 'mid set correctly');
         ok(int($found->thread_id) > 0, 'thread_id is an integer');
 
@@ -290,7 +289,6 @@ sub filter_mids {
                 body => "LOOP!\n"));
         ok($doc_id > 0, "doc_id defined with circular reference");
         my $smsg = $rw->lookup_message('circle@a');
-        $smsg->ensure_metadata;
         is($smsg->references, '', "no references created");
         my $msg = PublicInbox::SearchMsg->load_doc($smsg->{doc});
         is($s, $msg->subject, 'long subject not rewritten');
diff --git a/t/thread-all.t b/t/thread-all.t
new file mode 100644
index 00000000..8ccf4f8c
--- /dev/null
+++ b/t/thread-all.t
@@ -0,0 +1,38 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# real-world testing of search threading
+use strict;
+use warnings;
+use Test::More;
+use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
+my $pi_dir = $ENV{GIANT_PI_DIR};
+plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir;
+eval { require PublicInbox::Search; };
+plan skip_all => "Xapian missing for $0" if $@;
+my $srch = eval { PublicInbox::Search->new($pi_dir) };
+plan skip_all => "$pi_dir not initialized for $0" if $@;
+
+require PublicInbox::View;
+require PublicInbox::SearchThread;
+
+my $pfx = PublicInbox::Search::xpfx('thread');
+my $opts = { limit => 1000000, asc => 1 };
+my $t0 = clock_gettime(CLOCK_MONOTONIC);
+my $elapsed;
+
+my $sres = $srch->_do_enquire(undef, $opts);
+$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0;
+diag "enquire: $elapsed";
+
+$t0 = clock_gettime(CLOCK_MONOTONIC);
+my $msgs = PublicInbox::View::load_results($srch, $sres);
+$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0;
+diag "load_results $elapsed";
+
+$t0 = clock_gettime(CLOCK_MONOTONIC);
+PublicInbox::View::thread_results($msgs);
+$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0;
+diag "thread_results $elapsed";
+
+done_testing();
diff --git a/t/thread-cycle.t b/t/thread-cycle.t
index b0844490..16ceee71 100644
--- a/t/thread-cycle.t
+++ b/t/thread-cycle.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use Test::More;
-use_ok('PublicInbox::SearchMsg');
 use_ok('PublicInbox::SearchThread');
 use Email::Simple;
 my $mt = eval {
@@ -73,11 +72,11 @@ SKIP: {
 done_testing();
 
 sub thread_to_s {
-        my $th = PublicInbox::SearchThread->new(shift);
-        $th->thread;
-        $th->order(sub { [ sort { $a->{id} cmp $b->{id} } @{$_[0]} ] });
+        my ($msgs) = @_;
+        my $rootset = PublicInbox::SearchThread::thread($msgs, sub {
+                [ sort { $a->{id} cmp $b->{id} } @{$_[0]} ] });
         my $st = '';
-        my @q = map { (0, $_) } @{$th->{rootset}};
+        my @q = map { (0, $_) } @$rootset;
         while (@q) {
                 my $level = shift @q;
                 my $node = shift @q or next;