From aad1b9e70529b78d3d7d62b0920ad82ca40f9592 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 22 Aug 2020 06:06:26 +0000 Subject: search: support downloading mboxes results with full thread Finally, the addition of THREADID for collapsing results in Xapian lets us emulate the "mairix --threads" feature. That is, instead of returning only the matching messages, the entire thread is included in the downloaded mbox.gz This requires a "public-inbox-index --reindex" to be usable. --- lib/PublicInbox/Mbox.pm | 54 ++++++++++++++++++++++++++++++++++-------- lib/PublicInbox/Over.pm | 29 +++++++++++++++++++++++ lib/PublicInbox/Search.pm | 4 ++++ lib/PublicInbox/SearchQuery.pm | 8 ++++--- lib/PublicInbox/SearchView.pm | 21 +++++++++------- 5 files changed, 95 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index 873ff7be..c9b11c21 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -167,13 +167,13 @@ sub thread_mbox { sub emit_range { my ($ctx, $range) = @_; - my $query; + my $q; if ($range eq 'all') { # TODO: YYYY[-MM] - $query = ''; + $q = ''; } else { return [404, [qw(Content-Type text/plain)], []]; } - mbox_all($ctx, $query); + mbox_all($ctx, { q => $q }); } sub all_ids_cb { @@ -220,21 +220,55 @@ sub results_cb { } } -sub mbox_all { - my ($ctx, $query) = @_; +sub results_thread_cb { + my ($ctx) = @_; - return mbox_all_ids($ctx) if $query eq ''; - my $qopts = $ctx->{qopts} = { mset => 2 }; # order by docid + my $over = $ctx->{-inbox}->over or return; + while (1) { + while (defined(my $num = shift(@{$ctx->{xids}}))) { + my $smsg = $over->get_art($num) or next; + return $smsg; + } + + # refills ctx->{xids} + next if $over->expand_thread($ctx); + + # refill result set + my $srch = $ctx->{-inbox}->search(undef, $ctx) or return; + my $mset = $srch->query($ctx->{query}, $ctx->{qopts}); + my $size = $mset->size or return; + $ctx->{qopts}->{offset} += $size; + $ctx->{ids} = $srch->mset_to_artnums($mset); + } + +} + +sub mbox_all { + my ($ctx, $q) = @_; + my $q_string = $q->{'q'}; + return mbox_all_ids($ctx) if $q_string !~ /\S/; my $srch = $ctx->{-inbox}->search or return PublicInbox::WWW::need($ctx, 'Search'); - my $mset = $srch->query($query, $qopts); + my $over = $ctx->{-inbox}->over or + return PublicInbox::WWW::need($ctx, 'Overview'); + + my $qopts = $ctx->{qopts} = { mset => 2 }; # order by docid + $qopts->{thread} = 1 if $q->{t}; + my $mset = $srch->query($q_string, $qopts); $qopts->{offset} = $mset->size or return [404, [qw(Content-Type text/plain)], ["No results found\n"]]; - $ctx->{query} = $query; + $ctx->{query} = $q_string; $ctx->{ids} = $srch->mset_to_artnums($mset); require PublicInbox::MboxGz; - PublicInbox::MboxGz::mbox_gz($ctx, \&results_cb, 'results-'.$query); + my $fn; + if ($q->{t}) { + $fn = 'results-thread-'.$q_string; + PublicInbox::MboxGz::mbox_gz($ctx, \&results_thread_cb, $fn); + } else { + $fn = 'results-'.$q_string; + PublicInbox::MboxGz::mbox_gz($ctx, \&results_cb, $fn); + } } 1; diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm index 34d0b05d..fba58d17 100644 --- a/lib/PublicInbox/Over.pm +++ b/lib/PublicInbox/Over.pm @@ -179,6 +179,35 @@ ORDER BY $sort_col DESC ($nr, $msgs); } +# strict `tid' matches, only, for thread-expanded mbox.gz search results +# and future CLI interface +# returns true if we have IDs, undef if not +sub expand_thread { + my ($self, $ctx) = @_; + my $dbh = $self->connect; + do { + defined(my $num = $ctx->{ids}->[0]) or return; + my ($tid) = $dbh->selectrow_array(<<'', undef, $num); +SELECT tid FROM over WHERE num = ? + + if (defined($tid)) { + my $sql = <<''; +SELECT num FROM over WHERE tid = ? AND num > ? +ORDER BY num ASC LIMIT 1000 + + my $xids = $dbh->selectcol_arrayref($sql, undef, $tid, + $ctx->{prev} // 0); + if (scalar(@$xids)) { + $ctx->{prev} = $xids->[-1]; + $ctx->{xids} = $xids; + return 1; # success + } + } + $ctx->{prev} = 0; + shift @{$ctx->{ids}}; + } while (1); +} + sub recent { my ($self, $opts, $after, $before) = @_; my ($s, @v); diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index 4cfb7b38..bc820b64 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -326,6 +326,10 @@ sub _enquire_once { # retry_reopen callback } else { $enquire->set_sort_by_value_then_relevance(TS, $desc); } + + # `mairix -t / --threads' or JMAP collapseThreads + $enquire->set_collapse_key(THREADID) if $opts->{thread}; + my $offset = $opts->{offset} || 0; my $limit = $opts->{limit} || 50; my $mset = $enquire->get_mset($offset, $limit); diff --git a/lib/PublicInbox/SearchQuery.pm b/lib/PublicInbox/SearchQuery.pm index ce1eae12..6724ae39 100644 --- a/lib/PublicInbox/SearchQuery.pm +++ b/lib/PublicInbox/SearchQuery.pm @@ -12,7 +12,8 @@ our $LIM = 200; sub new { my ($class, $qp) = @_; - my $r = $qp->{r}; + my $r = $qp->{r}; # relevance + my $t = $qp->{t}; # collapse threads my ($l) = (($qp->{l} || '') =~ /([0-9]+)/); $l = $LIM if !$l || $l > $LIM; bless { @@ -21,6 +22,7 @@ sub new { o => (($qp->{o} || '0') =~ /(-?[0-9]+)/), l => $l, r => (defined $r && $r ne '0'), + t => (defined $t && $t ne '0'), }, $class; } @@ -41,8 +43,8 @@ sub qs_html { if (my $l = $self->{l}) { $qs .= "&l=$l" unless $l == $LIM; } - if (my $r = $self->{r}) { - $qs .= "&r"; + for my $bool (qw(r t)) { + $qs .= "&$bool" if $self->{$bool}; } if (my $x = $self->{x}) { $qs .= "&x=$x" if ($x eq 't' || $x eq 'A' || $x eq 'm'); diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm index 75e2d39d..dd69564a 100644 --- a/lib/PublicInbox/SearchView.pm +++ b/lib/PublicInbox/SearchView.pm @@ -19,10 +19,12 @@ my %rmap_inc; sub mbox_results { my ($ctx) = @_; my $q = PublicInbox::SearchQuery->new($ctx->{qp}); - my $x = $q->{x}; + if ($ctx->{env}->{'psgi.input'}->read(my $buf, 3)) { + $q->{t} = 1 if $buf =~ /\Ax=[^0]/; + } require PublicInbox::Mbox; - return PublicInbox::Mbox::mbox_all($ctx, $q->{'q'}) if $x eq 'm'; - sres_top_html($ctx); + $q->{x} eq 'm' ? PublicInbox::Mbox::mbox_all($ctx, $q) : + sres_top_html($ctx); } sub sres_top_html { @@ -46,6 +48,7 @@ sub sres_top_html { offset => $o, mset => 1, relevance => $q->{r}, + thread => $q->{t}, asc => $asc, }; my ($mset, $total, $err, $html); @@ -151,7 +154,7 @@ sub err_txt { sub search_nav_top { my ($mset, $q, $ctx) = @_; - my $m = $q->qs_html(x => 'm', r => undef); + my $m = $q->qs_html(x => 'm', r => undef, t => undef); my $rv = qq{
};
 	my $initial_q = $ctx->{-uxs_retried};
 	if (defined $initial_q) {
@@ -186,10 +189,12 @@ sub search_nav_top {
 	}
 	my $A = $q->qs_html(x => 'A', r => undef);
 	$rv .= qq{|Atom feed]} .
-		qq{\n\t\t\t\t\t\tdownload: } .
-		# lynx seems to require a name=, here, so just use 'z'
-		qq{} .
-		q{
};
+		qq{\n\t\t\tdownload mbox.gz: } .
+		# we set name=z w/o using it since it seems required for
+		# lynx (but works fine for w3m).
+		qq{|} .
+		qq{} .
+		qq{
};
 }
 
 sub search_nav_bot {
-- 
cgit v1.2.3-24-ge0c7