about summary refs log tree commit homepage
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2021-01-13 19:06:21 -1200
committerEric Wong <e@80x24.org>2021-01-14 23:14:08 +0000
commit97a3b1b79eda65b494e9c01bc99afb4df33a32d5 (patch)
treedb9a2f5880e5cccf2dccd2ec1e495437bc204dec
parent8591e28bd9acfb44bd07d9bf75c055dfe3edf1a7 (diff)
downloadpublic-inbox-97a3b1b79eda65b494e9c01bc99afb4df33a32d5.tar.gz
Now that dedupe is serialization and fork-safe, we can
wire it back up in our query results paths.
-rw-r--r--lib/PublicInbox/LeiQuery.pm5
-rw-r--r--lib/PublicInbox/LeiXSearch.pm8
2 files changed, 8 insertions, 5 deletions
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 1a3e1193..69d2f9a6 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -26,14 +26,13 @@ sub lei_q {
         my $sto = $self->_lei_store(1);
         my $cfg = $self->_lei_cfg(1);
         my $opt = $self->{opt};
-        require PublicInbox::LeiDedupe;
-        my $dd = PublicInbox::LeiDedupe->new($self);
 
         # --local is enabled by default
         # src: LeiXSearch || LeiSearch || Inbox
         my @srcs;
         require PublicInbox::LeiXSearch;
         require PublicInbox::LeiOverview;
+        require PublicInbox::LeiDedupe;
         my $lxs = PublicInbox::LeiXSearch->new;
 
         # --external is enabled by default, but allow --no-external
@@ -49,8 +48,8 @@ sub lei_q {
 
         unshift(@srcs, $sto->search) if $opt->{'local'};
         # no forking workers after this
-        require PublicInbox::LeiOverview;
         $self->{ovv} = PublicInbox::LeiOverview->new($self);
+        $self->{dd} = PublicInbox::LeiDedupe->new($self);
         my %mset_opt = map { $_ => $opt->{$_} } qw(thread limit offset);
         $mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
         $mset_opt{qstr} = join(' ', map {;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 68889e81..80e7a7f7 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -103,6 +103,8 @@ sub query_thread_mset { # for --thread
         my $mo = { %{$lei->{mset_opt}} };
         my $mset;
         my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
+        my $dd = $lei->{dd};
+        $dd->prepare_dedupe;
         do {
                 $mset = $srch->mset($mo->{qstr}, $mo);
                 my $ids = $srch->mset_to_artnums($mset, $mo);
@@ -112,7 +114,7 @@ sub query_thread_mset { # for --thread
                 while ($over->expand_thread($ctx)) {
                         for my $n (@{$ctx->{xids}}) {
                                 my $smsg = $over->get_art($n) or next;
-                                # next if $dd->is_smsg_dup($smsg); TODO
+                                next if $dd->is_smsg_dup($smsg);
                                 my $mitem = delete $n2item{$smsg->{num}};
                                 $each_smsg->($smsg, $mitem);
                                 # $self->out($buf .= $ORS);
@@ -132,11 +134,13 @@ sub query_mset { # non-parallel for non-"--thread" users
         my $mset;
         $self->attach_external($_) for @$srcs;
         my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
+        my $dd = $lei->{dd};
+        $dd->prepare_dedupe;
         do {
                 $mset = $self->mset($mo->{qstr}, $mo);
                 for my $it ($mset->items) {
                         my $smsg = smsg_for($self, $it) or next;
-                        # next if $dd->is_smsg_dup($smsg);
+                        next if $dd->is_smsg_dup($smsg);
                         $each_smsg->($smsg, $it);
                         # $self->out($buf .= $ORS) if defined $buf;
                         #$emit_cb->($smsg);