user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 7/9] lei q: fix JSON overview with remote externals
Date: Sun, 24 Jan 2021 04:46:53 -0700	[thread overview]
Message-ID: <20210124114655.12815-8-e@80x24.org> (raw)
In-Reply-To: <20210124114655.12815-1-e@80x24.org>

We can't (and don't need to) repeatedly get the $each_smsg
callback for each URI since that clobbers {ovv_buf} before
it can be output.

I initially thought this was a dedupe-related bug and
moved the dedupe code into the $each_smsg callback to
minimize differences.  Nevertheless it's a nice code
reduction.

I also thought it was related to incomplete smsg info,
so {references} is now filled in correctly for dedupe.
---
 lib/PublicInbox/LeiOverview.pm |  2 +-
 lib/PublicInbox/LeiXSearch.pm  | 15 +++++----------
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 49538a60..928d66cb 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -209,7 +209,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
 		$json->ascii(1) if $lei->{opt}->{ascii};
 	}
 	my $l2m = $lei->{l2m};
-	if ($l2m && $ibxish->can('scheme')) { # remote https?:// mboxrd
+	if ($l2m && !$ibxish) { # remote https?:// mboxrd
 		delete $l2m->{-wq_s1};
 		my $g2m = $l2m->can('git_to_mail');
 		my $wcb = $l2m->write_cb($lei);
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index c6ff5679..2bedf000 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -15,6 +15,7 @@ use File::Temp 0.19 (); # 0.19 for ->newdir
 use File::Spec ();
 use PublicInbox::Search qw(xap_terms);
 use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::MID qw(mids);
 
 sub new {
 	my ($class) = @_;
@@ -120,8 +121,6 @@ sub query_thread_mset { # for --thread
 	my $mo = { %{$lei->{mset_opt}} };
 	my $mset;
 	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $ibxish);
-	my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
-	$dedupe->prepare_dedupe;
 	do {
 		$mset = $srch->mset($mo->{qstr}, $mo);
 		my $ids = $srch->mset_to_artnums($mset, $mo);
@@ -132,7 +131,6 @@ sub query_thread_mset { # for --thread
 			for my $n (@{$ctx->{xids}}) {
 				my $smsg = $over->get_art($n) or next;
 				wait_startq($startq) if $startq;
-				next if $dedupe->is_smsg_dup($smsg);
 				my $mitem = delete $n2item{$smsg->{num}};
 				$each_smsg->($smsg, $mitem);
 			}
@@ -155,14 +153,11 @@ sub query_mset { # non-parallel for non-"--thread" users
 		attach_external($self, $loc);
 	}
 	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $self);
-	my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
-	$dedupe->prepare_dedupe;
 	do {
 		$mset = $self->mset($mo->{qstr}, $mo);
 		for my $mitem ($mset->items) {
 			my $smsg = smsg_for($self, $mitem) or next;
 			wait_startq($startq) if $startq;
-			next if $dedupe->is_smsg_dup($smsg);
 			$each_smsg->($smsg, $mitem);
 		}
 	} while (_mset_more($mset, $mo));
@@ -174,10 +169,10 @@ sub each_eml { # callback for MboxReader->mboxrd
 	my ($eml, $self, $lei, $each_smsg) = @_;
 	my $smsg = bless {}, 'PublicInbox::Smsg';
 	$smsg->populate($eml);
+	PublicInbox::OverIdx::parse_references($smsg, $eml, mids($eml));
 	$smsg->{$_} //= '' for qw(from to cc ds subject references mid);
 	delete @$smsg{qw(From Subject -ds -ts)};
 	if (my $startq = delete($self->{5})) { wait_startq($startq) }
-	return if !$lei->{l2m} && $lei->{dedupe}->is_smsg_dup($smsg);
 	$each_smsg->($smsg, undef, $eml);
 }
 
@@ -189,8 +184,6 @@ sub query_remote_mboxrd {
 	my ($opt, $env) = @$lei{qw(opt env)};
 	my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
 	push(@qform, t => 1) if $opt->{thread};
-	my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
-	$dedupe->prepare_dedupe;
 	my @cmd = (qw(curl -sSf -d), '');
 	my $verbose = $opt->{verbose};
 	push @cmd, '-v' if $verbose;
@@ -208,9 +201,9 @@ sub query_remote_mboxrd {
 	}
 	$opt->{torsocks} = 'false' if $opt->{'no-torsocks'};
 	my $tor = $opt->{torsocks} //= 'auto';
+	my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $uris->[0]);
 	for my $uri (@$uris) {
 		$uri->query_form(@qform);
-		my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $uri);
 		my $cmd = [ @cmd, $uri->as_string ];
 		if ($tor eq 'auto' && substr($uri->host, -6) eq '.onion' &&
 				(($env->{LD_PRELOAD}//'') !~ /torsocks/)) {
@@ -236,6 +229,7 @@ sub query_remote_mboxrd {
 			$lei->child_error($?);
 		}
 	}
+	undef $each_smsg;
 	$lei->{ovv}->ovv_atexit_child($lei);
 }
 
@@ -387,6 +381,7 @@ sub ipc_atfork_prepare {
 	my ($self) = @_;
 	if (exists $self->{remotes}) {
 		require PublicInbox::MboxReader;
+		require PublicInbox::OverIdx; # parse_references
 		require IO::Uncompress::Gunzip;
 	}
 	# FDS: (0: done_wr, 1: stdout|mbox, 2: stderr,

  parent reply	other threads:[~2021-01-24 11:46 UTC|newest]

Thread overview: 12+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-01-24 11:46 [PATCH 0/9] lei remotes fixes and updates Eric Wong
2021-01-24 11:46 ` [PATCH 1/9] lei q: limit concurrency to 4 remote connections Eric Wong
2021-01-24 11:46 ` [PATCH 2/9] ipc: wq supports arbitrarily large payloads Eric Wong
2021-01-24 11:46 ` [PATCH 3/9] ipc: get rid of wq_set_recv_modes Eric Wong
2021-01-24 11:46 ` [PATCH 4/9] lei q: disable remote externals if locals exist Eric Wong
2021-01-24 11:46 ` [PATCH 5/9] lei q: honor --no-local to force remote searches Eric Wong
2021-01-24 12:31   ` exit codes [was: [PATCH 5/9] lei q: honor --no-local to force remote searches] Eric Wong
2021-01-24 11:46 ` [PATCH 6/9] lei_xsearch: use curl -d '' for nginx compatibility Eric Wong
2021-01-24 11:46 ` Eric Wong [this message]
2021-01-24 12:37   ` [PATCH 7/9] lei q: fix JSON overview with remote externals Eric Wong
2021-01-24 11:46 ` [PATCH 8/9] smsg: make parse_references an object method Eric Wong
2021-01-24 11:46 ` [PATCH 9/9] smsg: parse_references: micro-optimization Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210124114655.12815-8-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 7/9] lei q: fix JSON overview with remote externals' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).