* [PATCH 0/9] lei remotes fixes and updates
@ 2021-01-24 11:46 6% Eric Wong
2021-01-24 11:46 5% ` [PATCH 7/9] lei q: fix JSON overview with remote externals Eric Wong
0 siblings, 1 reply; 4+ results
From: Eric Wong @ 2021-01-24 11:46 UTC (permalink / raw)
To: meta
Eric Wong (9):
lei q: limit concurrency to 4 remote connections
ipc: wq supports arbitrarily large payloads
ipc: get rid of wq_set_recv_modes
lei q: disable remote externals if locals exist
lei q: honor --no-local to force remote searches
lei_xsearch: use curl -d '' for nginx compatibility
lei q: fix JSON overview with remote externals
smsg: make parse_references an object method
smsg: parse_references: micro-optimization to avoid ++
lib/PublicInbox/IPC.pm | 85 +++++++++++++++++----------
lib/PublicInbox/LEI.pm | 9 ++-
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 13 ++++-
lib/PublicInbox/LeiToMail.pm | 7 +--
lib/PublicInbox/LeiXSearch.pm | 101 ++++++++++++++++++---------------
lib/PublicInbox/OverIdx.pm | 22 +------
lib/PublicInbox/SearchIdx.pm | 2 +-
lib/PublicInbox/Smsg.pm | 22 ++++++-
script/lei | 11 ++--
t/cmd_ipc.t | 16 ++++++
t/ipc.t | 21 ++++++-
t/lei.t | 3 +
13 files changed, 196 insertions(+), 118 deletions(-)
^ permalink raw reply [relevance 6%]
* [PATCH 7/9] lei q: fix JSON overview with remote externals
2021-01-24 11:46 6% [PATCH 0/9] lei remotes fixes and updates Eric Wong
@ 2021-01-24 11:46 5% ` Eric Wong
2021-01-24 12:37 7% ` Eric Wong
0 siblings, 1 reply; 4+ results
From: Eric Wong @ 2021-01-24 11:46 UTC (permalink / raw)
To: meta
We can't (and don't need to) repeatedly get the $each_smsg
callback for each URI since that clobbers {ovv_buf} before
it can be output.
I initially thought this was a dedupe-related bug and
moved the dedupe code into the $each_smsg callback to
minimize differences. Nevertheless it's a nice code
reduction.
I also thought it was related to incomplete smsg info,
so {references} is now filled in correctly for dedupe.
---
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 15 +++++----------
2 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 49538a60..928d66cb 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -209,7 +209,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
$json->ascii(1) if $lei->{opt}->{ascii};
}
my $l2m = $lei->{l2m};
- if ($l2m && $ibxish->can('scheme')) { # remote https?:// mboxrd
+ if ($l2m && !$ibxish) { # remote https?:// mboxrd
delete $l2m->{-wq_s1};
my $g2m = $l2m->can('git_to_mail');
my $wcb = $l2m->write_cb($lei);
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index c6ff5679..2bedf000 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -15,6 +15,7 @@ use File::Temp 0.19 (); # 0.19 for ->newdir
use File::Spec ();
use PublicInbox::Search qw(xap_terms);
use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::MID qw(mids);
sub new {
my ($class) = @_;
@@ -120,8 +121,6 @@ sub query_thread_mset { # for --thread
my $mo = { %{$lei->{mset_opt}} };
my $mset;
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $ibxish);
- my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
- $dedupe->prepare_dedupe;
do {
$mset = $srch->mset($mo->{qstr}, $mo);
my $ids = $srch->mset_to_artnums($mset, $mo);
@@ -132,7 +131,6 @@ sub query_thread_mset { # for --thread
for my $n (@{$ctx->{xids}}) {
my $smsg = $over->get_art($n) or next;
wait_startq($startq) if $startq;
- next if $dedupe->is_smsg_dup($smsg);
my $mitem = delete $n2item{$smsg->{num}};
$each_smsg->($smsg, $mitem);
}
@@ -155,14 +153,11 @@ sub query_mset { # non-parallel for non-"--thread" users
attach_external($self, $loc);
}
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $self);
- my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
- $dedupe->prepare_dedupe;
do {
$mset = $self->mset($mo->{qstr}, $mo);
for my $mitem ($mset->items) {
my $smsg = smsg_for($self, $mitem) or next;
wait_startq($startq) if $startq;
- next if $dedupe->is_smsg_dup($smsg);
$each_smsg->($smsg, $mitem);
}
} while (_mset_more($mset, $mo));
@@ -174,10 +169,10 @@ sub each_eml { # callback for MboxReader->mboxrd
my ($eml, $self, $lei, $each_smsg) = @_;
my $smsg = bless {}, 'PublicInbox::Smsg';
$smsg->populate($eml);
+ PublicInbox::OverIdx::parse_references($smsg, $eml, mids($eml));
$smsg->{$_} //= '' for qw(from to cc ds subject references mid);
delete @$smsg{qw(From Subject -ds -ts)};
if (my $startq = delete($self->{5})) { wait_startq($startq) }
- return if !$lei->{l2m} && $lei->{dedupe}->is_smsg_dup($smsg);
$each_smsg->($smsg, undef, $eml);
}
@@ -189,8 +184,6 @@ sub query_remote_mboxrd {
my ($opt, $env) = @$lei{qw(opt env)};
my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
push(@qform, t => 1) if $opt->{thread};
- my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
- $dedupe->prepare_dedupe;
my @cmd = (qw(curl -sSf -d), '');
my $verbose = $opt->{verbose};
push @cmd, '-v' if $verbose;
@@ -208,9 +201,9 @@ sub query_remote_mboxrd {
}
$opt->{torsocks} = 'false' if $opt->{'no-torsocks'};
my $tor = $opt->{torsocks} //= 'auto';
+ my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $uris->[0]);
for my $uri (@$uris) {
$uri->query_form(@qform);
- my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $uri);
my $cmd = [ @cmd, $uri->as_string ];
if ($tor eq 'auto' && substr($uri->host, -6) eq '.onion' &&
(($env->{LD_PRELOAD}//'') !~ /torsocks/)) {
@@ -236,6 +229,7 @@ sub query_remote_mboxrd {
$lei->child_error($?);
}
}
+ undef $each_smsg;
$lei->{ovv}->ovv_atexit_child($lei);
}
@@ -387,6 +381,7 @@ sub ipc_atfork_prepare {
my ($self) = @_;
if (exists $self->{remotes}) {
require PublicInbox::MboxReader;
+ require PublicInbox::OverIdx; # parse_references
require IO::Uncompress::Gunzip;
}
# FDS: (0: done_wr, 1: stdout|mbox, 2: stderr,
^ permalink raw reply related [relevance 5%]
* Re: [PATCH 7/9] lei q: fix JSON overview with remote externals
2021-01-24 11:46 5% ` [PATCH 7/9] lei q: fix JSON overview with remote externals Eric Wong
@ 2021-01-24 12:37 7% ` Eric Wong
0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-01-24 12:37 UTC (permalink / raw)
To: meta
Eric Wong <e@80x24.org> wrote:
> --- a/lib/PublicInbox/LeiOverview.pm
> +++ b/lib/PublicInbox/LeiOverview.pm
> @@ -209,7 +209,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
> $json->ascii(1) if $lei->{opt}->{ascii};
> }
> my $l2m = $lei->{l2m};
> - if ($l2m && $ibxish->can('scheme')) { # remote https?:// mboxrd
> + if ($l2m && !$ibxish) { # remote https?:// mboxrd
I made this change last, thus necessitating changes to callers:
> --- a/lib/PublicInbox/LeiXSearch.pm
> +++ b/lib/PublicInbox/LeiXSearch.pm
> @@ -208,9 +201,9 @@ sub query_remote_mboxrd {
> }
> $opt->{torsocks} = 'false' if $opt->{'no-torsocks'};
> my $tor = $opt->{torsocks} //= 'auto';
> + my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $uris->[0]);
Will squash this in, otherwise --remote --no-local won't work:
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 841257c1..fb608d00 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -201,7 +201,7 @@ sub query_remote_mboxrd {
}
$opt->{torsocks} = 'false' if $opt->{'no-torsocks'};
my $tor = $opt->{torsocks} //= 'auto';
- my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $uris->[0]);
+ my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
for my $uri (@$uris) {
$uri->query_form(@qform);
my $cmd = [ @cmd, $uri->as_string ];
^ permalink raw reply related [relevance 7%]
* [PATCH 1/5] lei: reinstate JSON smsg output deduplication
@ 2021-01-25 1:18 4% ` Eric Wong
0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-01-25 1:18 UTC (permalink / raw)
To: meta
This was accidentally clobbered completely in
("lei q: fix JSON overview with remote externals").
There are now more tests to prevent future regressions.
---
lib/PublicInbox/LeiOverview.pm | 7 ++++++-
t/lei.t | 19 ++++++++++++++++---
2 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 928d66cb..880c7acc 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -203,12 +203,14 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
my ($self, $lei, $ibxish) = @_;
my $json;
$lei->{1}->autoflush(1);
+ my $dedupe = $lei->{dedupe} // die 'BUG: {dedupe} missing';
if (my $pkg = $self->{json}) {
$json = $pkg->new;
$json->utf8->canonical;
$json->ascii(1) if $lei->{opt}->{ascii};
+ $lei->{ovv_buf} = \(my $buf = '');
}
- my $l2m = $lei->{l2m};
+ my $l2m = $lei->{l2m} or $dedupe->prepare_dedupe;
if ($l2m && !$ibxish) { # remote https?:// mboxrd
delete $l2m->{-wq_s1};
my $g2m = $l2m->can('git_to_mail');
@@ -241,6 +243,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
my $git = $ibxish->git; # (LeiXSearch|Inbox|ExtSearch)->git
$self->{git} = $git; # for ovv_atexit_child
my $g2m = $l2m->can('git_to_mail');
+ $dedupe->prepare_dedupe;
sub {
my ($smsg, $mitem) = @_;
$smsg->{pct} = get_pct($mitem) if $mitem;
@@ -251,6 +254,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
$lei->{ovv_buf} = \(my $buf = '');
sub { # DIY prettiness :P
my ($smsg, $mitem) = @_;
+ return if $dedupe->is_smsg_dup($smsg);
$smsg = _unbless_smsg($smsg, $mitem);
$buf .= "{\n";
$buf .= join(",\n", map {
@@ -274,6 +278,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
$lei->{ovv_buf} = \(my $buf = '');
sub {
my ($smsg, $mitem) = @_;
+ return if $dedupe->is_smsg_dup($smsg);
$buf .= $json->encode(_unbless_smsg(@_)) . $ORS;
if (length($buf) > 65536) {
my $lk = $self->lock_for_scope;
diff --git a/t/lei.t b/t/lei.t
index 3fd1d1fe..f826a966 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -17,6 +17,7 @@ my $err_filter;
my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
http://czquwvybam4bgbro.onion/meta/
http://ou63pmih66umazou.onion/meta/);
+my $json = ref(PublicInbox::Config->json)->new->utf8->canonical;
my $lei = sub {
my ($cmd, $env, $xopt) = @_;
$out = $err = '';
@@ -142,8 +143,7 @@ my $setup_publicinboxes = sub {
my ($ibx) = @_;
my $im = PublicInbox::InboxWritable->new($ibx)->importer(0);
my $V = $ibx->version;
- my @eml = glob('t/*.eml');
- push(@eml, 't/data/0001.patch') if $V == 2;
+ my @eml = (glob('t/*.eml'), 't/data/0001.patch');
for (@eml) {
next if $_ eq 't/psgi_v2-old.eml'; # dup mid
$im->add(eml_load($_)) or BAIL_OUT "v$V add $_";
@@ -176,7 +176,7 @@ SKIP: {
my $mid = '20140421094015.GA8962@dcvr.yhbt.net';
ok($lei->('q', "m:$mid"), "query $url");
is($err, '', "no errors on $url");
- my $res = PublicInbox::Config->json->decode($out);
+ my $res = $json->decode($out);
is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url");
ok($lei->('q', "m:$mid", 'd:..20101002'), 'no results, no error');
like($err, qr/404/, 'noted 404');
@@ -246,6 +246,19 @@ my $test_external = sub {
# No double-quoting should be imposed on users on the CLI
$lei->('q', 's:use boolean prefix');
like($out, qr/search: use boolean prefix/, 'phrase search got result');
+ my $res = $json->decode($out);
+ is(scalar(@$res), 2, 'only 2 element array (1 result)');
+ is($res->[1], undef, 'final element is undef'); # XXX should this be?
+ is(ref($res->[0]), 'HASH', 'first element is hashref');
+ $lei->('q', '--pretty', 's:use boolean prefix');
+ my $pretty = $json->decode($out);
+ is_deeply($res, $pretty, '--pretty is identical after decode');
+
+ for my $fmt (qw(ldjson ndjson jsonl)) {
+ $lei->('q', '-f', $fmt, 's:use boolean prefix');
+ is($out, $json->encode($pretty->[0])."\n", "-f $fmt");
+ }
+
require IO::Uncompress::Gunzip;
for my $sfx ('', '.gz') {
my $f = "$home/mbox$sfx";
^ permalink raw reply related [relevance 4%]
Results 1-4 of 4 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-01-24 11:46 6% [PATCH 0/9] lei remotes fixes and updates Eric Wong
2021-01-24 11:46 5% ` [PATCH 7/9] lei q: fix JSON overview with remote externals Eric Wong
2021-01-24 12:37 7% ` Eric Wong
2021-01-25 1:18 [PATCH 0/5] lei: more fixes and usability enhancement Eric Wong
2021-01-25 1:18 4% ` [PATCH 1/5] lei: reinstate JSON smsg output deduplication Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).