From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id ED9091FFB1 for ; Tue, 2 Feb 2021 11:47:02 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 04/16] lei q: support --only, --include and --exclude Date: Tue, 2 Feb 2021 11:46:50 +0000 Message-Id: <20210202114702.29886-5-e@80x24.org> In-Reply-To: <20210202114702.29886-1-e@80x24.org> References: <20210202114702.29886-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: -I is short for --include since it's standard for C compilers (along with Perl and Ruby). There are no single-character shortcuts for --exclude or --only, since I don't expect --exclude to be used very often and --only is already short (and will support shell completion). --- lib/PublicInbox/LEI.pm | 1 + lib/PublicInbox/LeiExternal.pm | 12 +++++----- lib/PublicInbox/LeiQuery.pm | 42 ++++++++++++++++++++++++---------- t/lei_external.t | 2 +- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 6c2515dc..ffbc2503 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -104,6 +104,7 @@ our %CMD = ( # sorted in order of importance/use: 'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw( save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a sort|s=s reverse|r offset=i remote! local! external! pretty + include|I=s@ exclude=s@ only=s@ mua-cmd|mua=s no-torsocks torsocks=s verbose|v quiet|q received-after=s received-before=s sent-after=s sent-since=s), PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ], diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm index b1176824..3853cfc1 100644 --- a/lib/PublicInbox/LeiExternal.pm +++ b/lib/PublicInbox/LeiExternal.pm @@ -9,7 +9,7 @@ use parent qw(Exporter); our @EXPORT = qw(lei_ls_external lei_add_external lei_forget_external); use PublicInbox::Config; -sub _externals_each { +sub externals_each { my ($self, $cb, @arg) = @_; my $cfg = $self->_lei_cfg(0); my %boost; @@ -32,14 +32,14 @@ sub _externals_each { sub lei_ls_external { my ($self, @argv) = @_; my ($OFS, $ORS) = $self->{opt}->{z} ? ("\0", "\0\0") : (" ", "\n"); - $self->_externals_each(sub { + externals_each($self, sub { my ($loc, $boost_val) = @_; $self->out($loc, $OFS, 'boost=', $boost_val, $ORS); }); } -sub _canonicalize { - my ($location) = @_; +sub ext_canonicalize { + my ($location) = $_[-1]; if ($location !~ m!\Ahttps?://!) { PublicInbox::Config::rel2abs_collapsed($location); } else { @@ -56,7 +56,7 @@ sub lei_add_external { my ($self, $location) = @_; my $cfg = $self->_lei_cfg(1); my $new_boost = $self->{opt}->{boost} // 0; - $location = _canonicalize($location); + $location = ext_canonicalize($location); if ($location !~ m!\Ahttps?://! && !-d $location) { return $self->fail("$location not a directory"); } @@ -74,7 +74,7 @@ sub lei_forget_external { my %seen; for my $loc (@locations) { my (@unset, @not_found); - for my $l ($loc, _canonicalize($loc)) { + for my $l ($loc, ext_canonicalize($loc)) { next if $seen{$l}++; my $key = "external.$l.boost"; delete($cfg->{$key}); diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index dea04c13..fd8a3bca 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -7,6 +7,11 @@ use strict; use v5.10.1; use PublicInbox::DS qw(dwaitpid); +sub prep_ext { # externals_each callback + my ($lxs, $exclude, $loc) = @_; + $lxs->prepare_external($loc) unless $exclude->{$loc}; +} + # the main "lei q SEARCH_TERMS" method sub lei_q { my ($self, @argv) = @_; @@ -14,22 +19,35 @@ sub lei_q { require PublicInbox::LeiOverview; PublicInbox::Config->json; # preload before forking my $opt = $self->{opt}; + # prepare any number of LeiXSearch || LeiSearch || Inbox || URL my $lxs = $self->{lxs} = PublicInbox::LeiXSearch->new; - # any number of LeiXSearch || LeiSearch || Inbox - if ($opt->{'local'} //= 1) { # --local is enabled by default + my @only = @{$opt->{only} // []}; + # --local is enabled by default unless --only is used + # we'll allow "--only $LOCATION --local" + if ($opt->{'local'} //= scalar(@only) ? 0 : 1) { my $sto = $self->_lei_store(1); $lxs->prepare_external($sto->search); } - - # --external is enabled by default, but allow --no-external - if ($opt->{external} //= 1) { - my $cb = $lxs->can('prepare_external'); - my $ne = $self->_externals_each($cb, $lxs); - $opt->{remote} //= $ne == $lxs->remotes; - if ($opt->{'local'}) { - delete($lxs->{remotes}) if !$opt->{remote}; - } else { - delete($lxs->{locals}); + if (@only) { + for my $loc (@only) { + $lxs->prepare_external($self->ext_canonicalize($loc)); + } + } else { + for my $loc (@{$opt->{include} // []}) { + $lxs->prepare_external($self->ext_canonicalize($loc)); + } + # --external is enabled by default, but allow --no-external + if ($opt->{external} //= 1) { + my %x = map {; + ($self->ext_canonicalize($_), 1) + } @{$self->{exclude} // []}; + my $ne = $self->externals_each(\&prep_ext, $lxs, \%x); + $opt->{remote} //= !($lxs->locals - $opt->{'local'}); + if ($opt->{'local'}) { + delete($lxs->{remotes}) if !$opt->{remote}; + } else { + delete($lxs->{locals}); + } } } unless ($lxs->locals || $lxs->remotes) { diff --git a/t/lei_external.t b/t/lei_external.t index 1f0048a1..587990db 100644 --- a/t/lei_external.t +++ b/t/lei_external.t @@ -4,7 +4,7 @@ use v5.10.1; use Test::More; my $cls = 'PublicInbox::LeiExternal'; require_ok $cls; -my $canon = $cls->can('_canonicalize'); +my $canon = $cls->can('ext_canonicalize'); my $exp = 'https://example.com/my-inbox/'; is($canon->('https://example.com/my-inbox'), $exp, 'trailing slash added'); is($canon->('https://example.com/my-inbox//'), $exp, 'trailing slash removed');