user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 04/16] lei q: support --only, --include and --exclude
Date: Tue,  2 Feb 2021 11:46:50 +0000	[thread overview]
Message-ID: <20210202114702.29886-5-e@80x24.org> (raw)
In-Reply-To: <20210202114702.29886-1-e@80x24.org>

-I is short for --include since it's standard for C compilers
(along with Perl and Ruby).  There are no single-character
shortcuts for --exclude or --only, since I don't expect
--exclude to be used very often and --only is already short (and
will support shell completion).
---
 lib/PublicInbox/LEI.pm         |  1 +
 lib/PublicInbox/LeiExternal.pm | 12 +++++-----
 lib/PublicInbox/LeiQuery.pm    | 42 ++++++++++++++++++++++++----------
 t/lei_external.t               |  2 +-
 4 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 6c2515dc..ffbc2503 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -104,6 +104,7 @@ our %CMD = ( # sorted in order of importance/use:
 'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw(
 	save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
 	sort|s=s reverse|r offset=i remote! local! external! pretty
+	include|I=s@ exclude=s@ only=s@
 	mua-cmd|mua=s no-torsocks torsocks=s verbose|v quiet|q
 	received-after=s received-before=s sent-after=s sent-since=s),
 	PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index b1176824..3853cfc1 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -9,7 +9,7 @@ use parent qw(Exporter);
 our @EXPORT = qw(lei_ls_external lei_add_external lei_forget_external);
 use PublicInbox::Config;
 
-sub _externals_each {
+sub externals_each {
 	my ($self, $cb, @arg) = @_;
 	my $cfg = $self->_lei_cfg(0);
 	my %boost;
@@ -32,14 +32,14 @@ sub _externals_each {
 sub lei_ls_external {
 	my ($self, @argv) = @_;
 	my ($OFS, $ORS) = $self->{opt}->{z} ? ("\0", "\0\0") : (" ", "\n");
-	$self->_externals_each(sub {
+	externals_each($self, sub {
 		my ($loc, $boost_val) = @_;
 		$self->out($loc, $OFS, 'boost=', $boost_val, $ORS);
 	});
 }
 
-sub _canonicalize {
-	my ($location) = @_;
+sub ext_canonicalize {
+	my ($location) = $_[-1];
 	if ($location !~ m!\Ahttps?://!) {
 		PublicInbox::Config::rel2abs_collapsed($location);
 	} else {
@@ -56,7 +56,7 @@ sub lei_add_external {
 	my ($self, $location) = @_;
 	my $cfg = $self->_lei_cfg(1);
 	my $new_boost = $self->{opt}->{boost} // 0;
-	$location = _canonicalize($location);
+	$location = ext_canonicalize($location);
 	if ($location !~ m!\Ahttps?://! && !-d $location) {
 		return $self->fail("$location not a directory");
 	}
@@ -74,7 +74,7 @@ sub lei_forget_external {
 	my %seen;
 	for my $loc (@locations) {
 		my (@unset, @not_found);
-		for my $l ($loc, _canonicalize($loc)) {
+		for my $l ($loc, ext_canonicalize($loc)) {
 			next if $seen{$l}++;
 			my $key = "external.$l.boost";
 			delete($cfg->{$key});
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index dea04c13..fd8a3bca 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -7,6 +7,11 @@ use strict;
 use v5.10.1;
 use PublicInbox::DS qw(dwaitpid);
 
+sub prep_ext { # externals_each callback
+	my ($lxs, $exclude, $loc) = @_;
+	$lxs->prepare_external($loc) unless $exclude->{$loc};
+}
+
 # the main "lei q SEARCH_TERMS" method
 sub lei_q {
 	my ($self, @argv) = @_;
@@ -14,22 +19,35 @@ sub lei_q {
 	require PublicInbox::LeiOverview;
 	PublicInbox::Config->json; # preload before forking
 	my $opt = $self->{opt};
+	# prepare any number of LeiXSearch || LeiSearch || Inbox || URL
 	my $lxs = $self->{lxs} = PublicInbox::LeiXSearch->new;
-	# any number of LeiXSearch || LeiSearch || Inbox
-	if ($opt->{'local'} //= 1) { # --local is enabled by default
+	my @only = @{$opt->{only} // []};
+	# --local is enabled by default unless --only is used
+	# we'll allow "--only $LOCATION --local"
+	if ($opt->{'local'} //= scalar(@only) ? 0 : 1) {
 		my $sto = $self->_lei_store(1);
 		$lxs->prepare_external($sto->search);
 	}
-
-	# --external is enabled by default, but allow --no-external
-	if ($opt->{external} //= 1) {
-		my $cb = $lxs->can('prepare_external');
-		my $ne = $self->_externals_each($cb, $lxs);
-		$opt->{remote} //= $ne == $lxs->remotes;
-		if ($opt->{'local'}) {
-			delete($lxs->{remotes}) if !$opt->{remote};
-		} else {
-			delete($lxs->{locals});
+	if (@only) {
+		for my $loc (@only) {
+			$lxs->prepare_external($self->ext_canonicalize($loc));
+		}
+	} else {
+		for my $loc (@{$opt->{include} // []}) {
+			$lxs->prepare_external($self->ext_canonicalize($loc));
+		}
+		# --external is enabled by default, but allow --no-external
+		if ($opt->{external} //= 1) {
+			my %x = map {;
+				($self->ext_canonicalize($_), 1)
+			} @{$self->{exclude} // []};
+			my $ne = $self->externals_each(\&prep_ext, $lxs, \%x);
+			$opt->{remote} //= !($lxs->locals - $opt->{'local'});
+			if ($opt->{'local'}) {
+				delete($lxs->{remotes}) if !$opt->{remote};
+			} else {
+				delete($lxs->{locals});
+			}
 		}
 	}
 	unless ($lxs->locals || $lxs->remotes) {
diff --git a/t/lei_external.t b/t/lei_external.t
index 1f0048a1..587990db 100644
--- a/t/lei_external.t
+++ b/t/lei_external.t
@@ -4,7 +4,7 @@ use v5.10.1;
 use Test::More;
 my $cls = 'PublicInbox::LeiExternal';
 require_ok $cls;
-my $canon = $cls->can('_canonicalize');
+my $canon = $cls->can('ext_canonicalize');
 my $exp = 'https://example.com/my-inbox/';
 is($canon->('https://example.com/my-inbox'), $exp, 'trailing slash added');
 is($canon->('https://example.com/my-inbox//'), $exp, 'trailing slash removed');

  parent reply	other threads:[~2021-02-02 11:47 UTC|newest]

Thread overview: 17+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-02-02 11:46 [PATCH 00/16] lei: -I/--include and more Eric Wong
2021-02-02 11:46 ` [PATCH 01/16] lei: switch to use SEQPACKET socketpair instead of pipe Eric Wong
2021-02-02 11:46 ` [PATCH 02/16] lei_query: default to 10000 messages as documented Eric Wong
2021-02-02 11:46 ` [PATCH 03/16] lei q: emit progress and counting via PktOp Eric Wong
2021-02-02 11:46 ` Eric Wong [this message]
2021-02-02 11:46 ` [PATCH 05/16] lei: complete: do not complete non-arg options w/ help text Eric Wong
2021-02-02 11:46 ` [PATCH 06/16] lei: q: shell completion for --(include|exclude|only) Eric Wong
2021-02-02 11:46 ` [PATCH 07/16] lei_xsearch: truncate curl stderr after reading it Eric Wong
2021-02-02 11:46 ` [PATCH 08/16] lib: explicitly distinguish oneshot use Eric Wong
2021-02-02 11:46 ` [PATCH 09/16] lei q: do not leave temporary files after oneshot exit Eric Wong
2021-02-02 11:46 ` [PATCH 10/16] cmd_ipc4: fix comments and formatting Eric Wong
2021-02-02 11:46 ` [PATCH 11/16] pktop: fix potential undefined var Eric Wong
2021-02-02 11:46 ` [PATCH 12/16] lei_xsearch: ensure curl.err and tail(1) cleanup happens Eric Wong
2021-02-02 11:46 ` [PATCH 13/16] doc: lei-q: note "-a" and link to Xapian QueryParser Eric Wong
2021-02-02 11:47 ` [PATCH 14/16] lei_overview: avoid unnecessary {l2m} delete Eric Wong
2021-02-02 11:47 ` [PATCH 15/16] lei q: tidy up progress reporting Eric Wong
2021-02-02 11:47 ` [PATCH 16/16] lei q: support --jobs [SEARCHERS],[WRITERS] Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210202114702.29886-5-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 04/16] lei q: support --only, --include and --exclude' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).