diff options
Diffstat (limited to 'lib/PublicInbox/LeiQuery.pm')
| -rw-r--r-- | lib/PublicInbox/LeiQuery.pm | 103 |
1 files changed, 56 insertions, 47 deletions
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index cb5ac8fb..eadf811f 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -1,11 +1,10 @@ -# Copyright (C) 2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # handles "lei q" command and provides internals for # several other sub-commands (up, lcat, ...) package PublicInbox::LeiQuery; -use strict; -use v5.10.1; +use v5.12; sub prep_ext { # externals_each callback my ($lxs, $exclude, $loc) = @_; @@ -17,26 +16,32 @@ sub _start_query { # used by "lei q" and "lei up" require PublicInbox::LeiOverview; PublicInbox::LeiOverview->new($self) or return; my $opt = $self->{opt}; + require PublicInbox::OverIdx; # loads DBI + PublicInbox::OverIdx::fork_ok($opt); my ($xj, $mj) = split(/,/, $opt->{jobs} // ''); - if (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) { - return $self->fail("`$xj' search jobs must be >= 1"); - } + (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) and + die "`$xj' search jobs must be >= 1\n"; my $lxs = $self->{lxs}; $xj ||= $lxs->concurrency($opt); # allow: "--jobs ,$WRITER_ONLY" my $nproc = $lxs->detect_nproc || 1; # don't memoize, schedtool(1) exists $xj = $nproc if $xj > $nproc; $lxs->{-wq_nr_workers} = $xj; - if (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) { - return $self->fail("`$mj' writer jobs must be >= 1"); - } + (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) and + die "`$mj' writer jobs must be >= 1\n"; my $l2m = $self->{l2m}; # we use \1 (a ref) to distinguish between default vs. user-supplied if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote import-before))) { $self->_lei_store(1)->write_prepare($self); + if ($opt->{'mail-sync'}) { + my $lms = $l2m->{-lms_rw} = $self->lms(1); + $lms->lms_write_prepare->lms_pause; # just create + } } - $l2m and $l2m->{-wq_nr_workers} = $mj // do { - $mj = int($nproc * 0.75 + 0.5); # keep some CPU for git + $l2m and $l2m->{-wq_nr_workers} //= $mj // do { + # keep some CPU for git, and don't overload IMAP destinations + my $n = int($nproc * 0.75 + 0.5); + $self->{net} && $n > 4 ? 4 : $n; }; # descending docid order is cheapest, MUA controls sorting order @@ -54,20 +59,19 @@ sub _start_query { # used by "lei q" and "lei up" $lxs->do_query($self); } -sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin - my ($self) = @_; # $_[1] = $rbuf - if (defined($_[1])) { - $_[1] eq '' and return eval { - $self->fchdir or return; - $self->{mset_opt}->{q_raw} = $self->{mset_opt}->{qstr}; - $self->{lse}->query_approxidate($self->{lse}->git, - $self->{mset_opt}->{qstr}); - _start_query($self); - }; - $self->{mset_opt}->{qstr} .= $_[1]; - } else { - $self->fail("error reading stdin: $!"); - } +sub do_qry { # do_env cb + my ($lei) = @_; + $lei->{mset_opt}->{q_raw} = $lei->{mset_opt}->{qstr} + = delete $lei->{stdin_buf}; + $lei->{lse}->query_approxidate($lei->{lse}->git, + $lei->{mset_opt}->{qstr}); + _start_query($lei); +} + +# make the URI||PublicInbox::{Inbox,ExtSearch} a config-file friendly string +sub cfg_ext ($) { + my ($x) = @_; + $x->isa('URI') ? "$x" : ($x->{inboxdir} // $x->{topdir}); } sub lxs_prepare { @@ -85,36 +89,40 @@ sub lxs_prepare { $lxs->prepare_external($self->{lse}); } if (@only) { + my $only; for my $loc (@only) { my @loc = $self->get_externals($loc) or return; - $lxs->prepare_external($_) for @loc; + for (@loc) { + my $x = $lxs->prepare_external($_); + push(@$only, cfg_ext($x)) if $x; + } } + $opt->{only} = $only if $only; } else { - my (@ilocals, @iremotes); + my (@ilocals, @iremotes, $incl); for my $loc (@{$opt->{include} // []}) { my @loc = $self->get_externals($loc) or return; - $lxs->prepare_external($_) for @loc; + for (@loc) { + my $x = $lxs->prepare_external($_); + push(@$incl, cfg_ext($x)) if $x; + } @ilocals = @{$lxs->{locals} // []}; @iremotes = @{$lxs->{remotes} // []}; } + $opt->{include} = $incl if $incl; # --external is enabled by default, but allow --no-external if ($opt->{external} //= 1) { - my %x; - for my $loc (@{$opt->{exclude} // []}) { - my @l = $self->get_externals($loc, 1) or return; - $x{$_} = 1 for @l; - } - my $ne = $self->externals_each(\&prep_ext, $lxs, \%x); + my $ex = $self->canonicalize_excludes($opt->{exclude}); + my @excl = keys %$ex; + $opt->{exclude} = \@excl if scalar(@excl); + $self->externals_each(\&prep_ext, $lxs, $ex); $opt->{remote} //= !($lxs->locals - $opt->{'local'}); - if ($opt->{'local'}) { - $lxs->{remotes} = \@iremotes if !$opt->{remote}; - } else { - $lxs->{locals} = \@ilocals; - } + $lxs->{locals} = \@ilocals if !$opt->{'local'}; + $lxs->{remotes} = \@iremotes if !$opt->{remote}; } } ($lxs->locals || $lxs->remotes) ? ($self->{lxs} = $lxs) : - $self->fail('no local or remote inboxes to search'); + die("no local or remote inboxes to search\n"); } # the main "lei q SEARCH_TERMS" method @@ -145,10 +153,9 @@ sub lei_q { return $self->fail(<<'') if @argv; no query allowed on command-line with --stdin - require PublicInbox::InputPipe; - PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self); - return; + return $self->slurp_stdin(\&do_qry); } + chomp(@argv) and $self->qerr("# trailing `\\n' removed"); $mset_opt{q_raw} = [ @argv ]; # copy $mset_opt{qstr} = $self->{lse}->query_argv_to_string($self->{lse}->git, \@argv); @@ -158,11 +165,13 @@ no query allowed on command-line with --stdin # shell completion helper called by lei__complete sub _complete_q { my ($self, @argv) = @_; - my $ext = qr/\A(?:-I|(?:--(?:include|exclude|only)))\z/; + join('', @argv) =~ /\bL:\S*\z/ and + return eval { $self->_lei_store->search->all_terms('L') }; my @cur; + my $cb = $self->lazy_cb(qw(forget-external _complete_)); while (@argv) { - if ($argv[-1] =~ $ext) { - my @c = $self->_complete_forget_external(@cur); + if ($argv[-1] =~ /\A(?:-I|(?:--(?:include|exclude|only)))\z/) { + my @c = $cb->($self, @cur); # try basename match: if (scalar(@cur) == 1 && index($cur[0], '/') < 0) { my $all = $self->externals_each; @@ -192,7 +201,7 @@ sub _complete_q { # FIXME: Getopt::Long doesn't easily let us support support options with # '.' in them (e.g. --http1.1) # TODO: should we depend on "-c http.*" options for things which have -# analogues in git(1)? that would reduce likelyhood of conflicts with +# analogues in git(1)? that would reduce likelihood of conflicts with # our other CLI options # Note: some names are renamed to avoid potential conflicts, # see %lei2curl in lib/PublicInbox/LeiCurl.pm |