From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id AE3F71F93C for ; Thu, 1 Dec 2022 11:21:33 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669893693; bh=KuLXbuOY2kSvug5mDmmg3eSkP9ZyEkkomdMSiw0Q4JQ=; h=From:To:Subject:Date:In-Reply-To:References:From; b=GoFb6N+cNl9emjA5pCBjQnKfgTNsgSsfl769Gqwli4alL5SKeTRVvSqsw1ywiBmcI TsD90cOA+DClb8gMR8NHFz/pte1VauBsAOcV6EgaF+jayJYOPcyesT60lzGvzs7dip QKezuHOTKSs0+eCFuMtUXtoHDhuKg6chbEsy8RAA= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 2/2] lei_saved_search: expand only/include/exclude to absolute paths Date: Thu, 1 Dec 2022 11:21:32 +0000 Message-Id: <20221201112132.4005900-3-e@80x24.org> In-Reply-To: <20221201112132.4005900-1-e@80x24.org> References: <20221201112132.4005900-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: While users may specify relative paths for convenience on the command-line, absolute paths are required for `lei up' since that (especially `lei up --all') could run from anywhere. Note that we need to do this when parsing the command-line options, since shortcuts for URL matching on URL path components are allowed for `lei q', and those same shortcuts may remain in effect across to `lei up' as the underlying external may be moved to a different URI host. --- lib/PublicInbox/LeiQuery.pm | 23 ++++++++++++++++++++--- lib/PublicInbox/LeiXSearch.pm | 9 +++++++-- t/lei-q-save.t | 13 +++++++++---- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index 0f839236..358574ea 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -74,6 +74,12 @@ sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin $lei->fail($@) if $@; } +# make the URI||PublicInbox::{Inbox,ExtSearch} a config-file friendly string +sub cfg_ext ($) { + my ($x) = @_; + $x->isa('URI') ? "$x" : ($x->{inboxdir} // $x->{topdir}); +} + sub lxs_prepare { my ($self) = @_; require PublicInbox::LeiXSearch; @@ -89,21 +95,32 @@ sub lxs_prepare { $lxs->prepare_external($self->{lse}); } if (@only) { + my $only; for my $loc (@only) { my @loc = $self->get_externals($loc) or return; - $lxs->prepare_external($_) for @loc; + for (@loc) { + my $x = $lxs->prepare_external($_); + push(@$only, cfg_ext($x)) if $x; + } } + $opt->{only} = $only if $only; } else { - my (@ilocals, @iremotes); + my (@ilocals, @iremotes, $incl); for my $loc (@{$opt->{include} // []}) { my @loc = $self->get_externals($loc) or return; - $lxs->prepare_external($_) for @loc; + for (@loc) { + my $x = $lxs->prepare_external($_); + push(@$incl, cfg_ext($x)) if $x; + } @ilocals = @{$lxs->{locals} // []}; @iremotes = @{$lxs->{remotes} // []}; } + $opt->{include} = $incl if $incl; # --external is enabled by default, but allow --no-external if ($opt->{external} //= 1) { my $ex = $self->canonicalize_excludes($opt->{exclude}); + my @excl = keys %$ex; + $opt->{exclude} = \@excl if scalar(@excl); $self->externals_each(\&prep_ext, $lxs, $ex); $opt->{remote} //= !($lxs->locals - $opt->{'local'}); $lxs->{locals} = \@ilocals if !$opt->{'local'}; diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index 8e195c4c..730df1f7 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -605,15 +605,19 @@ sub add_uri { require IO::Uncompress::Gunzip; require PublicInbox::LeiCurl; push @{$self->{remotes}}, $uri; + $uri; } else { warn "curl missing, ignoring $uri\n"; + undef; } } +# returns URI or PublicInbox::Inbox-like object sub prepare_external { my ($self, $loc, $boost) = @_; # n.b. already ordered by boost if (ref $loc) { # already a URI, or PublicInbox::Inbox-like object return add_uri($self, $loc) if $loc->can('scheme'); + # fall-through on Inbox-like objects } elsif ($loc =~ m!\Ahttps?://!) { require URI; return add_uri($self, URI->new($loc)); @@ -628,12 +632,13 @@ sub prepare_external { $loc = bless { inboxdir => $loc }, 'PublicInbox::Inbox'; } elsif (!-e $loc) { warn "W: $loc gone, perhaps run: lei forget-external $loc\n"; - return; + return undef; } else { warn "W: $loc ignored, unable to determine external type\n"; - return; + return undef; } push @{$self->{locals}}, $loc; + $loc; } sub _lcat_i { # LeiMailSync->each_src iterator callback diff --git a/t/lei-q-save.t b/t/lei-q-save.t index 3d09fe37..d09c8397 100644 --- a/t/lei-q-save.t +++ b/t/lei-q-save.t @@ -1,5 +1,5 @@ #!perl -w -# Copyright (C) 2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ use strict; use v5.10.1; use PublicInbox::TestCommon; use PublicInbox::Smsg; @@ -12,7 +12,7 @@ my $doc2 = eml_load('t/utf8.eml'); $doc2->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 4)})); my $doc3 = eml_load('t/msg_iter-order.eml'); $doc3->header_set('Date', PublicInbox::Smsg::date({ds => time - (86400 * 4)})); - +my $cat_env = { VISUAL => 'cat', EDITOR => 'cat' }; my $pre_existing = <<'EOF'; From x Mon Sep 17 00:00:00 2001 Message-ID: @@ -183,7 +183,12 @@ test_lei(sub { lei_ok(qw(q z:0.. -o), "v2:$v2"); like($lei_err, qr/^# ([1-9][0-9]*) written to \Q$v2\E/sm, 'non-zero write output to stderr'); - lei_ok(qw(q z:0.. -o), "mboxrd:$home/before", '--only', $v2, '-j1,1'); + lei_ok('-C', $v2, qw(q z:0.. -o), "mboxrd:$home/before", + '--only', '.', '-j1,1'); + lei_ok(['edit-search', "$home/before"], $cat_env); + like($lei_out, qr/^\tonly = \Q$v2\E$/sm, + 'relative --only saved to absolute path'); + open my $fh, '<', "$home/before"; PublicInbox::MboxReader->mboxrd($fh, sub { push @before, $_[0] }); isnt(scalar(@before), 0, 'initial v2 written'); @@ -207,7 +212,7 @@ test_lei(sub { ok($shared < $orig, 'fewer bytes stored with --shared') or diag "shared=$shared orig=$orig"; - lei_ok([qw(edit-search), $v2s], { VISUAL => 'cat', EDITOR => 'cat' }); + lei_ok([qw(edit-search), $v2s], $cat_env); like($lei_out, qr/^\[lei/sm, 'edit-search can cat'); lei_ok('-C', "$home/v2s", qw(q -q -o ../s m:testmessage@example.com));