From d99020ac3612308d04e3760bd780417218748168 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 19 Aug 2021 01:36:38 +0000 Subject: lei q: make --save the default Since "lei up" is more often useful than not and incurs neglible overhead; enable --save by default and allow --no-save to work. This also fixes a long-standing when overwriting --output destinations with saved searches: dedupe data from previous searches are reset and no longer influences the new (changed) search, so results no longer go missing if two sequential invocations of "lei q --save" point to the same --output. --- lib/PublicInbox/LEI.pm | 4 ++-- lib/PublicInbox/LeiSavedSearch.pm | 10 ++++++++++ lib/PublicInbox/LeiToMail.pm | 31 +++++++++++++++++++++++++++---- lib/PublicInbox/LeiUp.pm | 1 - 4 files changed, 39 insertions(+), 7 deletions(-) (limited to 'lib/PublicInbox') diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index e5232f1b..79dc9bf9 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -173,7 +173,7 @@ our %CMD = ( # sorted in order of importance/use: 'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms', 'stdin|', # /|\z/ must be first for lone dash @lxs_opt, - qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+ + qw(save! output|mfolder|o=s format|f=s dedupe|d=s threads|t+ sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+ shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ], @@ -337,7 +337,7 @@ my %OPTDESC = ( 'torsocks=s' => ['VAL|auto|no|yes', 'whether or not to wrap git and curl commands with torsocks'], 'no-torsocks' => 'alias for --torsocks=no', -'save' => "save a search for `lei up'", +'save!' => "do not save a search for `lei up'", 'import-remote!' => 'do not memoize remote messages into local store', 'type=s' => [ 'any|mid|git', 'disambiguate type' ], diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm index 2a0e9321..fd51fe38 100644 --- a/lib/PublicInbox/LeiSavedSearch.pm +++ b/lib/PublicInbox/LeiSavedSearch.pm @@ -243,6 +243,16 @@ sub pause_dedupe { $oidx->commit_lazy; } +sub reset_dedupe { + my ($self) = @_; + prepare_dedupe($self); + my $lk = $self->lock_for_scope_fast; + for my $t (qw(xref3 over id2num)) { + $self->{oidx}->{dbh}->do("DELETE FROM $t"); + } + pause_dedupe($self); +} + sub mm { undef } sub altid_map { {} } diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index d782d4c7..be6e178f 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -390,10 +390,16 @@ sub new { -e $dst && !-d _ and die "$dst exists and is not a directory\n"; $lei->{ovv}->{dst} = $dst .= '/' if substr($dst, -1) ne '/'; + $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q'; } elsif (substr($fmt, 0, 4) eq 'mbox') { require PublicInbox::MboxReader; $self->can("eml2$fmt") or die "bad mbox format: $fmt\n"; $self->{base_type} = 'mbox'; + if ($lei->{cmd} eq 'q' && + (($lei->path_to_fd($dst) // -1) < 0) && + (-f $dst || !-e _)) { + $lei->{opt}->{save} //= \1; + } } elsif ($fmt =~ /\Aimaps?\z/) { require PublicInbox::NetWriter; require PublicInbox::URIimap; @@ -408,6 +414,7 @@ sub new { $dst = $lei->{ovv}->{dst} = $$uri; # canonicalized $lei->{net} = $net; $self->{base_type} = 'imap'; + $lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q'; } elsif ($fmt eq 'text') { require PublicInbox::LeiViewText; $lei->{lvt} = PublicInbox::LeiViewText->new($lei); @@ -454,9 +461,17 @@ sub _pre_augment_maildir { open $self->{poke_dh}, '<', "${dst}cur" or die "open ${dst}cur: $!"; } +sub clobber_dst_prepare ($;$) { + my ($lei, $f) = @_; + my $wait = (defined($f) && $lei->{sto}) ? + $lei->{sto}->ipc_do('lms_forget_folders', $f) : undef; + my $dedupe = $lei->{dedupe} or return; + $dedupe->reset_dedupe if $dedupe->can('reset_dedupe'); +} + sub _do_augment_maildir { my ($self, $lei) = @_; - return if ($lei->{opt}->{save} // 0) < 0; + return if $lei->{cmd} eq 'up'; my $dst = $lei->{ovv}->{dst}; my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef; my $mdr = PublicInbox::MdirReader->new; @@ -468,9 +483,11 @@ sub _do_augment_maildir { $dedupe->pause_dedupe; } } elsif ($lse) { + clobber_dst_prepare($lei, "maildir:$dst"); $mdr->{shard_info} = $self->{shard_info}; $mdr->maildir_each_eml($dst, \&_md_update, $lei, $lse, 1); } else {# clobber existing Maildir + clobber_dst_prepare($lei, "maildir:$dst"); $mdr->maildir_each_file($dst, \&_unlink); } } @@ -487,7 +504,7 @@ sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb sub _do_augment_imap { my ($self, $lei) = @_; - return if ($lei->{opt}->{save} // 0) < 0; + return if $lei->{cmd} eq 'up'; my $net = $lei->{net}; my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef; if ($lei->{opt}->{augment}) { @@ -499,11 +516,13 @@ sub _do_augment_imap { } } elsif ($lse) { my $delete_mic; + clobber_dst_prepare($lei, "$self->{uri}"); $net->imap_each($self->{uri}, \&_imap_augment_or_delete, $lei, $lse, \$delete_mic); $delete_mic->expunge if $delete_mic; } elsif (!$self->{-wq_worker_nr}) { # undef or 0 # clobber existing IMAP folder + clobber_dst_prepare($lei, "$self->{uri}"); $net->imap_delete_all($self->{uri}); } } @@ -563,6 +582,8 @@ sub _pre_augment_mbox { if $imp_before && !ref($imp_before); die "--augment specified but $dst is not seekable\n" if $lei->{opt}->{augment}; + die "cannot --save with unseekable $dst\n" if + $lei->{dedupe} && $lei->{dedupe}->can('reset_dedupe'); } if ($self->{zsfx} = PublicInbox::MboxReader::zsfx($dst)) { pipe(my ($r, $w)) or die "pipe: $!"; @@ -581,10 +602,10 @@ sub _do_augment_mbox { my ($self, $lei) = @_; return unless $self->{seekable}; my $opt = $lei->{opt}; - return if ($opt->{save} // 0) < 0; + return if $lei->{cmd} eq 'up'; my $out = $lei->{1}; my ($fmt, $dst) = @{$lei->{ovv}}{qw(fmt dst)}; - return unless -s $out; + return clobber_dst_prepare($lei) unless -s $out; unless ($opt->{augment} || $opt->{'import-before'}) { truncate($out, 0) or die "truncate($dst): $!"; return; @@ -599,6 +620,8 @@ sub _do_augment_mbox { if ($opt->{augment}) { $dedupe = $lei->{dedupe}; $dedupe->prepare_dedupe if $dedupe; + } else { + clobber_dst_prepare($lei); } if ($opt->{'import-before'}) { # the default my $lse = $lei->{lse}; diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm index c5a01527..ba11761a 100644 --- a/lib/PublicInbox/LeiUp.pm +++ b/lib/PublicInbox/LeiUp.pm @@ -64,7 +64,6 @@ sub lei_up { my ($lei, @outs) = @_; $lei->{lse} = $lei->_lei_store(1)->search; my $opt = $lei->{opt}; - $opt->{save} = -1; my @local; if (defined $opt->{all}) { return $lei->fail("--all and @outs incompatible") if @outs; -- cgit v1.2.3-24-ge0c7