From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 836D11FC0D for ; Sat, 18 Sep 2021 09:33:34 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 9/9] lei up: automatically use dt: for remote externals Date: Sat, 18 Sep 2021 09:33:32 +0000 Message-Id: <20210918093332.16054-10-e@80x24.org> In-Reply-To: <20210918093332.16054-1-e@80x24.org> References: <20210918093332.16054-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Since we can't use maxuid for remote externals, automatically maintaining the last time we got results and appending a dt: range to the query will prevent HTTP(S) responses from getting too big. We could be using "rt:", but no stable release of public-inbox supports it, yet, so we'll use dt:, instead. By default, there's a two day fudge factor to account for MTA downtime and delays; which is hopefully enough. The fudge factor may be changed per-invocation with the --remote-fudge-factor=INTERVAL option Since different externals can have different message transport routes, "lastresult" entries are stored on a per-external basis. --- Documentation/lei-up.pod | 15 ++++++++++ lib/PublicInbox/LEI.pm | 7 +++-- lib/PublicInbox/LeiSavedSearch.pm | 1 + lib/PublicInbox/LeiToMail.pm | 4 ++- lib/PublicInbox/LeiUp.pm | 2 +- lib/PublicInbox/LeiXSearch.pm | 50 ++++++++++++++++++++++++++----- t/lei-q-remote-import.t | 4 +++ 7 files changed, 71 insertions(+), 12 deletions(-) diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod index e5d97f43..4dc6d3b8 100644 --- a/Documentation/lei-up.pod +++ b/Documentation/lei-up.pod @@ -22,6 +22,21 @@ C<--all> updates all saved searches (listed in L). C<--all=local> only updates local mailboxes, C<--all=remote> only updates remote mailboxes (currently C and C). +=item --remote-fudge-time=INTERVAL + +Look for mail older than the time of the last successful query. +Using a small interval will reduce bandwidth use. A larger +interval reduces the likelyhood missing a result due to MTA +delays or downtime. + +The time(s) of the last successful queries are the C +values visible from L. + +Date formats understood by L may be used. +e.g C<1.hour> or C<3.days> + +Default: 2.days + =back The following options, described in L, are supported. diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 053b6174..8b0614f2 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -183,7 +183,8 @@ our %CMD = ( # sorted in order of importance/use: shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ], 'up' => [ 'OUTPUT...|--all', 'update saved search', - qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ], + qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ + remote-fudge-time=s all:s), @c_opt ], 'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)', 'stdin|', # /|\z/ must be first for lone dash @@ -420,7 +421,9 @@ my %OPTDESC = ( 'remote' => 'limit operations to those requiring network access', 'remote!' => 'prevent operations requiring network access', -'all:s up' => ['local', 'update all (local) saved searches' ], +'all:s up' => ['local|remote', 'update all remote or local saved searches' ], +'remote-fudge-time=s' => [ 'INTERVAL', + 'look for mail INTERVAL older than the last successful query' ], 'mid=s' => 'specify the Message-ID of a message', 'oid=s' => 'specify the git object ID of a message', diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm index 96ff816e..754f8294 100644 --- a/lib/PublicInbox/LeiSavedSearch.pm +++ b/lib/PublicInbox/LeiSavedSearch.pm @@ -149,6 +149,7 @@ sub new { # new saved search "lei q --save" $dst = "$lei->{ovv}->{fmt}:$dst" if $dst !~ m!\Aimaps?://!i; print $fh <{opt}->{save} ? 'LeiSavedSearch' : 'LeiDedupe'); eval "require $dd_cls"; die "$dd_cls: $@" if $@; - $dd_cls->new($lei); + my $dd = $dd_cls->new($lei); + $lei->{lss} //= $dd if $dd && $dd->can('cfg_set'); + $dd; }; $self; } diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm index 4637cb46..abb05d46 100644 --- a/lib/PublicInbox/LeiUp.pm +++ b/lib/PublicInbox/LeiUp.pm @@ -45,7 +45,7 @@ sub up1 ($$) { ref($v) and return $lei->fail("multiple values of $c in $f"); $lei->{opt}->{$k} = $v; } - $lei->{lss} = $lss; # for LeiOverview->new + $lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd my $lxs = $lei->lxs_prepare or return; $lei->ale->refresh_externals($lxs, $lei); $lei->_start_query; diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index 50cadb5e..1d49da3d 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -18,6 +18,7 @@ use PublicInbox::Smsg; use PublicInbox::Eml; use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR); use PublicInbox::ContentHash qw(git_sha); +use POSIX qw(strftime); sub new { my ($class) = @_; @@ -168,8 +169,7 @@ sub query_one_mset { # for --threads and l2m w/o sort my $can_kw = !!$ibxish->can('msg_keywords'); my $threads = $lei->{opt}->{threads} // 0; my $fl = $threads > 1 ? 1 : undef; - my $lss = $lei->{dedupe}; - $lss = undef unless $lss && $lss->can('cfg_set'); # saved search + my $lss = $lei->{lss}; my $maxk = "external.$dir.maxuid"; my $stop_at = $lss ? $lss->{-cfg}->{$maxk} : undef; if (defined $stop_at) { @@ -292,6 +292,37 @@ sub each_remote_eml { # callback for MboxReader->mboxrd $each_smsg->($smsg, undef, $eml); } +sub fudge_qstr_time ($$$) { + my ($lei, $uri, $qstr) = @_; + return ($qstr, undef) unless $lei->{lss}; + my $cfg = $lei->{lss}->{-cfg} // die 'BUG: no lss->{-cfg}'; + my $cfg_key = "external.$uri.lastresult"; + my $lr = $cfg->{$cfg_key} or return ($qstr, $cfg_key); + if ($lr !~ /\A\-?[0-9]+\z/) { + $lei->child_error(0, + "$cfg->{-f}: $cfg_key=$lr not an integer, ignoring"); + return ($qstr, $cfg_key); + } + my $rft = $lei->{opt}->{'remote-fudge-time'}; + if ($rft && $rft !~ /\A-?[0-9]+\z/) { + my @t = $lei->{lss}->git->date_parse($rft); + my $diff = time - $t[0]; + $lei->qerr("# $rft => $diff seconds"); + $rft = $diff; + } + $lr -= ($rft || (48 * 60 * 60)); + $lei->qerr("# $uri limiting to ". + strftime('%Y-%m-%d %k:%M', gmtime($lr)). ' and newer'); + # this should really be rt: (received-time), but no stable + # public-inbox releases support it, yet. + my $dt = 'dt:'.strftime('%Y%m%d%H%M%S', gmtime($lr)).'..'; + if ($qstr =~ /\S/) { + substr($qstr, 0, 0, '('); + $qstr .= ') AND '; + } + ($qstr .= $dt, $cfg_key); +} + sub query_remote_mboxrd { my ($self, $uris) = @_; local $0 = "$0 query_remote_mboxrd"; @@ -300,7 +331,7 @@ sub query_remote_mboxrd { my $opt = $lei->{opt}; my $qstr = $lei->{mset_opt}->{qstr}; $qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly - my @qform = (q => $qstr, x => 'm'); + my @qform = (x => 'm'); push(@qform, t => 1) if $opt->{threads}; my $verbose = $opt->{verbose}; my ($reap_tail, $reap_curl); @@ -323,7 +354,9 @@ sub query_remote_mboxrd { for my $uri (@$uris) { $lei->{-current_url} = $uri->as_string; $lei->{-nr_remote_eml} = 0; - $uri->query_form(@qform); + my $start = time; + my ($q, $key) = fudge_qstr_time($lei, $uri, $qstr); + $uri->query_form(@qform, q => $q); my $cmd = $curl->for_uri($lei, $uri); $lei->qerr("# $cmd"); my ($fh, $pid) = popen_rd($cmd, undef, $rdr); @@ -336,10 +369,11 @@ sub query_remote_mboxrd { @$reap_curl = (); # cancel OnDestroy die $err if $err; my $nr = $lei->{-nr_remote_eml}; - if ($nr && $lei->{sto}) { - my $wait = $lei->{sto}->ipc_do('done'); - } + my $wait = $lei->{sto}->ipc_do('done') if $nr && $lei->{sto}; if ($? == 0) { + # don't update if no results, maybe MTA is down + $key && $nr and + $lei->{lss}->cfg_set($key, $start); mset_progress($lei, $lei->{-current_url}, $nr, $nr); next; } @@ -353,7 +387,7 @@ sub query_remote_mboxrd { $lei->err("truncate($cmd stderr): $!"); } next if (($? >> 8) == 22 && $err =~ /\b404\b/); - $uri->query_form(q => $lei->{mset_opt}->{qstr}); + $uri->query_form(q => $qstr); $lei->child_error($?, "E: <$uri> $err"); } undef $each_smsg; diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t index 9131c01b..fdf6a11e 100644 --- a/t/lei-q-remote-import.t +++ b/t/lei-q-remote-import.t @@ -99,5 +99,9 @@ EOF lei_ok('up', "$ENV{HOME}/md"); is_deeply(\@f, [ glob("$ENV{HOME}/md/*/*") ], 'lei up remote dedupe works on maildir'); + my $edit_env = { VISUAL => 'cat' }; + lei_ok([qw(edit-search), "$ENV{HOME}/md"], $edit_env); + like($lei_out, qr/^\Q[external "$url"]\E\n\s*lastresult = \d+/sm, + 'lastresult set'); }); done_testing;