lei up: automatically use dt: for remote externals

Since we can't use maxuid for remote externals, automatically
maintaining the last time we got results and appending a dt:
range to the query will prevent HTTP(S) responses from getting
too big.

We could be using "rt:", but no stable release of public-inbox
supports it, yet, so we'll use dt:, instead.

By default, there's a two day fudge factor to account for MTA
downtime and delays; which is hopefully enough.  The fudge
factor may be changed per-invocation with the
--remote-fudge-factor=INTERVAL option

Since different externals can have different message transport
routes, "lastresult" entries are stored on a per-external basis.

5 files changed, 52 insertions, 12 deletions

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 053b6174..8b0614f2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -183,7 +183,8 @@ our %CMD = ( # sorted in order of importance/use:
         shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
 
 'up' => [ 'OUTPUT...|--all', 'update saved search',
-        qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
+        qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+
+        remote-fudge-time=s all:s), @c_opt ],
 
 'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
         'stdin|', # /|\z/ must be first for lone dash
@@ -420,7 +421,9 @@ my %OPTDESC = (
 'remote' => 'limit operations to those requiring network access',
 'remote!' => 'prevent operations requiring network access',
 
-'all:s        up' => ['local', 'update all (local) saved searches' ],
+'all:s        up' => ['local|remote', 'update all remote or local saved searches' ],
+'remote-fudge-time=s' => [ 'INTERVAL',
+        'look for mail INTERVAL older than the last successful query' ],
 
 'mid=s' => 'specify the Message-ID of a message',
 'oid=s' => 'specify the git object ID of a message',
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 96ff816e..754f8294 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -149,6 +149,7 @@ sub new { # new saved search "lei q --save"
         $dst = "$lei->{ovv}->{fmt}:$dst" if $dst !~ m!\Aimaps?://!i;
         print $fh <<EOM;
 ; to refresh with new results, run: lei up $sq_dst
+; `maxuid' and `lastresult' lines are maintained by "lei up" for optimization
 [lei]
 $q
 [lei "q"]
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d3253d9b..9f7171fb 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -437,7 +437,9 @@ sub new {
                         ($lei->{opt}->{save} ? 'LeiSavedSearch' : 'LeiDedupe');
                 eval "require $dd_cls";
                 die "$dd_cls: $@" if $@;
-                $dd_cls->new($lei);
+                my $dd = $dd_cls->new($lei);
+                $lei->{lss} //= $dd if $dd && $dd->can('cfg_set');
+                $dd;
         };
         $self;
 }
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 4637cb46..abb05d46 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -45,7 +45,7 @@ sub up1 ($$) {
                 ref($v) and return $lei->fail("multiple values of $c in $f");
                 $lei->{opt}->{$k} = $v;
         }
-        $lei->{lss} = $lss; # for LeiOverview->new
+        $lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
         my $lxs = $lei->lxs_prepare or return;
         $lei->ale->refresh_externals($lxs, $lei);
         $lei->_start_query;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 50cadb5e..1d49da3d 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -18,6 +18,7 @@ use PublicInbox::Smsg;
 use PublicInbox::Eml;
 use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
 use PublicInbox::ContentHash qw(git_sha);
+use POSIX qw(strftime);
 
 sub new {
         my ($class) = @_;
@@ -168,8 +169,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
         my $can_kw = !!$ibxish->can('msg_keywords');
         my $threads = $lei->{opt}->{threads} // 0;
         my $fl = $threads > 1 ? 1 : undef;
-        my $lss = $lei->{dedupe};
-        $lss = undef unless $lss && $lss->can('cfg_set'); # saved search
+        my $lss = $lei->{lss};
         my $maxk = "external.$dir.maxuid";
         my $stop_at = $lss ? $lss->{-cfg}->{$maxk} : undef;
         if (defined $stop_at) {
@@ -292,6 +292,37 @@ sub each_remote_eml { # callback for MboxReader->mboxrd
         $each_smsg->($smsg, undef, $eml);
 }
 
+sub fudge_qstr_time ($$$) {
+        my ($lei, $uri, $qstr) = @_;
+        return ($qstr, undef) unless $lei->{lss};
+        my $cfg = $lei->{lss}->{-cfg} // die 'BUG: no lss->{-cfg}';
+        my $cfg_key = "external.$uri.lastresult";
+        my $lr = $cfg->{$cfg_key} or return ($qstr, $cfg_key);
+        if ($lr !~ /\A\-?[0-9]+\z/) {
+                $lei->child_error(0,
+                        "$cfg->{-f}: $cfg_key=$lr not an integer, ignoring");
+                return ($qstr, $cfg_key);
+        }
+        my $rft = $lei->{opt}->{'remote-fudge-time'};
+        if ($rft && $rft !~ /\A-?[0-9]+\z/) {
+                my @t = $lei->{lss}->git->date_parse($rft);
+                my $diff = time - $t[0];
+                $lei->qerr("# $rft => $diff seconds");
+                $rft = $diff;
+        }
+        $lr -= ($rft || (48 * 60 * 60));
+        $lei->qerr("# $uri limiting to ".
+                strftime('%Y-%m-%d %k:%M', gmtime($lr)). ' and newer');
+        # this should really be rt: (received-time), but no stable
+        # public-inbox releases support it, yet.
+        my $dt = 'dt:'.strftime('%Y%m%d%H%M%S', gmtime($lr)).'..';
+        if ($qstr =~ /\S/) {
+                substr($qstr, 0, 0, '(');
+                $qstr .= ') AND ';
+        }
+        ($qstr .= $dt, $cfg_key);
+}
+
 sub query_remote_mboxrd {
         my ($self, $uris) = @_;
         local $0 = "$0 query_remote_mboxrd";
@@ -300,7 +331,7 @@ sub query_remote_mboxrd {
         my $opt = $lei->{opt};
         my $qstr = $lei->{mset_opt}->{qstr};
         $qstr =~ s/[ \n\t]+/ /sg; # make URLs less ugly
-        my @qform = (q => $qstr, x => 'm');
+        my @qform = (x => 'm');
         push(@qform, t => 1) if $opt->{threads};
         my $verbose = $opt->{verbose};
         my ($reap_tail, $reap_curl);
@@ -323,7 +354,9 @@ sub query_remote_mboxrd {
         for my $uri (@$uris) {
                 $lei->{-current_url} = $uri->as_string;
                 $lei->{-nr_remote_eml} = 0;
-                $uri->query_form(@qform);
+                my $start = time;
+                my ($q, $key) = fudge_qstr_time($lei, $uri, $qstr);
+                $uri->query_form(@qform, q => $q);
                 my $cmd = $curl->for_uri($lei, $uri);
                 $lei->qerr("# $cmd");
                 my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
@@ -336,10 +369,11 @@ sub query_remote_mboxrd {
                 @$reap_curl = (); # cancel OnDestroy
                 die $err if $err;
                 my $nr = $lei->{-nr_remote_eml};
-                if ($nr && $lei->{sto}) {
-                        my $wait = $lei->{sto}->ipc_do('done');
-                }
+                my $wait = $lei->{sto}->ipc_do('done') if $nr && $lei->{sto};
                 if ($? == 0) {
+                        # don't update if no results, maybe MTA is down
+                        $key && $nr and
+                                $lei->{lss}->cfg_set($key, $start);
                         mset_progress($lei, $lei->{-current_url}, $nr, $nr);
                         next;
                 }
@@ -353,7 +387,7 @@ sub query_remote_mboxrd {
                                         $lei->err("truncate($cmd stderr): $!");
                 }
                 next if (($? >> 8) == 22 && $err =~ /\b404\b/);
-                $uri->query_form(q => $lei->{mset_opt}->{qstr});
+                $uri->query_form(q => $qstr);
                 $lei->child_error($?, "E: <$uri> $err");
         }
         undef $each_smsg;