inbox: rewrite cleanup to be more aggressive

Avoid relying on a giant cleanup hash and instead use the new
DS->add_uniq_timer API to amortize the pause times associated
with having to cleanup many inboxes.  We can also use smaller
intervals for this, as well.

We now discard SQLite DB handles at cleanup.  Each of these can
use several megabytes of memory, which adds up with
hundreds/thousands of inboxes.  Since per-inbox access intervals
are unpredictable and opening an SQLite handle is relatively
inexpensive, release memory more aggressively to avoid the heap
having to hit swap.

1 files changed, 7 insertions, 5 deletions

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 3c577ab3..d5b1d39d 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -392,8 +392,10 @@ sub async_wait_all ($) {
 
 # returns true if there are pending "git cat-file" processes
 sub cleanup {
-        my ($self) = @_;
+        my ($self, $lazy) = @_;
         local $in_cleanup = 1;
+        return 1 if $lazy && (scalar(@{$self->{inflight_c} // []}) ||
+                                scalar(@{$self->{inflight} // []}));
         delete $self->{async_cat};
         async_wait_all($self);
         delete $self->{inflight};
@@ -403,7 +405,6 @@ sub cleanup {
         defined($self->{pid}) || defined($self->{pid_c});
 }
 
-
 # assuming a well-maintained repo, this should be a somewhat
 # accurate estimation of its size
 # TODO: show this in the WWW UI as a hint to potential cloners
@@ -526,18 +527,19 @@ sub manifest_entry {
 # returns true if there are pending cat-file processes
 sub cleanup_if_unlinked {
         my ($self) = @_;
-        return cleanup($self) if $^O ne 'linux';
+        return cleanup($self, 1) if $^O ne 'linux';
         # Linux-specific /proc/$PID/maps access
         # TODO: support this inside git.git
         my $ret = 0;
         for my $fld (qw(pid pid_c)) {
                 my $pid = $self->{$fld} // next;
-                open my $fh, '<', "/proc/$pid/maps" or return cleanup($self);
+                open my $fh, '<', "/proc/$pid/maps" or return cleanup($self, 1);
                 while (<$fh>) {
                         # n.b. we do not restart for unlinked multi-pack-index
                         # since it's not too huge, and the startup cost may
                         # be higher.
-                        return cleanup($self) if /\.(?:idx|pack) \(deleted\)$/;
+                        /\.(?:idx|pack) \(deleted\)$/ and
+                                return cleanup($self, 1);
                 }
                 ++$ret;
         }