* [PATCH 2/5] inbox: perform cleanup of Git objects for coderepos
2019-01-31 4:27 5% [PATCH 0/5] a few more solver fixups and improvements Eric Wong
@ 2019-01-31 4:27 7% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2019-01-31 4:27 UTC (permalink / raw)
To: meta
Otherwise, long-running but idle git processes may keep unlinked
packs around indefinitely and waste disk space.
---
lib/PublicInbox/Git.pm | 18 ++++++++++++++----
lib/PublicInbox/Inbox.pm | 17 +++++++++++++++--
t/git.t | 4 ++++
3 files changed, 33 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index e844884..a756684 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -206,7 +206,15 @@ sub check {
}
sub _destroy {
- my ($self, $in, $out, $pid) = @_;
+ my ($self, $in, $out, $pid, $expire) = @_;
+ my $rfh = $self->{$in} or return;
+ if (defined $expire) {
+ # at least FreeBSD 11.2 and Linux 4.20 update mtime of the
+ # read end of a pipe when the pipe is written to; dunno
+ # about other OSes.
+ my $mtime = (stat($rfh))[9];
+ return if $mtime > $expire;
+ }
my $p = delete $self->{$pid} or return;
foreach my $f ($in, $out) {
delete $self->{$f};
@@ -236,10 +244,12 @@ sub qx {
<$fh>
}
+# returns true if there are pending "git cat-file" processes
sub cleanup {
- my ($self) = @_;
- _destroy($self, qw(in out pid));
- _destroy($self, qw(in_c out_c pid_c));
+ my ($self, $expire) = @_;
+ _destroy($self, qw(in out pid), $expire);
+ _destroy($self, qw(in_c out_c pid_c), $expire);
+ !!($self->{pid} || $self->{pid_c});
}
# assuming a well-maintained repo, this should be a somewhat
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index d57e46d..6fe896f 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -22,12 +22,25 @@ my $cleanup_broken = $@;
my $CLEANUP = {}; # string(inbox) -> inbox
sub cleanup_task () {
$cleanup_timer = undef;
+ my $next = {};
for my $ibx (values %$CLEANUP) {
- foreach my $f (qw(git mm search)) {
+ my $again;
+ foreach my $f (qw(mm search)) {
delete $ibx->{$f} if SvREFCNT($ibx->{$f}) == 1;
}
+ my $expire = time - 60;
+ if (my $git = $ibx->{git}) {
+ $again = $git->cleanup($expire);
+ }
+ if (my $gits = $ibx->{-repo_objs}) {
+ foreach my $git (@$gits) {
+ $again = 1 if $git->cleanup($expire);
+ }
+ }
+ $again ||= !!($ibx->{mm} || $ibx->{search});
+ $next->{"$ibx"} = $ibx if $again;
}
- $CLEANUP = {};
+ $CLEANUP = $next;
}
sub _cleanup_later ($) {
diff --git a/t/git.t b/t/git.t
index 9c80fbb..d637e63 100644
--- a/t/git.t
+++ b/t/git.t
@@ -142,6 +142,10 @@ if ('alternates reloaded') {
open $fh, '<', "$alt/config" or die "open failed: $!\n";
my $config = eval { local $/; <$fh> };
is($$found, $config, 'alternates reloaded');
+
+ ok($gcf->cleanup(time - 30), 'cleanup did not expire');
+ ok(!$gcf->cleanup(time + 30), 'cleanup can expire');
+ ok(!$gcf->cleanup, 'cleanup idempotent');
}
use_ok 'PublicInbox::Git', qw(git_unquote git_quote);
--
EW
^ permalink raw reply related [relevance 7%]
* [PATCH 0/5] a few more solver fixups and improvements
@ 2019-01-31 4:27 5% Eric Wong
2019-01-31 4:27 7% ` [PATCH 2/5] inbox: perform cleanup of Git objects for coderepos Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2019-01-31 4:27 UTC (permalink / raw)
To: meta
All going into master and seems to run OK on the main
public-inbox.org server without problems.
Eric Wong (5):
t/config.t: test PublicInbox::Git sharing between inboxes
inbox: perform cleanup of Git objects for coderepos
solvergit: allow searching on longer-than-needed OIDs
solvergit: allow shorter-than-necessary OIDs from user
viewvcs: support streaming large blobs
lib/PublicInbox/Git.pm | 18 ++++++++++++----
lib/PublicInbox/Inbox.pm | 17 +++++++++++++--
lib/PublicInbox/SolverGit.pm | 41 +++++++++++++++++++++++++++++++++---
lib/PublicInbox/ViewVCS.pm | 37 ++++++++++++++++++++++++++++----
t/config.t | 19 +++++++++++++++++
t/git.t | 4 ++++
t/solver_git.t | 9 ++++++++
7 files changed, 132 insertions(+), 13 deletions(-)
--
EW
^ permalink raw reply [relevance 5%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2019-01-31 4:27 5% [PATCH 0/5] a few more solver fixups and improvements Eric Wong
2019-01-31 4:27 7% ` [PATCH 2/5] inbox: perform cleanup of Git objects for coderepos Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).