* [PATCH 0/2] www: per-inbox manifest.js.gz improvements
@ 2021-09-11 23:30 Eric Wong
2021-09-11 23:30 ` [PATCH 1/2] manifest.js.gz: avoid long-lived per-epoch cat-file processes Eric Wong
2021-09-11 23:30 ` [PATCH 2/2] www: use ->ALL for per-inbox manifest.js.gz, too Eric Wong
0 siblings, 2 replies; 3+ messages in thread
From: Eric Wong @ 2021-09-11 23:30 UTC (permalink / raw)
To: meta
Speed things up for people wanting to clone a single inbox and
significantly reduce server overheads. [1/2] should be
noticeable to -httpd admins if users are cloning single inboxes.
Eric Wong (2):
manifest.js.gz: avoid long-lived per-epoch cat-file processes
www: use ->ALL for per-inbox manifest.js.gz, too
lib/PublicInbox/Git.pm | 25 +++++--------------------
lib/PublicInbox/ManifestJsGz.pm | 16 ++++++++--------
t/extindex-psgi.t | 20 ++++++++++++++++++--
3 files changed, 31 insertions(+), 30 deletions(-)
^ permalink raw reply [flat|nested] 3+ messages in thread
* [PATCH 1/2] manifest.js.gz: avoid long-lived per-epoch cat-file processes
2021-09-11 23:30 [PATCH 0/2] www: per-inbox manifest.js.gz improvements Eric Wong
@ 2021-09-11 23:30 ` Eric Wong
2021-09-11 23:30 ` [PATCH 2/2] www: use ->ALL for per-inbox manifest.js.gz, too Eric Wong
1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2021-09-11 23:30 UTC (permalink / raw)
To: meta
When generating per-inbox manifests, we were forgetting to
cleanup per-epoch "git cat-file --batch" processes. Our
previous method of generating modified times was also stupidly
inefficient, so replace the pipeline with a single
"git for-each-ref" invocation.
---
lib/PublicInbox/Git.pm | 25 +++++--------------------
lib/PublicInbox/ManifestJsGz.pm | 2 +-
2 files changed, 6 insertions(+), 21 deletions(-)
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index e557f6d6..219a1732 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -16,7 +16,7 @@ use Errno qw(EINTR EAGAIN);
use File::Glob qw(bsd_glob GLOB_NOSORT);
use File::Spec ();
use Time::HiRes qw(stat);
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd spawn);
use PublicInbox::Tmpfile;
use IO::Poll qw(POLLIN);
use Carp qw(croak);
@@ -466,28 +466,13 @@ sub cat_async ($$$;$) {
push(@$inflight, $oid, $cb, $arg);
}
-sub extract_cmt_time {
- my ($bref, undef, undef, undef, $modified) = @_;
-
- if ($$bref =~ /^committer .*?> ([0-9]+) [\+\-]?[0-9]+/sm) {
- my $cmt_time = $1 + 0;
- $$modified = $cmt_time if $cmt_time > $$modified;
- }
-}
-
# returns the modified time of a git repo, same as the "modified" field
# of a grokmirror manifest
sub modified ($) {
- my ($self) = @_;
- my $modified = 0;
- my $fh = popen($self, qw(rev-parse --branches));
- local $/ = "\n";
- while (my $oid = <$fh>) {
- chomp $oid;
- cat_async($self, $oid, \&extract_cmt_time, \$modified);
- }
- cat_async_wait($self);
- $modified || time;
+ # committerdate:unix is git 2.9.4+ (2017-05-05), so using raw instead
+ my $fh = popen($_[0], qw[for-each-ref --sort=-committerdate
+ --format=%(committerdate:raw) --count=1]);
+ (split(/ /, <$fh> // time))[0] + 0; # integerize for JSON
}
# for grokmirror, which doesn't read gitweb.description
diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index 69d81fa1..cde60245 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -27,7 +27,7 @@ sub inject_entry ($$$;$) {
$ctx->{manifest}->{$url_path} = $ent;
}
-sub manifest_add ($$;$$) { # slow path w/o extindex "all"
+sub manifest_add ($$;$$) { # slow path w/o extindex "all" (or per-inbox)
my ($ctx, $ibx, $epoch, $default_desc) = @_;
my $url_path = "/$ibx->{name}";
my $git;
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [PATCH 2/2] www: use ->ALL for per-inbox manifest.js.gz, too
2021-09-11 23:30 [PATCH 0/2] www: per-inbox manifest.js.gz improvements Eric Wong
2021-09-11 23:30 ` [PATCH 1/2] manifest.js.gz: avoid long-lived per-epoch cat-file processes Eric Wong
@ 2021-09-11 23:30 ` Eric Wong
1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2021-09-11 23:30 UTC (permalink / raw)
To: meta
With 11 epochs on LKML, it's the 11 current epoch time goes from
around 60ms to around 10ms, so it's a significant improvement.
And improve test coverage while we're at it.
---
lib/PublicInbox/ManifestJsGz.pm | 14 +++++++-------
t/extindex-psgi.t | 20 ++++++++++++++++++--
2 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index cde60245..d5048a96 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -53,6 +53,7 @@ sub slow_manifest_add ($$) {
manifest_add($ctx, $ibx);
}
};
+ warn "E: $@" if $@;
}
sub eidx_manifest_add ($$$) {
@@ -65,6 +66,10 @@ sub eidx_manifest_add ($$$) {
}
} else {
warn "E: `${\$ibx->eidx_key}' not indexed by $ALL->{topdir}\n";
+ # do not use slow path for global manifest since
+ # it can become catastrophically slow. per-inbox manifest
+ # is not too bad with dozens of epochs, so never fail that:
+ slow_manifest_add($ctx, $ibx) if $ibx == $ctx->{ibx};
}
}
@@ -85,12 +90,8 @@ sub response {
sub ibx_entry {
my ($ctx, $ibx) = @_;
my $ALL = $ctx->{www}->{pi_cfg}->ALL;
- if ($ALL) { # FIXME: test this in t/
- eidx_manifest_add($ctx, $ALL, $ibx);
- } else {
+ $ALL ? eidx_manifest_add($ctx, $ALL, $ibx) :
slow_manifest_add($ctx, $ibx);
- warn "E: $@" if $@;
- }
}
sub hide_key { 'manifest' } # for WwwListing->list_match_i
@@ -112,8 +113,7 @@ sub psgi_triple {
sub per_inbox {
my ($ctx) = @_;
- # only one inbox, slow is probably OK
- slow_manifest_add($ctx, $ctx->{ibx});
+ ibx_entry($ctx, $ctx->{ibx});
psgi_triple($ctx);
}
diff --git a/t/extindex-psgi.t b/t/extindex-psgi.t
index 4e26962e..98dc2e48 100644
--- a/t/extindex-psgi.t
+++ b/t/extindex-psgi.t
@@ -11,6 +11,7 @@ require_git(2.6);
require_mods(qw(json DBD::SQLite Search::Xapian
HTTP::Request::Common Plack::Test URI::Escape Plack::Builder));
use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
+use IO::Uncompress::Gunzip qw(gunzip);
require PublicInbox::WWW;
my ($ro_home, $cfg_path) = setup_public_inboxes;
my ($tmpdir, $for_destroy) = tmpdir;
@@ -18,11 +19,11 @@ my $home = "$tmpdir/home";
mkdir $home or BAIL_OUT $!;
mkdir "$home/.public-inbox" or BAIL_OUT $!;
my $pi_config = "$home/.public-inbox/config";
-cp("$ro_home/.public-inbox/config", $pi_config) or BAIL_OUT;
+cp($cfg_path, $pi_config) or BAIL_OUT;
my $env = { HOME => $home };
run_script([qw(-extindex --all), "$tmpdir/eidx"], $env) or BAIL_OUT;
{
- open my $cfgfh, '>', $pi_config or BAIL_OUT;
+ open my $cfgfh, '>>', $pi_config or BAIL_OUT;
$cfgfh->autoflush(1);
print $cfgfh <<EOM or BAIL_OUT;
[extindex "all"]
@@ -30,6 +31,7 @@ run_script([qw(-extindex --all), "$tmpdir/eidx"], $env) or BAIL_OUT;
url = http://bogus.example.com/all
[publicinbox]
wwwlisting = all
+ grokManifest = all
EOM
}
my $www = PublicInbox::WWW->new(PublicInbox::Config->new($pi_config));
@@ -67,6 +69,20 @@ my $client = sub {
is($res->code, 404, 'no inboxes matched');
unlike($res->content, qr!no inboxes, yet!,
'we have inboxes, just no matches');
+
+ my $m = {};
+ for my $pfx (qw(/t1 /t2), '') {
+ $res = $cb->(GET($pfx.'/manifest.js.gz'));
+ gunzip(\($res->content) => \(my $js));
+ $m->{$pfx} = json_utf8->decode($js);
+ }
+ is_deeply([sort keys %{$m->{''}}],
+ [ sort(keys %{$m->{'/t1'}}, keys %{$m->{'/t2'}}) ],
+ 't1 + t2 = all');
+ is_deeply([ sort keys %{$m->{'/t2'}} ], [ '/t2/git/0.git' ],
+ 't2 manifest');
+ is_deeply([ sort keys %{$m->{'/t1'}} ], [ '/t1' ],
+ 't2 manifest');
};
test_psgi(sub { $www->call(@_) }, $client);
%$env = (%$env, TMPDIR => $tmpdir, PI_CONFIG => $pi_config);
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-09-11 23:30 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-09-11 23:30 [PATCH 0/2] www: per-inbox manifest.js.gz improvements Eric Wong
2021-09-11 23:30 ` [PATCH 1/2] manifest.js.gz: avoid long-lived per-epoch cat-file processes Eric Wong
2021-09-11 23:30 ` [PATCH 2/2] www: use ->ALL for per-inbox manifest.js.gz, too Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).