From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 1B1241F934 for ; Sat, 11 Sep 2021 23:30:47 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 2/2] www: use ->ALL for per-inbox manifest.js.gz, too Date: Sat, 11 Sep 2021 23:30:46 +0000 Message-Id: <20210911233046.23862-3-e@80x24.org> In-Reply-To: <20210911233046.23862-1-e@80x24.org> References: <20210911233046.23862-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: With 11 epochs on LKML, it's the 11 current epoch time goes from around 60ms to around 10ms, so it's a significant improvement. And improve test coverage while we're at it. --- lib/PublicInbox/ManifestJsGz.pm | 14 +++++++------- t/extindex-psgi.t | 20 ++++++++++++++++++-- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm index cde60245..d5048a96 100644 --- a/lib/PublicInbox/ManifestJsGz.pm +++ b/lib/PublicInbox/ManifestJsGz.pm @@ -53,6 +53,7 @@ sub slow_manifest_add ($$) { manifest_add($ctx, $ibx); } }; + warn "E: $@" if $@; } sub eidx_manifest_add ($$$) { @@ -65,6 +66,10 @@ sub eidx_manifest_add ($$$) { } } else { warn "E: `${\$ibx->eidx_key}' not indexed by $ALL->{topdir}\n"; + # do not use slow path for global manifest since + # it can become catastrophically slow. per-inbox manifest + # is not too bad with dozens of epochs, so never fail that: + slow_manifest_add($ctx, $ibx) if $ibx == $ctx->{ibx}; } } @@ -85,12 +90,8 @@ sub response { sub ibx_entry { my ($ctx, $ibx) = @_; my $ALL = $ctx->{www}->{pi_cfg}->ALL; - if ($ALL) { # FIXME: test this in t/ - eidx_manifest_add($ctx, $ALL, $ibx); - } else { + $ALL ? eidx_manifest_add($ctx, $ALL, $ibx) : slow_manifest_add($ctx, $ibx); - warn "E: $@" if $@; - } } sub hide_key { 'manifest' } # for WwwListing->list_match_i @@ -112,8 +113,7 @@ sub psgi_triple { sub per_inbox { my ($ctx) = @_; - # only one inbox, slow is probably OK - slow_manifest_add($ctx, $ctx->{ibx}); + ibx_entry($ctx, $ctx->{ibx}); psgi_triple($ctx); } diff --git a/t/extindex-psgi.t b/t/extindex-psgi.t index 4e26962e..98dc2e48 100644 --- a/t/extindex-psgi.t +++ b/t/extindex-psgi.t @@ -11,6 +11,7 @@ require_git(2.6); require_mods(qw(json DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test URI::Escape Plack::Builder)); use_ok($_) for (qw(HTTP::Request::Common Plack::Test)); +use IO::Uncompress::Gunzip qw(gunzip); require PublicInbox::WWW; my ($ro_home, $cfg_path) = setup_public_inboxes; my ($tmpdir, $for_destroy) = tmpdir; @@ -18,11 +19,11 @@ my $home = "$tmpdir/home"; mkdir $home or BAIL_OUT $!; mkdir "$home/.public-inbox" or BAIL_OUT $!; my $pi_config = "$home/.public-inbox/config"; -cp("$ro_home/.public-inbox/config", $pi_config) or BAIL_OUT; +cp($cfg_path, $pi_config) or BAIL_OUT; my $env = { HOME => $home }; run_script([qw(-extindex --all), "$tmpdir/eidx"], $env) or BAIL_OUT; { - open my $cfgfh, '>', $pi_config or BAIL_OUT; + open my $cfgfh, '>>', $pi_config or BAIL_OUT; $cfgfh->autoflush(1); print $cfgfh <new(PublicInbox::Config->new($pi_config)); @@ -67,6 +69,20 @@ my $client = sub { is($res->code, 404, 'no inboxes matched'); unlike($res->content, qr!no inboxes, yet!, 'we have inboxes, just no matches'); + + my $m = {}; + for my $pfx (qw(/t1 /t2), '') { + $res = $cb->(GET($pfx.'/manifest.js.gz')); + gunzip(\($res->content) => \(my $js)); + $m->{$pfx} = json_utf8->decode($js); + } + is_deeply([sort keys %{$m->{''}}], + [ sort(keys %{$m->{'/t1'}}, keys %{$m->{'/t2'}}) ], + 't1 + t2 = all'); + is_deeply([ sort keys %{$m->{'/t2'}} ], [ '/t2/git/0.git' ], + 't2 manifest'); + is_deeply([ sort keys %{$m->{'/t1'}} ], [ '/t1' ], + 't2 manifest'); }; test_psgi(sub { $www->call(@_) }, $client); %$env = (%$env, TMPDIR => $tmpdir, PI_CONFIG => $pi_config);