* [PATCH] www: fix manifest.js.gz for default publicInbox.grokManifest
@ 2021-06-28 21:59 5% ` Eric Wong
2021-06-30 2:37 7% ` Kyle Meyer
0 siblings, 1 reply; 4+ results
From: Eric Wong @ 2021-06-28 21:59 UTC (permalink / raw)
To: meta; +Cc: Kyle Meyer
ManifestJsGz->response was not invoking the new "url_filter"
method properly. Furthermore, fix url_filter for returning 404
responses.
Reported-by: Kyle Meyer <kyle@kyleam.com>
Link: https://public-inbox.org/meta/87fsx3128a.fsf@kyleam.com/
Fixes: 520be116e8a686cb ("www_listing: start updating for pagination + search")
---
lib/PublicInbox/ManifestJsGz.pm | 4 +--
lib/PublicInbox/WwwListing.pm | 2 +-
t/www_listing.t | 53 ++++++++++++++++++++-------------
3 files changed, 35 insertions(+), 24 deletions(-)
diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index 9dc10791..7fee78dd 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -13,11 +13,11 @@ use HTTP::Date qw(time2str);
my $json = PublicInbox::Config::json();
-sub url_regexp {
+sub url_filter {
my ($ctx) = @_;
# grokmirror uses relative paths, so it's domain-dependent
# SUPER calls PublicInbox::WwwListing::url_filter
- ($ctx->url_filter('publicInbox.grokManifest', 'match=domain'))[0];
+ $ctx->SUPER::url_filter('publicInbox.grokManifest', 'match=domain');
}
sub inject_entry ($$$;$) {
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index 76c76663..98a69986 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -55,7 +55,7 @@ again:
} elsif ($v eq 'all') {
(qr/./, undef);
} elsif ($v eq '404') {
- undef;
+ (undef, undef);
} else {
warn <<EOF;
`$v' is not a valid value for `$key'
diff --git a/t/www_listing.t b/t/www_listing.t
index 6a2892de..6b3b408f 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -1,14 +1,15 @@
+#!perl -w
# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# manifest.js.gz generation and grok-pull integration test
use strict;
-use warnings;
+use v5.10.1;
use Test::More;
use PublicInbox::Spawn qw(which);
use PublicInbox::TestCommon;
use PublicInbox::Import;
-require_mods(qw(json URI::Escape Plack::Builder Digest::SHA
- IO::Compress::Gzip IO::Uncompress::Gunzip HTTP::Tiny));
+use IO::Uncompress::Gunzip qw(gunzip);
+require_mods(qw(json URI::Escape Plack::Builder Digest::SHA HTTP::Tiny));
require PublicInbox::WwwListing;
require PublicInbox::ManifestJsGz;
use PublicInbox::Config;
@@ -32,21 +33,23 @@ like($bare->manifest_entry->{fingerprint}, qr/\A[a-f0-9]{40}\z/,
'got fingerprint with non-empty repo');
sub tiny_test {
- my ($json, $host, $port) = @_;
- my $tmp;
+ my ($json, $host, $port, $html) = @_;
+ my ($tmp, $res);
my $http = HTTP::Tiny->new;
- my $res = $http->get("http://$host:$port/");
- is($res->{status}, 200, 'got HTML listing');
- like($res->{content}, qr!</html>!si, 'listing looks like HTML');
-
- $res = $http->get("http://$host:$port/", {'Accept-Encoding'=>'gzip'});
- is($res->{status}, 200, 'got gzipped HTML listing');
- IO::Uncompress::Gunzip::gunzip(\(delete $res->{content}) => \$tmp);
- like($tmp, qr!</html>!si, 'unzipped listing looks like HTML');
-
+ if ($html) {
+ $res = $http->get("http://$host:$port/");
+ is($res->{status}, 200, 'got HTML listing');
+ like($res->{content}, qr!</html>!si, 'listing looks like HTML');
+
+ $res = $http->get("http://$host:$port/",
+ {'Accept-Encoding'=>'gzip'});
+ is($res->{status}, 200, 'got gzipped HTML listing');
+ gunzip(\(delete $res->{content}) => \$tmp);
+ like($tmp, qr!</html>!si, 'unzipped listing looks like HTML');
+ }
$res = $http->get("http://$host:$port/manifest.js.gz");
is($res->{status}, 200, 'got manifest');
- IO::Uncompress::Gunzip::gunzip(\(delete $res->{content}) => \$tmp);
+ gunzip(\(delete $res->{content}) => \$tmp);
unlike($tmp, qr/"modified":\s*"/, 'modified is an integer');
my $manifest = $json->decode($tmp);
ok(my $clone = $manifest->{'/alt'}, '/alt in manifest');
@@ -95,10 +98,9 @@ SKIP: {
"lorelei \xc4\x80"), 0,
'set gitweb user');
ok(unlink("$bare->{git_dir}/description"), 'removed bare/description');
- open $fh, '>', $cfgfile or die;
- print $fh <<"" or die;
-[publicinbox]
- wwwlisting = all
+ open $fh, '>', $cfgfile or xbail "open $cfgfile: $!";
+ $fh->autoflush(1);
+ print $fh <<"" or xbail "print $!";
[publicinbox "bare"]
inboxdir = $bare->{git_dir}
url = http://$host/bare
@@ -112,13 +114,22 @@ SKIP: {
url = http://$host/v2
address = v2\@example.com
- close $fh or die;
my $env = { PI_CONFIG => $cfgfile };
my $cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err" ];
$td = start_script($cmd, $env, { 3 => $sock });
- $sock = undef;
+ # default publicinboxGrokManifest match=domain default
tiny_test($json, $host, $port);
+ undef $td;
+
+ print $fh <<"" or xbail "print $!";
+[publicinbox]
+ wwwlisting = all
+
+ close $fh or xbail "close $!";
+ $td = start_script($cmd, $env, { 3 => $sock });
+ tiny_test($json, $host, $port, 1);
+ undef $sock;
my $grok_pull = which('grok-pull') or
skip('grok-pull not available', 12);
^ permalink raw reply related [relevance 5%]
* [PATCH] www_listing: fix manifest.js.gz generation with extindex "all"
@ 2021-06-24 12:59 6% Eric Wong
0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-06-24 12:59 UTC (permalink / raw)
To: meta
WwwListing and ManifestJsGz may be too different nowadays to
be worth the code sharing between them.
Update some comments and note we still needs better tests :x
Fixes: 520be116e8a686cb ("www_listing: start updating for pagination + search")
---
lib/PublicInbox/ManifestJsGz.pm | 26 +++++++++++++++++++-------
lib/PublicInbox/WwwListing.pm | 3 ++-
2 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index e7bb0e86..9dc10791 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -13,12 +13,11 @@ use HTTP::Date qw(time2str);
my $json = PublicInbox::Config::json();
-# called by WwwListing
sub url_regexp {
my ($ctx) = @_;
# grokmirror uses relative paths, so it's domain-dependent
- # SUPER calls PublicInbox::WwwListing::url_regexp
- $ctx->SUPER::url_regexp('publicInbox.grokManifest', 'match=domain');
+ # SUPER calls PublicInbox::WwwListing::url_filter
+ ($ctx->url_filter('publicInbox.grokManifest', 'match=domain'))[0];
}
sub inject_entry ($$$;$) {
@@ -29,7 +28,7 @@ sub inject_entry ($$$;$) {
$ctx->{manifest}->{$url_path} = $ent;
}
-sub manifest_add ($$;$$) {
+sub manifest_add ($$;$$) { # slow path w/o extindex "all"
my ($ctx, $ibx, $epoch, $default_desc) = @_;
my $url_path = "/$ibx->{name}";
my $git;
@@ -70,20 +69,33 @@ sub eidx_manifest_add ($$$) {
}
}
+sub response {
+ my ($class, $ctx) = @_;
+ bless $ctx, $class;
+ my ($re, undef) = $ctx->url_filter;
+ $re // return psgi_triple($ctx);
+ my $iter = PublicInbox::ConfigIter->new($ctx->{www}->{pi_cfg},
+ $ctx->can('list_match_i'), $re, $ctx);
+ sub {
+ $ctx->{-wcb} = $_[0]; # HTTP server callback
+ $ctx->{env}->{'pi-httpd.async'} ?
+ $iter->event_step : $iter->each_section;
+ }
+}
+
sub ibx_entry {
my ($ctx, $ibx) = @_;
my $ALL = $ctx->{www}->{pi_cfg}->ALL;
- if ($ALL) {
+ if ($ALL) { # FIXME: test this in t/
eidx_manifest_add($ctx, $ALL, $ibx);
} else {
slow_manifest_add($ctx, $ibx);
+ warn "E: $@" if $@;
}
- warn "E: $@" if $@;
}
sub hide_key { 'manifest' } # for WwwListing->list_match_i
-# overrides WwwListing->psgi_triple
sub psgi_triple {
my ($ctx) = @_;
my $abs2urlpath = delete($ctx->{-abs2urlpath}) // {};
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index eb015742..76c76663 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -90,7 +90,7 @@ sub add_misc_ibx { # MiscSearch->retry_reopen callback
$qs = "( $qs ) AND ( $user_query )";
}
my $mset = $misc->mset($qs, $opt); # sorts by $MODIFIED (mtime)
- $ctx->{-list} = [];
+ delete $ctx->{-list}; # reset if retried
my $pi_cfg = $ctx->{www}->{pi_cfg};
for my $mi ($mset->items) {
my $doc = $mi->get_document;
@@ -122,6 +122,7 @@ sub response {
my ($re, $qs) = $ctx->url_filter;
$re // return $ctx->psgi_triple;
if (my $ALL = $ctx->{www}->{pi_cfg}->ALL) { # fast path
+ # FIXME: test this in t/
$ALL->misc->reopen->retry_reopen(\&add_misc_ibx,
$ctx, $re, $qs);
} else { # slow path, no [extindex "all"] configured
^ permalink raw reply related [relevance 6%]
* Re: [PATCH] www: fix manifest.js.gz for default publicInbox.grokManifest
2021-06-28 21:59 5% ` [PATCH] www: fix manifest.js.gz for default publicInbox.grokManifest Eric Wong
@ 2021-06-30 2:37 7% ` Kyle Meyer
0 siblings, 0 replies; 4+ results
From: Kyle Meyer @ 2021-06-30 2:37 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Eric Wong writes:
> Fixes: 520be116e8a686cb ("www_listing: start updating for pagination + search")
Confirmed. Thank you for the quick fix.
^ permalink raw reply [relevance 7%]
* [SQUASH] fix manifest
@ 2021-09-16 9:46 7% ` Eric Wong
0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-09-16 9:46 UTC (permalink / raw)
To: meta
Ugh... (mainly posting this trivial patch for SolverGit :P).
---
MANIFEST | 1 -
1 file changed, 1 deletion(-)
diff --git a/MANIFEST b/MANIFEST
index 221cb992..9f11f2f9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -235,7 +235,6 @@ lib/PublicInbox/LeiNoteEvent.pm
lib/PublicInbox/LeiOverview.pm
lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiPmdir.pm
-lib/PublicInbox/LeiPruneMailSync.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiRediff.pm
lib/PublicInbox/LeiRefreshMailSync.pm
^ permalink raw reply related [relevance 7%]
Results 1-4 of 4 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-06-24 12:59 6% [PATCH] www_listing: fix manifest.js.gz generation with extindex "all" Eric Wong
2021-06-27 20:28 empty /manifest.js.gz response as of 520be116 Kyle Meyer
2021-06-28 21:59 5% ` [PATCH] www: fix manifest.js.gz for default publicInbox.grokManifest Eric Wong
2021-06-30 2:37 7% ` Kyle Meyer
2021-09-16 9:41 [PATCH 0/3] lei refresh-mail-sync Eric Wong
2021-09-16 9:41 ` [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
2021-09-16 9:46 7% ` [SQUASH] fix manifest Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).