user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
thread overview below | download mbox.gz: |
* [PATCH] www_listing: fix manifest.js.gz generation with extindex "all"
@ 2021-06-24 12:59  6% Eric Wong
  0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-06-24 12:59 UTC (permalink / raw)
  To: meta

WwwListing and ManifestJsGz may be too different nowadays to
be worth the code sharing between them.

Update some comments and note we still needs better tests :x

Fixes: 520be116e8a686cb ("www_listing: start updating for pagination + search")
---
 lib/PublicInbox/ManifestJsGz.pm | 26 +++++++++++++++++++-------
 lib/PublicInbox/WwwListing.pm   |  3 ++-
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index e7bb0e86..9dc10791 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -13,12 +13,11 @@ use HTTP::Date qw(time2str);
 
 my $json = PublicInbox::Config::json();
 
-# called by WwwListing
 sub url_regexp {
 	my ($ctx) = @_;
 	# grokmirror uses relative paths, so it's domain-dependent
-	# SUPER calls PublicInbox::WwwListing::url_regexp
-	$ctx->SUPER::url_regexp('publicInbox.grokManifest', 'match=domain');
+	# SUPER calls PublicInbox::WwwListing::url_filter
+	($ctx->url_filter('publicInbox.grokManifest', 'match=domain'))[0];
 }
 
 sub inject_entry ($$$;$) {
@@ -29,7 +28,7 @@ sub inject_entry ($$$;$) {
 	$ctx->{manifest}->{$url_path} = $ent;
 }
 
-sub manifest_add ($$;$$) {
+sub manifest_add ($$;$$) { # slow path w/o extindex "all"
 	my ($ctx, $ibx, $epoch, $default_desc) = @_;
 	my $url_path = "/$ibx->{name}";
 	my $git;
@@ -70,20 +69,33 @@ sub eidx_manifest_add ($$$) {
 	}
 }
 
+sub response {
+	my ($class, $ctx) = @_;
+	bless $ctx, $class;
+	my ($re, undef) = $ctx->url_filter;
+	$re // return psgi_triple($ctx);
+	my $iter = PublicInbox::ConfigIter->new($ctx->{www}->{pi_cfg},
+					$ctx->can('list_match_i'), $re, $ctx);
+	sub {
+		$ctx->{-wcb} = $_[0]; # HTTP server callback
+		$ctx->{env}->{'pi-httpd.async'} ?
+				$iter->event_step : $iter->each_section;
+	}
+}
+
 sub ibx_entry {
 	my ($ctx, $ibx) = @_;
 	my $ALL = $ctx->{www}->{pi_cfg}->ALL;
-	if ($ALL) {
+	if ($ALL) { # FIXME: test this in t/
 		eidx_manifest_add($ctx, $ALL, $ibx);
 	} else {
 		slow_manifest_add($ctx, $ibx);
+		warn "E: $@" if $@;
 	}
-	warn "E: $@" if $@;
 }
 
 sub hide_key { 'manifest' } # for WwwListing->list_match_i
 
-# overrides WwwListing->psgi_triple
 sub psgi_triple {
 	my ($ctx) = @_;
 	my $abs2urlpath = delete($ctx->{-abs2urlpath}) // {};
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index eb015742..76c76663 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -90,7 +90,7 @@ sub add_misc_ibx { # MiscSearch->retry_reopen callback
 		$qs = "( $qs ) AND ( $user_query )";
 	}
 	my $mset = $misc->mset($qs, $opt); # sorts by $MODIFIED (mtime)
-	$ctx->{-list} = [];
+	delete $ctx->{-list}; # reset if retried
 	my $pi_cfg = $ctx->{www}->{pi_cfg};
 	for my $mi ($mset->items) {
 		my $doc = $mi->get_document;
@@ -122,6 +122,7 @@ sub response {
 	my ($re, $qs) = $ctx->url_filter;
 	$re // return $ctx->psgi_triple;
 	if (my $ALL = $ctx->{www}->{pi_cfg}->ALL) { # fast path
+		# FIXME: test this in t/
 		$ALL->misc->reopen->retry_reopen(\&add_misc_ibx,
 						$ctx, $re, $qs);
 	} else { # slow path, no [extindex "all"] configured

^ permalink raw reply related	[relevance 6%]

* [PATCH] www: fix manifest.js.gz for default publicInbox.grokManifest
  @ 2021-06-28 21:59  5% ` Eric Wong
  2021-06-30  2:37  7%   ` Kyle Meyer
  0 siblings, 1 reply; 4+ results
From: Eric Wong @ 2021-06-28 21:59 UTC (permalink / raw)
  To: meta; +Cc: Kyle Meyer

ManifestJsGz->response was not invoking the new "url_filter"
method properly.  Furthermore, fix url_filter for returning 404
responses.

Reported-by: Kyle Meyer <kyle@kyleam.com>
Link: https://public-inbox.org/meta/87fsx3128a.fsf@kyleam.com/
Fixes: 520be116e8a686cb ("www_listing: start updating for pagination + search")
---
 lib/PublicInbox/ManifestJsGz.pm |  4 +--
 lib/PublicInbox/WwwListing.pm   |  2 +-
 t/www_listing.t                 | 53 ++++++++++++++++++++-------------
 3 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index 9dc10791..7fee78dd 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -13,11 +13,11 @@ use HTTP::Date qw(time2str);
 
 my $json = PublicInbox::Config::json();
 
-sub url_regexp {
+sub url_filter {
 	my ($ctx) = @_;
 	# grokmirror uses relative paths, so it's domain-dependent
 	# SUPER calls PublicInbox::WwwListing::url_filter
-	($ctx->url_filter('publicInbox.grokManifest', 'match=domain'))[0];
+	$ctx->SUPER::url_filter('publicInbox.grokManifest', 'match=domain');
 }
 
 sub inject_entry ($$$;$) {
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index 76c76663..98a69986 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -55,7 +55,7 @@ again:
 	} elsif ($v eq 'all') {
 		(qr/./, undef);
 	} elsif ($v eq '404') {
-		undef;
+		(undef, undef);
 	} else {
 		warn <<EOF;
 `$v' is not a valid value for `$key'
diff --git a/t/www_listing.t b/t/www_listing.t
index 6a2892de..6b3b408f 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -1,14 +1,15 @@
+#!perl -w
 # Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # manifest.js.gz generation and grok-pull integration test
 use strict;
-use warnings;
+use v5.10.1;
 use Test::More;
 use PublicInbox::Spawn qw(which);
 use PublicInbox::TestCommon;
 use PublicInbox::Import;
-require_mods(qw(json URI::Escape Plack::Builder Digest::SHA
-		IO::Compress::Gzip IO::Uncompress::Gunzip HTTP::Tiny));
+use IO::Uncompress::Gunzip qw(gunzip);
+require_mods(qw(json URI::Escape Plack::Builder Digest::SHA HTTP::Tiny));
 require PublicInbox::WwwListing;
 require PublicInbox::ManifestJsGz;
 use PublicInbox::Config;
@@ -32,21 +33,23 @@ like($bare->manifest_entry->{fingerprint}, qr/\A[a-f0-9]{40}\z/,
 	'got fingerprint with non-empty repo');
 
 sub tiny_test {
-	my ($json, $host, $port) = @_;
-	my $tmp;
+	my ($json, $host, $port, $html) = @_;
+	my ($tmp, $res);
 	my $http = HTTP::Tiny->new;
-	my $res = $http->get("http://$host:$port/");
-	is($res->{status}, 200, 'got HTML listing');
-	like($res->{content}, qr!</html>!si, 'listing looks like HTML');
-
-	$res = $http->get("http://$host:$port/", {'Accept-Encoding'=>'gzip'});
-	is($res->{status}, 200, 'got gzipped HTML listing');
-	IO::Uncompress::Gunzip::gunzip(\(delete $res->{content}) => \$tmp);
-	like($tmp, qr!</html>!si, 'unzipped listing looks like HTML');
-
+	if ($html) {
+		$res = $http->get("http://$host:$port/");
+		is($res->{status}, 200, 'got HTML listing');
+		like($res->{content}, qr!</html>!si, 'listing looks like HTML');
+
+		$res = $http->get("http://$host:$port/",
+				{'Accept-Encoding'=>'gzip'});
+		is($res->{status}, 200, 'got gzipped HTML listing');
+		gunzip(\(delete $res->{content}) => \$tmp);
+		like($tmp, qr!</html>!si, 'unzipped listing looks like HTML');
+	}
 	$res = $http->get("http://$host:$port/manifest.js.gz");
 	is($res->{status}, 200, 'got manifest');
-	IO::Uncompress::Gunzip::gunzip(\(delete $res->{content}) => \$tmp);
+	gunzip(\(delete $res->{content}) => \$tmp);
 	unlike($tmp, qr/"modified":\s*"/, 'modified is an integer');
 	my $manifest = $json->decode($tmp);
 	ok(my $clone = $manifest->{'/alt'}, '/alt in manifest');
@@ -95,10 +98,9 @@ SKIP: {
 		"lorelei \xc4\x80"), 0,
 		'set gitweb user');
 	ok(unlink("$bare->{git_dir}/description"), 'removed bare/description');
-	open $fh, '>', $cfgfile or die;
-	print $fh <<"" or die;
-[publicinbox]
-	wwwlisting = all
+	open $fh, '>', $cfgfile or xbail "open $cfgfile: $!";
+	$fh->autoflush(1);
+	print $fh <<"" or xbail "print $!";
 [publicinbox "bare"]
 	inboxdir = $bare->{git_dir}
 	url = http://$host/bare
@@ -112,13 +114,22 @@ SKIP: {
 	url = http://$host/v2
 	address = v2\@example.com
 
-	close $fh or die;
 	my $env = { PI_CONFIG => $cfgfile };
 	my $cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err" ];
 	$td = start_script($cmd, $env, { 3 => $sock });
-	$sock = undef;
 
+	# default publicinboxGrokManifest match=domain default
 	tiny_test($json, $host, $port);
+	undef $td;
+
+	print $fh <<"" or xbail "print $!";
+[publicinbox]
+	wwwlisting = all
+
+	close $fh or xbail "close $!";
+	$td = start_script($cmd, $env, { 3 => $sock });
+	tiny_test($json, $host, $port, 1);
+	undef $sock;
 
 	my $grok_pull = which('grok-pull') or
 		skip('grok-pull not available', 12);

^ permalink raw reply related	[relevance 5%]

* Re: [PATCH] www: fix manifest.js.gz for default publicInbox.grokManifest
  2021-06-28 21:59  5% ` [PATCH] www: fix manifest.js.gz for default publicInbox.grokManifest Eric Wong
@ 2021-06-30  2:37  7%   ` Kyle Meyer
  0 siblings, 0 replies; 4+ results
From: Kyle Meyer @ 2021-06-30  2:37 UTC (permalink / raw)
  To: Eric Wong; +Cc: meta

Eric Wong writes:

> Fixes: 520be116e8a686cb ("www_listing: start updating for pagination + search")

Confirmed.  Thank you for the quick fix.

^ permalink raw reply	[relevance 7%]

* [SQUASH] fix manifest
  @ 2021-09-16  9:46  7%   ` Eric Wong
  0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-09-16  9:46 UTC (permalink / raw)
  To: meta

Ugh... (mainly posting this trivial patch for SolverGit :P).
---
 MANIFEST | 1 -
 1 file changed, 1 deletion(-)

diff --git a/MANIFEST b/MANIFEST
index 221cb992..9f11f2f9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -235,7 +235,6 @@ lib/PublicInbox/LeiNoteEvent.pm
 lib/PublicInbox/LeiOverview.pm
 lib/PublicInbox/LeiP2q.pm
 lib/PublicInbox/LeiPmdir.pm
-lib/PublicInbox/LeiPruneMailSync.pm
 lib/PublicInbox/LeiQuery.pm
 lib/PublicInbox/LeiRediff.pm
 lib/PublicInbox/LeiRefreshMailSync.pm

^ permalink raw reply related	[relevance 7%]

Results 1-4 of 4 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-06-24 12:59  6% [PATCH] www_listing: fix manifest.js.gz generation with extindex "all" Eric Wong
2021-06-27 20:28     empty /manifest.js.gz response as of 520be116 Kyle Meyer
2021-06-28 21:59  5% ` [PATCH] www: fix manifest.js.gz for default publicInbox.grokManifest Eric Wong
2021-06-30  2:37  7%   ` Kyle Meyer
2021-09-16  9:41     [PATCH 0/3] lei refresh-mail-sync Eric Wong
2021-09-16  9:41     ` [PATCH 2/3] lei refresh-mail-sync: replace prune-mail-sync Eric Wong
2021-09-16  9:46  7%   ` [SQUASH] fix manifest Eric Wong

Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).