From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <e@80x24.org>
X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net
X-Spam-Level: 
X-Spam-ASN:  
X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00
	shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2
Received: from localhost (dcvr.yhbt.net [127.0.0.1])
	by dcvr.yhbt.net (Postfix) with ESMTP id 157861F8C6
	for <meta@public-inbox.org>; Thu, 24 Jun 2021 12:59:09 +0000 (UTC)
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH] www_listing: fix manifest.js.gz generation with extindex "all"
Date: Thu, 24 Jun 2021 12:59:09 +0000
Message-Id: <20210624125909.10938-1-e@80x24.org>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
List-Id: <meta.public-inbox.org>

WwwListing and ManifestJsGz may be too different nowadays to
be worth the code sharing between them.

Update some comments and note we still needs better tests :x

Fixes: 520be116e8a686cb ("www_listing: start updating for pagination + search")
---
 lib/PublicInbox/ManifestJsGz.pm | 26 +++++++++++++++++++-------
 lib/PublicInbox/WwwListing.pm   |  3 ++-
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index e7bb0e86..9dc10791 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -13,12 +13,11 @@ use HTTP::Date qw(time2str);
 
 my $json = PublicInbox::Config::json();
 
-# called by WwwListing
 sub url_regexp {
 	my ($ctx) = @_;
 	# grokmirror uses relative paths, so it's domain-dependent
-	# SUPER calls PublicInbox::WwwListing::url_regexp
-	$ctx->SUPER::url_regexp('publicInbox.grokManifest', 'match=domain');
+	# SUPER calls PublicInbox::WwwListing::url_filter
+	($ctx->url_filter('publicInbox.grokManifest', 'match=domain'))[0];
 }
 
 sub inject_entry ($$$;$) {
@@ -29,7 +28,7 @@ sub inject_entry ($$$;$) {
 	$ctx->{manifest}->{$url_path} = $ent;
 }
 
-sub manifest_add ($$;$$) {
+sub manifest_add ($$;$$) { # slow path w/o extindex "all"
 	my ($ctx, $ibx, $epoch, $default_desc) = @_;
 	my $url_path = "/$ibx->{name}";
 	my $git;
@@ -70,20 +69,33 @@ sub eidx_manifest_add ($$$) {
 	}
 }
 
+sub response {
+	my ($class, $ctx) = @_;
+	bless $ctx, $class;
+	my ($re, undef) = $ctx->url_filter;
+	$re // return psgi_triple($ctx);
+	my $iter = PublicInbox::ConfigIter->new($ctx->{www}->{pi_cfg},
+					$ctx->can('list_match_i'), $re, $ctx);
+	sub {
+		$ctx->{-wcb} = $_[0]; # HTTP server callback
+		$ctx->{env}->{'pi-httpd.async'} ?
+				$iter->event_step : $iter->each_section;
+	}
+}
+
 sub ibx_entry {
 	my ($ctx, $ibx) = @_;
 	my $ALL = $ctx->{www}->{pi_cfg}->ALL;
-	if ($ALL) {
+	if ($ALL) { # FIXME: test this in t/
 		eidx_manifest_add($ctx, $ALL, $ibx);
 	} else {
 		slow_manifest_add($ctx, $ibx);
+		warn "E: $@" if $@;
 	}
-	warn "E: $@" if $@;
 }
 
 sub hide_key { 'manifest' } # for WwwListing->list_match_i
 
-# overrides WwwListing->psgi_triple
 sub psgi_triple {
 	my ($ctx) = @_;
 	my $abs2urlpath = delete($ctx->{-abs2urlpath}) // {};
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index eb015742..76c76663 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -90,7 +90,7 @@ sub add_misc_ibx { # MiscSearch->retry_reopen callback
 		$qs = "( $qs ) AND ( $user_query )";
 	}
 	my $mset = $misc->mset($qs, $opt); # sorts by $MODIFIED (mtime)
-	$ctx->{-list} = [];
+	delete $ctx->{-list}; # reset if retried
 	my $pi_cfg = $ctx->{www}->{pi_cfg};
 	for my $mi ($mset->items) {
 		my $doc = $mi->get_document;
@@ -122,6 +122,7 @@ sub response {
 	my ($re, $qs) = $ctx->url_filter;
 	$re // return $ctx->psgi_triple;
 	if (my $ALL = $ctx->{www}->{pi_cfg}->ALL) { # fast path
+		# FIXME: test this in t/
 		$ALL->misc->reopen->retry_reopen(\&add_misc_ibx,
 						$ctx, $re, $qs);
 	} else { # slow path, no [extindex "all"] configured