[PATCH 0/2] mbox improvements

[PATCH 0/2] mbox improvements

[Eric Wong]

1/1 is obvious and should help with caching/mirroring, though
we could also make the gzip header itself consistent.

2/2 is an interesting use of Danga::Socket::write callbacks.
I'm still evaluating different ways to make gigantic responses
saner and I'm not sure if I like the way this is currently
done; but for now it works.

 lib/PublicInbox/Mbox.pm | 83 ++++++++++++++++++++++++++++++++++++++++++++-----
 lib/PublicInbox/WWW.pm  | 11 +++++++
 2 files changed, 86 insertions(+), 8 deletions(-)

[PATCH 1/2] mbox: consistent header order when decompressed

[Eric Wong]

This should make validating the output easier
when testing between different servers.
---
 lib/PublicInbox/Mbox.pm | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index b81ec0a..efb13e5 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -48,12 +48,16 @@ sub emit_msg {
 	my $base = $feed_opts->{url};
 	my $mid = mid_clean($header_obj->header('Message-ID'));
 	$mid = uri_escape_utf8($mid);
-	my %append = (
-		'Archived-At' => "<$base$mid/>",
-		'List-Archive' => "<$base>",
-		'List-Post' => "<mailto:$feed_opts->{id_addr}>",
+	my @append = (
+		'Archived-At', "<$base$mid/>",
+		'List-Archive', "<$base>",
+		'List-Post', "<mailto:$feed_opts->{id_addr}>",
 	);
-	while (my ($k, $v) = each %append) {
+	my $append = '';
+	my $crlf = $simple->crlf;
+	for (my $i = 0; $i < @append; $i += 2) {
+		my $k = $append[$i];
+		my $v = $append[$i + 1];
 		my @v = $header_obj->header($k);
 		foreach (@v) {
 			if ($v eq $_) {
@@ -61,14 +65,14 @@ sub emit_msg {
 				last;
 			}
 		}
-		$header_obj->header_set($k, @v, $v) if defined $v;
+		$append .= "$k: $v$crlf" if defined $v;
 	}
-
 	my $buf = $header_obj->as_string;
 	unless ($buf =~ /\AFrom /) {
 		$fh->write("From mboxrd\@z Thu Jan  1 00:00:00 1970\n");
 	}
-	$fh->write($buf .= $simple->crlf);
+	$append .= $crlf;
+	$fh->write($buf .= $append);
 
 	$buf = $simple->body;
 	$simple->body_set('');

[PATCH 2/2] mbox: support /$INBOX/all.mbox.gz endpoint

[Eric Wong]

Allows easily downloading the entire archive without
special tools.  In any case, it's not yet advertised to via
HTML until we can test it better.  It'll also support range
queries in the future to avoid wasting bandwidth.
---
 lib/PublicInbox/Mbox.pm | 63 +++++++++++++++++++++++++++++++++++++++++++++++++
 lib/PublicInbox/WWW.pm  | 11 +++++++++
 2 files changed, 74 insertions(+)

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index efb13e5..4c4b74f 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -121,6 +121,69 @@ sub emit_mbox {
 	$fh->close;
 }
 
+sub emit_range {
+	my ($ctx, $range) = @_;
+	sub { _emit_range($_[0], $ctx, $range) };
+}
+
+sub _emit_range {
+	my ($res, $ctx, $range) = @_;
+
+	eval { require IO::Compress::Gzip };
+	return need_gzip($res) if $@;
+	my $query;
+	if ($range eq 'all') { # TODO: YYYY[-MM]
+		$query = '';
+	} else {
+		$res->([404, [qw(Content-Type text/plain)], []]);
+		return;
+	}
+
+	# http://www.iana.org/assignments/media-types/application/gzip
+	my $fh = $res->([200, [qw(Content-Type application/gzip)]]);
+	$fh = PublicInbox::MboxGz->new($fh);
+	my $env = $ctx->{cgi}->env;
+	my $srch = $ctx->{srch};
+	my $git = $ctx->{git};
+	my %opts = (offset => 0, asc => 1);
+	my $nr;
+	my $cb = sub {
+		my $res = $srch->query($query, \%opts);
+		my $msgs = $res->{msgs};
+		$nr = scalar @$msgs;
+		while (defined(my $smsg = shift @$msgs)) {
+			my $msg = eval {
+				my $p = 'HEAD:'.mid2path($smsg->mid);
+				Email::Simple->new($git->cat_file($p));
+			};
+			emit_msg($ctx, $fh, $msg) if $msg;
+		}
+
+		$opts{offset} += $nr;
+	};
+
+	$cb->(); # first part is free
+	return $fh->close if $nr == 0;
+
+	if ($env->{'pi-httpd.async'}) {
+		my $io = $env->{'psgix.io'} or die "no IO";
+		my $next;
+		$next = sub {
+			$cb->();
+			if ($nr > 0) {
+				$io->write($next);
+			} else {
+				$next = undef;
+				$fh->close;
+			}
+		};
+		$io->write($next); # Danga::Socket::write
+		return;
+	}
+	$cb->() while ($nr > 0);
+	$fh->close;
+}
+
 sub need_gzip {
 	my $fh = $_[0]->([501, ['Content-Type' => 'text/html']]);
 	my $title = 'gzipped mbox not available';
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 85cb234..51dc3da 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -68,6 +68,8 @@ sub call {
 		my $path = $2;
 		invalid_inbox($self, $ctx, $1) ||
 			serve_git($cgi, $ctx->{git}, $path);
+	} elsif ($path_info =~ m!$INBOX_RE/([\w-]+).mbox\.gz\z!o) {
+		serve_mbox_range($self, $ctx, $1, $2);
 	} elsif ($path_info =~ m!$INBOX_RE/$MID_RE/$END_RE\z!o) {
 		msg_page($self, $ctx, $1, $2, $3);
 
@@ -430,6 +432,15 @@ sub serve_git {
 	PublicInbox::GitHTTPBackend::serve($cgi, $git, $path);
 }
 
+sub serve_mbox_range {
+	my ($self, $ctx, $inbox, $range) = @_;
+	invalid_inbox($self, $ctx, $inbox) || eval {
+		require PublicInbox::Mbox;
+		searcher($ctx);
+		PublicInbox::Mbox::emit_range($ctx, $range);
+	}
+}
+
 sub news_www {
 	my ($self) = @_;
 	my $nw = $self->{news_www};