user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH] lei_input: support compressed mboxes
Date: Mon, 29 Mar 2021 17:49:42 +0000	[thread overview]
Message-ID: <20210329174942.26212-1-e@80x24.org> (raw)

Since "lei q" and "lei convert" already support writing these
compressed inboxes, it makes sense that all mbox readers support
them, as well.

Using compression is one reliable way to know an mboxrd or mboxo
hasn't been unexpectedly truncated.
---
 lib/PublicInbox/LeiConvert.pm |  1 +
 lib/PublicInbox/LeiInput.pm   |  7 ++++
 lib/PublicInbox/LeiToMail.pm  | 62 +++--------------------------------
 lib/PublicInbox/MboxReader.pm | 52 +++++++++++++++++++++++++++++
 t/lei-convert.t               | 20 +++++++++--
 t/lei_to_mail.t               |  4 +--
 6 files changed, 85 insertions(+), 61 deletions(-)

diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index da3b50cc..fb7a2f3b 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -34,6 +34,7 @@ sub input_maildir_cb {
 
 sub do_convert { # via wq_do
 	my ($self) = @_;
+	$PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid
 	for my $input (@{$self->{inputs}}) {
 		$self->input_path_url($input);
 	}
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index c04fc2f8..505b73ff 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -75,6 +75,13 @@ sub input_path_url {
 		my $m = $lei->{opt}->{'lock'} //
 			PublicInbox::MboxLock->defaults;
 		my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+		my $zsfx = PublicInbox::MboxReader::zsfx($input);
+		if ($zsfx) {
+			my $in = delete $mbl->{fh};
+			$mbl->{fh} =
+			     PublicInbox::MboxReader::zsfxcat($in, $zsfx, $lei);
+		}
+		local $PublicInbox::DS::in_loop = 0 if $zsfx; # dwaitpid
 		$self->input_fh($ifmt, $mbl->{fh}, $input, @args);
 	} elsif (-d _ && (-d "$input/cur" || -d "$input/new")) {
 		return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 88468c34..847e5cb9 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -9,7 +9,7 @@ use parent qw(PublicInbox::IPC);
 use PublicInbox::Eml;
 use PublicInbox::Lock;
 use PublicInbox::ProcessPipe;
-use PublicInbox::Spawn qw(which spawn popen_rd);
+use PublicInbox::Spawn qw(spawn);
 use PublicInbox::LeiDedupe;
 use PublicInbox::Git;
 use PublicInbox::GitAsyncCat;
@@ -167,56 +167,10 @@ sub reap_compress { # dwaitpid callback
 	$lei->fail("@$cmd failed", $? >> 8);
 }
 
-# all of these support -c for stdout and -d for decompression,
-# mutt is commonly distributed with hooks for gz, bz2 and xz, at least
-# { foo => '' } means "--foo" is passed to the command-line,
-# otherwise { foo => '--bar' } passes "--bar"
-our %zsfx2cmd = (
-	gz => [ qw(GZIP pigz gzip), { rsyncable => '' } ],
-	bz2 => [ 'bzip2', {} ],
-	xz => [ 'xz', {} ],
-	# XXX does anybody care for these?  I prefer zstd on entire FSes,
-	# so it's probably not necessary on a per-file basis
-	# zst => [ 'zstd', { -default => [ qw(-q) ], # it's noisy by default
-	#	rsyncable => '', threads => '-T' } ],
-	# zz => [ 'pigz', { -default => [ '--zlib' ],
-	#	rsyncable => '', threads => '-p' }],
-	# lzo => [ 'lzop', {} ],
-	# lzma => [ 'lzma', {} ],
-);
-
-sub zsfx2cmd ($$$) {
-	my ($zsfx, $decompress, $lei) = @_;
-	my $x = $zsfx2cmd{$zsfx} // die "no support for suffix=.$zsfx";
-	my @info = @$x;
-	my $cmd_opt = pop @info;
-	my @cmd = (undef, $decompress ? qw(-dc) : qw(-c));
-	for my $exe (@info) {
-		# I think respecting client's ENV{GZIP} is OK, not sure
-		# about ENV overrides for other, less-common compressors
-		if ($exe eq uc($exe)) {
-			$exe = $lei->{env}->{$exe} or next;
-		}
-		$cmd[0] = which($exe) and last;
-	}
-	$cmd[0] // die join(' or ', @info)." missing for .$zsfx";
-	# push @cmd, @{$cmd_opt->{-default}} if $cmd_opt->{-default};
-	for my $bool (qw(rsyncable)) {
-		my $switch = $cmd_opt->{rsyncable} // next;
-		push @cmd, '--'.($switch || $bool);
-	}
-	for my $key (qw(rsyncable)) { # support compression level?
-		my $switch = $cmd_opt->{$key} // next;
-		my $val = $lei->{opt}->{$key} // next;
-		push @cmd, $switch, $val;
-	}
-	\@cmd;
-}
-
 sub _post_augment_mbox { # open a compressor process
 	my ($self, $lei) = @_;
 	my $zsfx = $self->{zsfx} or return;
-	my $cmd = zsfx2cmd($zsfx, undef, $lei);
+	my $cmd = PublicInbox::MboxReader::zsfx2cmd($zsfx, undef, $lei);
 	my ($r, $w) = @{delete $lei->{zpipe}};
 	my $rdr = { 0 => $r, 1 => $lei->{1}, 2 => $lei->{2} };
 	my $pid = spawn($cmd, undef, $rdr);
@@ -227,12 +181,6 @@ sub _post_augment_mbox { # open a compressor process
 	$lei->{1} = $pp;
 }
 
-sub decompress_src ($$$) {
-	my ($in, $zsfx, $lei) = @_;
-	my $cmd = zsfx2cmd($zsfx, 1, $lei);
-	popen_rd($cmd, undef, { 0 => $in, 2 => $lei->{2} });
-}
-
 sub dup_src ($) {
 	my ($in) = @_;
 	open my $dup, '+>>&', $in or die "dup: $!";
@@ -533,8 +481,7 @@ sub _pre_augment_mbox {
 		die "--augment specified but $dst is not seekable\n" if
 			$lei->{opt}->{augment};
 	}
-	state $zsfx_allow = join('|', keys %zsfx2cmd);
-	if (($self->{zsfx}) = ($dst =~ /\.($zsfx_allow)\z/)) {
+	if ($self->{zsfx} = PublicInbox::MboxReader::zsfx($dst)) {
 		pipe(my ($r, $w)) or die "pipe: $!";
 		$lei->{zpipe} = [ $r, $w ];
 		$lei->{ovv}->{lock_path} and
@@ -559,7 +506,8 @@ sub _do_augment_mbox {
 		return;
 	}
 	my $zsfx = $self->{zsfx};
-	my $rd = $zsfx ? decompress_src($out, $zsfx, $lei) : dup_src($out);
+	my $rd = $zsfx ? PublicInbox::MboxReader::zsfxcat($out, $zsfx, $lei)
+			: dup_src($out);
 	my $dedupe;
 	if ($opt->{augment}) {
 		$dedupe = $lei->{dedupe};
diff --git a/lib/PublicInbox/MboxReader.pm b/lib/PublicInbox/MboxReader.pm
index 6302d1fa..9291f00b 100644
--- a/lib/PublicInbox/MboxReader.pm
+++ b/lib/PublicInbox/MboxReader.pm
@@ -138,4 +138,56 @@ sub reads {
 	$ifmt =~ /\Ambox(?:rd|cl|cl2|o)\z/ ? __PACKAGE__->can($ifmt) : undef
 }
 
+# all of these support -c for stdout and -d for decompression,
+# mutt is commonly distributed with hooks for gz, bz2 and xz, at least
+# { foo => '' } means "--foo" is passed to the command-line,
+# otherwise { foo => '--bar' } passes "--bar"
+my %zsfx2cmd = (
+	gz => [ qw(GZIP pigz gzip), { rsyncable => '' } ],
+	bz2 => [ 'bzip2', {} ],
+	xz => [ 'xz', {} ],
+	# don't add new entries here unless MUA support is widely available
+);
+
+sub zsfx ($) {
+	my ($pathname) = @_;
+	my $allow = join('|', keys %zsfx2cmd);
+	$pathname =~ /\.($allow)\z/ ? $1 : undef;
+}
+
+sub zsfx2cmd ($$$) {
+	my ($zsfx, $decompress, $lei) = @_;
+	my $x = $zsfx2cmd{$zsfx} // die "BUG: no support for suffix=.$zsfx";
+	my @info = @$x;
+	my $cmd_opt = pop @info;
+	my @cmd = (undef, $decompress ? qw(-dc) : qw(-c));
+	require PublicInbox::Spawn;
+	for my $exe (@info) {
+		# I think respecting client's ENV{GZIP} is OK, not sure
+		# about ENV overrides for other, less-common compressors
+		if ($exe eq uc($exe)) {
+			$exe = $lei->{env}->{$exe} or next;
+		}
+		$cmd[0] = PublicInbox::Spawn::which($exe) and last;
+	}
+	$cmd[0] // die join(' or ', @info)." missing for .$zsfx";
+	# push @cmd, @{$cmd_opt->{-default}} if $cmd_opt->{-default};
+	for my $bool (qw(rsyncable)) {
+		my $switch = $cmd_opt->{rsyncable} // next;
+		push @cmd, '--'.($switch || $bool);
+	}
+	for my $key (qw(rsyncable)) { # support compression level?
+		my $switch = $cmd_opt->{$key} // next;
+		my $val = $lei->{opt}->{$key} // next;
+		push @cmd, $switch, $val;
+	}
+	\@cmd;
+}
+
+sub zsfxcat ($$$) {
+	my ($in, $zsfx, $lei) = @_;
+	my $cmd = zsfx2cmd($zsfx, 1, $lei);
+	PublicInbox::Spawn::popen_rd($cmd, undef, { 0 => $in, 2 => $lei->{2} });
+}
+
 1;
diff --git a/t/lei-convert.t b/t/lei-convert.t
index 7ff628f9..dc53b82c 100644
--- a/t/lei-convert.t
+++ b/t/lei-convert.t
@@ -6,6 +6,7 @@ use PublicInbox::MboxReader;
 use PublicInbox::MdirReader;
 use PublicInbox::NetReader;
 use PublicInbox::Eml;
+use IO::Uncompress::Gunzip;
 require_mods(qw(lei -imapd -nntpd Mail::IMAPClient Net::NNTP));
 my ($tmpdir, $for_destroy) = tmpdir;
 my $sock = tcp_server;
@@ -91,13 +92,28 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	@bar = ();
 	PublicInbox::MboxReader->mboxcl2($fh, sub {
 		my $eml = shift;
-		for my $h (qw(Status Content-Length Lines)) {
+		for my $h (qw(Content-Length Lines)) {
 			ok(defined($eml->header_raw($h)),
 				"$h defined for mboxcl2");
 			$eml->header_set($h);
 		}
 		push @bar, $eml;
 	});
-	is_deeply(\@bar, [ eml_load('t/plack-qp.eml') ], 'eml => mboxcl2');
+	my $qp_eml = eml_load('t/plack-qp.eml');
+	$qp_eml->header_set('Status', 'O');
+	is_deeply(\@bar, [ $qp_eml ], 'eml => mboxcl2');
+
+	lei_ok qw(convert t/plack-qp.eml -o), "mboxrd:$d/qp.gz";
+	open $fh, '<', "$d/qp.gz" or xbail $!;
+	ok(-s $fh, 'not empty');
+	$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+	@bar = ();
+	PublicInbox::MboxReader->mboxrd($fh, sub { push @bar, shift });
+	is_deeply(\@bar, [ $qp_eml ], 'wrote gzipped mboxrd');
+	lei_ok qw(convert -o mboxrd:/dev/stdout), "mboxrd:$d/qp.gz";
+	open $fh, '<', \$lei_out or xbail;
+	@bar = ();
+	PublicInbox::MboxReader->mboxrd($fh, sub { push @bar, shift });
+	is_deeply(\@bar, [ $qp_eml ], 'readed gzipped mboxrd');
 });
 done_testing;
diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index 626bdab3..75314add 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -149,8 +149,8 @@ test_lei(sub {
 	is($res->[1], undef, 'only one result');
 });
 
-for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma?
-	my $zsfx2cmd = PublicInbox::LeiToMail->can('zsfx2cmd');
+my $zsfx2cmd = PublicInbox::MboxReader->can('zsfx2cmd');
+for my $zsfx (qw(gz bz2 xz)) {
 	SKIP: {
 		my $cmd = eval { $zsfx2cmd->($zsfx, 0, $lei) };
 		skip $@, 3 if $@;

                 reply	other threads:[~2021-03-29 17:49 UTC|newest]

Thread overview: [no followups] expand[flat|nested]  mbox.gz  Atom feed

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210329174942.26212-1-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH] lei_input: support compressed mboxes' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.mail.public-inbox.meta
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.mail.public-inbox.meta
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git