user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
* [PATCH] lei_input: support PublicInbox::WWW mboxrd URLs
@ 2021-04-26  8:30 Eric Wong
  0 siblings, 0 replies; 2+ messages in thread
From: Eric Wong @ 2021-04-26  8:30 UTC (permalink / raw)
  To: meta

This gives "lei import", "lei tag", and similar commands
the ability to use URLs recognized by our PSGI frontend
directly.

This is more convenient than an equivalent shell pipeline
since "set -o pipefail" is not portable and errors may be
lost.
---
 MANIFEST                    |  1 +
 lib/PublicInbox/LeiInput.pm | 76 +++++++++++++++++++++++++++++++++++++
 t/lei-import-http.t         | 43 +++++++++++++++++++++
 3 files changed, 120 insertions(+)
 create mode 100644 t/lei-import-http.t

diff --git a/MANIFEST b/MANIFEST
index 79d393c5..ce824fcf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -394,6 +394,7 @@ t/kqnotify.t
 t/lei-convert.t
 t/lei-daemon.t
 t/lei-externals.t
+t/lei-import-http.t
 t/lei-import-imap.t
 t/lei-import-maildir.t
 t/lei-import-nntp.t
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index d11d23d4..785e607d 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiInput;
 use strict;
 use v5.10.1;
 use PublicInbox::DS;
+use PublicInbox::Spawn qw(which popen_rd);
 
 # JMAP RFC 8621 4.1.1
 # https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
@@ -77,6 +78,32 @@ error reading $name: $!
 	}
 }
 
+# handles mboxrd endpoints described in Documentation/design_notes.txt
+sub handle_http_input ($$@) {
+	my ($self, $url, @args) = @_;
+	my $lei = $self->{lei} or die 'BUG: {lei} missing';
+	my $curl_opt = delete $self->{"-curl-$url"} or
+				die("BUG: $url curl options not prepared");
+	my $uri = pop @$curl_opt;
+	my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
+	push @$curl, '-s', @$curl_opt;
+	my $cmd = $curl->for_uri($lei, $uri);
+	$lei->qerr("# $cmd");
+	my $rdr = { 2 => $lei->{2}, pgid => 0 };
+	my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
+	grep(/\A--compressed\z/, @$curl) or
+		$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+	eval {
+		PublicInbox::MboxReader->mboxrd($fh,
+						$self->can('input_mbox_cb'),
+						$self, @args);
+	};
+	my $err = $@;
+	waitpid($pid, 0);
+	$? || $err and
+		$lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+}
+
 sub input_path_url {
 	my ($self, $input, @args) = @_;
 	my $lei = $self->{lei};
@@ -92,6 +119,9 @@ sub input_path_url {
 						$self->can('input_net_cb'),
 					$self, @args);
 		return;
+	} elsif ($input =~ m!\Ahttps?://!i) {
+		handle_http_input($self, $input, @args);
+		return;
 	}
 	if ($input =~ s!\A([a-z0-9]+):!!i) {
 		$ifmt = lc($1);
@@ -129,6 +159,50 @@ EOM
 	}
 }
 
+sub bad_http ($$;$) {
+	my ($lei, $url, $alt) = @_;
+	my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
+	$lei->fail("E: <$url> not recognized, $x");
+}
+
+sub prepare_http_input ($$$) {
+	my ($self, $lei, $url) = @_;
+	require URI;
+	require PublicInbox::MboxReader;
+	require PublicInbox::LeiCurl;
+	require IO::Uncompress::Gunzip;
+	$self->{curl} //= which('curl') or
+				return $lei->fail("curl missing for <$url>");
+	my $uri = URI->new($url);
+	my $path = $uri->path;
+	my %qf = $uri->query_form;
+	my @curl_opt;
+	if ($path =~ m!/(?:t\.mbox\.gz|all\.mbox\.gz)\z!) {
+		# OK
+	} elsif ($path =~ m!/raw\z!) {
+		push @curl_opt, '--compressed';
+	# convert search query to mboxrd request since they require POST
+	# this is only intended for PublicInbox::WWW, and will false-positive
+	# on many other search engines... oh well
+	} elsif (defined $qf{'q'}) {
+		$qf{x} = 'm';
+		$uri->query_form(\%qf);
+		push @curl_opt, '-d', '';
+		$$uri ne $url and $lei->qerr(<<"");
+# <$url> rewritten to <$$uri> with HTTP POST
+
+	# try to provide hints for /$INBOX/$MSGID/T/ and /$INBOX/
+	} elsif ($path =~ s!/[tT]/\z!/t.mbox.gz! ||
+			$path =~ s!/t\.atom\z!/t.mbox.gz! ||
+			$path =~ s!/([^/]+\@[^/]+)/\z!/$1/raw!) {
+		$uri->path($path);
+		return bad_http($lei, $url, $$uri);
+	} else {
+		return bad_http($lei, $url);
+	}
+	$self->{"-curl-$url"} = [ @curl_opt, $uri ]; # for handle_http_input
+}
+
 sub prepare_inputs { # returns undef on error
 	my ($self, $lei, $inputs) = @_;
 	my $in_fmt = $lei->{opt}->{'in-format'};
@@ -156,6 +230,8 @@ sub prepare_inputs { # returns undef on error
 					push @{$sync->{no}}, $input;
 				}
 			}
+		} elsif ($input_path =~ m!\Ahttps?://!i) {
+			prepare_http_input($self, $lei, $input_path) or return;
 		} elsif ($input_path =~ s/\A([a-z0-9]+)://is) {
 			my $ifmt = lc $1;
 			if (($in_fmt // $ifmt) ne $ifmt) {
diff --git a/t/lei-import-http.t b/t/lei-import-http.t
new file mode 100644
index 00000000..35cbf369
--- /dev/null
+++ b/t/lei-import-http.t
@@ -0,0 +1,43 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use PublicInbox::Spawn qw(which);
+require_mods(qw(lei -httpd));
+which('curl') or plan skip_all => "curl required for $0";
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my ($tmpdir, $for_destroy) = tmpdir;
+my $sock = tcp_server;
+my $cmd = [ '-httpd', '-W0', "--stdout=$tmpdir/1", "--stderr=$tmpdir/2" ];
+my $env = { PI_CONFIG => $cfg_path };
+my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-httpd $?");
+my $host_port = tcp_host_port($sock);
+undef $sock;
+test_lei({ tmpdir => $tmpdir }, sub {
+	my $url = "http://$host_port/t2";
+	for my $p (qw(bogus@x/t.mbox.gz bogus@x/raw ?q=noresultever)) {
+		ok(!lei('import', "$url/$p"), "/$p fails properly");
+	}
+	for my $p (qw(/ /T/ /t/ /t.atom)) {
+		ok(!lei('import', "$url/m\@example$p"), "/$p fails");
+		like($lei_err, qr/did you mean/, "gave hint for $p");
+	}
+	lei_ok 'import', "$url/testmessage\@example.com/raw";
+	lei_ok 'q', 'm:testmessage@example.com';
+	my $res = json_utf8->decode($lei_out);
+	is($res->[0]->{'m'}, 'testmessage@example.com', 'imported raw')
+		or diag explain($res);
+
+	lei_ok 'import', "$url/qp\@example.com/t.mbox.gz";
+	lei_ok 'q', 'm:qp@example.com';
+	$res = json_utf8->decode($lei_out);
+	is($res->[0]->{'m'}, 'qp@example.com', 'imported t.mbox.gz')
+		or diag explain($res);
+
+	lei_ok 'import', "$url/?q=s:boolean";
+	lei_ok 'q', 'm:20180720072141.GA15957@example';
+	$res = json_utf8->decode($lei_out);
+	is($res->[0]->{'m'}, '20180720072141.GA15957@example',
+			'imported search result') or diag explain($res);
+});
+done_testing;

^ permalink raw reply	[flat|nested] 2+ messages in thread

* [PATCH] lei_input: support PublicInbox::WWW mboxrd URLs
@ 2021-04-26  8:27 Eric Wong
  0 siblings, 0 replies; 2+ messages in thread
From: Eric Wong @ 2021-04-26  8:27 UTC (permalink / raw)
  To: meta

This gives "lei import", "lei tag", and similar commands
the ability to use URLs recognized by our PSGI frontend
directly.

This is more convenient than an equivalent shell pipeline
since "set -o pipefail" is not portable and errors may be
lost.
---
 MANIFEST                    |  1 +
 lib/PublicInbox/LeiInput.pm | 76 +++++++++++++++++++++++++++++++++++++
 t/lei-import-http.t         | 43 +++++++++++++++++++++
 3 files changed, 120 insertions(+)
 create mode 100644 t/lei-import-http.t

diff --git a/MANIFEST b/MANIFEST
index 79d393c5..ce824fcf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -394,6 +394,7 @@ t/kqnotify.t
 t/lei-convert.t
 t/lei-daemon.t
 t/lei-externals.t
+t/lei-import-http.t
 t/lei-import-imap.t
 t/lei-import-maildir.t
 t/lei-import-nntp.t
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index d11d23d4..785e607d 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiInput;
 use strict;
 use v5.10.1;
 use PublicInbox::DS;
+use PublicInbox::Spawn qw(which popen_rd);
 
 # JMAP RFC 8621 4.1.1
 # https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
@@ -77,6 +78,32 @@ error reading $name: $!
 	}
 }
 
+# handles mboxrd endpoints described in Documentation/design_notes.txt
+sub handle_http_input ($$@) {
+	my ($self, $url, @args) = @_;
+	my $lei = $self->{lei} or die 'BUG: {lei} missing';
+	my $curl_opt = delete $self->{"-curl-$url"} or
+				die("BUG: $url curl options not prepared");
+	my $uri = pop @$curl_opt;
+	my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return;
+	push @$curl, '-s', @$curl_opt;
+	my $cmd = $curl->for_uri($lei, $uri);
+	$lei->qerr("# $cmd");
+	my $rdr = { 2 => $lei->{2}, pgid => 0 };
+	my ($fh, $pid) = popen_rd($cmd, undef, $rdr);
+	grep(/\A--compressed\z/, @$curl) or
+		$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
+	eval {
+		PublicInbox::MboxReader->mboxrd($fh,
+						$self->can('input_mbox_cb'),
+						$self, @args);
+	};
+	my $err = $@;
+	waitpid($pid, 0);
+	$? || $err and
+		$lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+}
+
 sub input_path_url {
 	my ($self, $input, @args) = @_;
 	my $lei = $self->{lei};
@@ -92,6 +119,9 @@ sub input_path_url {
 						$self->can('input_net_cb'),
 					$self, @args);
 		return;
+	} elsif ($input =~ m!\Ahttps?://!i) {
+		handle_http_input($self, $input, @args);
+		return;
 	}
 	if ($input =~ s!\A([a-z0-9]+):!!i) {
 		$ifmt = lc($1);
@@ -129,6 +159,50 @@ EOM
 	}
 }
 
+sub bad_http ($$;$) {
+	my ($lei, $url, $alt) = @_;
+	my $x = $alt ? "did you mean <$alt>?" : 'download and import manually';
+	$lei->fail("E: <$url> not recognized, $x");
+}
+
+sub prepare_http_input ($$$) {
+	my ($self, $lei, $url) = @_;
+	require URI;
+	require PublicInbox::MboxReader;
+	require PublicInbox::LeiCurl;
+	require IO::Uncompress::Gunzip;
+	$self->{curl} //= which('curl') or
+				return $lei->fail("curl missing for <$url>");
+	my $uri = URI->new($url);
+	my $path = $uri->path;
+	my %qf = $uri->query_form;
+	my @curl_opt;
+	if ($path =~ m!/(?:t\.mbox\.gz|all\.mbox\.gz)\z!) {
+		# OK
+	} elsif ($path =~ m!/raw\z!) {
+		push @curl_opt, '--compressed';
+	# convert search query to mboxrd request since they require POST
+	# this is only intended for PublicInbox::WWW, and will false-positive
+	# on many other search engines... oh well
+	} elsif (defined $qf{'q'}) {
+		$qf{x} = 'm';
+		$uri->query_form(\%qf);
+		push @curl_opt, '-d', '';
+		$$uri ne $url and $lei->qerr(<<"");
+# <$url> rewritten to <$$uri> with HTTP POST
+
+	# try to provide hints for /$INBOX/$MSGID/T/ and /$INBOX/
+	} elsif ($path =~ s!/[tT]/\z!/t.mbox.gz! ||
+			$path =~ s!/t\.atom\z!/t.mbox.gz! ||
+			$path =~ s!/([^/]+\@[^/]+)/\z!/$1/raw!) {
+		$uri->path($path);
+		return bad_http($lei, $url, $$uri);
+	} else {
+		return bad_http($lei, $url);
+	}
+	$self->{"-curl-$url"} = [ @curl_opt, $uri ]; # for handle_http_input
+}
+
 sub prepare_inputs { # returns undef on error
 	my ($self, $lei, $inputs) = @_;
 	my $in_fmt = $lei->{opt}->{'in-format'};
@@ -156,6 +230,8 @@ sub prepare_inputs { # returns undef on error
 					push @{$sync->{no}}, $input;
 				}
 			}
+		} elsif ($input_path =~ m!\Ahttps?://!i) {
+			prepare_http_input($self, $lei, $input_path) or return;
 		} elsif ($input_path =~ s/\A([a-z0-9]+)://is) {
 			my $ifmt = lc $1;
 			if (($in_fmt // $ifmt) ne $ifmt) {
diff --git a/t/lei-import-http.t b/t/lei-import-http.t
new file mode 100644
index 00000000..35cbf369
--- /dev/null
+++ b/t/lei-import-http.t
@@ -0,0 +1,43 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+use PublicInbox::Spawn qw(which);
+require_mods(qw(lei -httpd));
+which('curl') or plan skip_all => "curl required for $0";
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+my ($tmpdir, $for_destroy) = tmpdir;
+my $sock = tcp_server;
+my $cmd = [ '-httpd', '-W0', "--stdout=$tmpdir/1", "--stderr=$tmpdir/2" ];
+my $env = { PI_CONFIG => $cfg_path };
+my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-httpd $?");
+my $host_port = tcp_host_port($sock);
+undef $sock;
+test_lei({ tmpdir => $tmpdir }, sub {
+	my $url = "http://$host_port/t2";
+	for my $p (qw(bogus@x/t.mbox.gz bogus@x/raw ?q=noresultever)) {
+		ok(!lei('import', "$url/$p"), "/$p fails properly");
+	}
+	for my $p (qw(/ /T/ /t/ /t.atom)) {
+		ok(!lei('import', "$url/m\@example$p"), "/$p fails");
+		like($lei_err, qr/did you mean/, "gave hint for $p");
+	}
+	lei_ok 'import', "$url/testmessage\@example.com/raw";
+	lei_ok 'q', 'm:testmessage@example.com';
+	my $res = json_utf8->decode($lei_out);
+	is($res->[0]->{'m'}, 'testmessage@example.com', 'imported raw')
+		or diag explain($res);
+
+	lei_ok 'import', "$url/qp\@example.com/t.mbox.gz";
+	lei_ok 'q', 'm:qp@example.com';
+	$res = json_utf8->decode($lei_out);
+	is($res->[0]->{'m'}, 'qp@example.com', 'imported t.mbox.gz')
+		or diag explain($res);
+
+	lei_ok 'import', "$url/?q=s:boolean";
+	lei_ok 'q', 'm:20180720072141.GA15957@example';
+	$res = json_utf8->decode($lei_out);
+	is($res->[0]->{'m'}, '20180720072141.GA15957@example',
+			'imported search result') or diag explain($res);
+});
+done_testing;

^ permalink raw reply	[flat|nested] 2+ messages in thread

end of thread, other threads:[~2021-04-26  8:30 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-04-26  8:30 [PATCH] lei_input: support PublicInbox::WWW mboxrd URLs Eric Wong
  -- strict thread matches above, loose matches on Subject: below --
2021-04-26  8:27 Eric Wong

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.mail.public-inbox.meta
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.mail.public-inbox.meta
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git