* [PATCH v2] newswww: add /$MESSAGE_ID global redirector endpoint
@ 2019-02-01 18:31 4% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2019-02-01 18:31 UTC (permalink / raw)
To: Konstantin Ryabitsev; +Cc: meta
Eric Wong <e@80x24.org> wrote:
> However, my screwing up solver hrefs in the Atom feeds got me
> thinking this can even be a 404 handler at the top level
> (similar to how PublicInbox::NewsWWW works). That would allow
> it to be mapped to any path (or domain) via the PSGI builder
> file...
Or just use NewsWWW, because nntp://<HOSTNAME>/<Message-ID> is valid.
Going to think about it while I eat and do other things, but
will very likely merge it to master, soon.
--------8<-----------
Subject: [PATCH] newswww: add /$MESSAGE_ID global redirector endpoint
This is the fallback for the normal WWW endpoint.
Adding this to the top-level seems to be alright, since lynx and
w3m both understand nntp://<HOSTNAME>/<Message-ID> anyways.
If newsgroup and inbox names conflict, then consider it the
fault of the original sender.
Since NewsWWW is intended to support buggy linkifiers in mail clients,
they can interpret nntp:// URLs as http://<HOSTNAME>/<Message-ID>
Inbox ordering from the config file is preserved since
commit cfa8ff7c256e20f3240aed5f98d155c019788e3b
("config: each_inbox iteration preserves config order"),
so admins can rely on that to configure how scanning
works.
Requested-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
cf. https://public-inbox.org/meta/20190107190719.GE9442@pure.paranoia.local/
nntp://news.public-inbox.org/20190107190719.GE9442@pure.paranoia.local
---
MANIFEST | 1 +
lib/PublicInbox/NewsWWW.pm | 50 ++++++++++++++++++++++-----
t/psgi_scan_all.t | 69 ++++++++++++++++++++++++++++++++++++++
3 files changed, 111 insertions(+), 9 deletions(-)
create mode 100644 t/psgi_scan_all.t
diff --git a/MANIFEST b/MANIFEST
index c4a9349..6ff2bfe 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -208,6 +208,7 @@ t/psgi_attach.t
t/psgi_bad_mids.t
t/psgi_mount.t
t/psgi_multipart_not.t
+t/psgi_scan_all.t
t/psgi_search.t
t/psgi_text.t
t/psgi_v2.t
diff --git a/lib/PublicInbox/NewsWWW.pm b/lib/PublicInbox/NewsWWW.pm
index 01e34d7..d7fcb0d 100644
--- a/lib/PublicInbox/NewsWWW.pm
+++ b/lib/PublicInbox/NewsWWW.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Plack app redirector for mapping /$NEWSGROUP requests to
@@ -17,16 +17,34 @@ sub new {
bless { pi_config => $pi_config }, $class;
}
+sub redirect ($$) {
+ my ($code, $url) = @_;
+ [ $code,
+ [ Location => $url, 'Content-Type' => 'text/plain' ],
+ [ "Redirecting to $url\n" ] ]
+}
+
+sub try_inbox ($$) {
+ my ($ibx, $mid) = @_;
+ # do not pass $env since HTTP_HOST may differ
+ my $url = $ibx->base_url or return;
+
+ eval { $ibx->mm->num_for($mid) } or return;
+
+ # 302 since the same message may show up on
+ # multiple inboxes and inboxes can be added/reordered
+ redirect(302, $url .= mid_escape($mid) . '/');
+}
+
sub call {
my ($self, $env) = @_;
- my $path = $env->{PATH_INFO};
- $path =~ s!\A/+!!;
- $path =~ s!/+\z!!;
# some links may have the article number in them:
# /inbox.foo.bar/123456
- my ($ng, $article) = split(m!/+!, $path, 2);
- if (my $inbox = $self->{pi_config}->lookup_newsgroup($ng)) {
+ my (undef, @parts) = split(m!/!, $env->{PATH_INFO});
+ my ($ng, $article) = @parts;
+ my $pi_config = $self->{pi_config};
+ if (my $inbox = $pi_config->lookup_newsgroup($ng)) {
my $url = PublicInbox::Hval::prurl($env, $inbox->{url});
my $code = 301;
if (defined $article && $article =~ /\A\d+\z/) {
@@ -38,12 +56,26 @@ sub call {
$url .= mid_escape($mid) . '/';
}
}
+ return redirect($code, $url);
+ }
- my $h = [ Location => $url, 'Content-Type' => 'text/plain' ];
+ my $res;
+ my @try = (join('/', @parts));
+
+ # trailing slash is in the rest of our WWW, so maybe some users
+ # will assume it:
+ if ($parts[-1] eq '') {
+ pop @parts;
+ push @try, join('/', @parts);
+ }
- return [ $code, $h, [ "Redirecting to $url\n" ] ]
+ foreach my $mid (@try) {
+ $pi_config->each_inbox(sub {
+ $res ||= try_inbox($_[0], $mid);
+ });
+ last if defined $res;
}
- [ 404, [ 'Content-Type' => 'text/plain' ], [ "404 Not Found\n" ] ];
+ $res || [ 404, [qw(Content-Type text/plain)], ["404 Not Found\n"] ];
}
1;
diff --git a/t/psgi_scan_all.t b/t/psgi_scan_all.t
new file mode 100644
index 0000000..e9c439e
--- /dev/null
+++ b/t/psgi_scan_all.t
@@ -0,0 +1,69 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use Email::MIME;
+use File::Temp qw/tempdir/;
+use PublicInbox::Config;
+my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape Search::Xapian
+ DBD::SQLite);
+foreach my $mod (@mods) {
+ eval "require $mod";
+ plan skip_all => "$mod missing for psgi_scan_all.t" if $@;
+}
+use_ok 'PublicInbox::V2Writable';
+foreach my $mod (@mods) { use_ok $mod; }
+my $tmp = tempdir('pi-scan_all-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my $cfg = {};
+
+foreach my $i (1..2) {
+ my $cfgpfx = "publicinbox.test-$i";
+ my $addr = $cfg->{"$cfgpfx.address"} = "test-$i\@example.com";
+ my $mainrepo = $cfg->{"$cfgpfx.mainrepo"} = "$tmp/$i";
+ $cfg->{"$cfgpfx.url"} = "http://example.com/$i";
+ my $opt = {
+ mainrepo => $mainrepo,
+ name => "test-$i",
+ version => 2,
+ -primary_address => $addr,
+ };
+ my $ibx = PublicInbox::Inbox->new($opt);
+ my $im = PublicInbox::V2Writable->new($ibx, 1);
+ $im->{parallel} = 0;
+ $im->init_inbox(0);
+ my $mime = PublicInbox::MIME->new(<<EOF);
+From: a\@example.com
+To: $addr
+Subject: s$i
+Message-ID: <a-mid-$i\@b>
+Date: Fri, 02 Oct 1993 00:00:00 +0000
+
+hello world
+EOF
+
+ ok($im->add($mime), "added message to $i");
+ $im->done;
+}
+my $config = PublicInbox::Config->new($cfg);
+use_ok 'PublicInbox::WWW';
+my $www = PublicInbox::WWW->new($config);
+
+test_psgi(sub { $www->call(@_) }, sub {
+ my ($cb) = @_;
+ foreach my $i (1..2) {
+ foreach my $end ('', '/') {
+ my $res = $cb->(GET("/a-mid-$i\@b$end"));
+ is($res->code, 302, 'got 302');
+ is($res->header('Location'),
+ "http://example.com/$i/a-mid-$i\@b/",
+ "redirected OK to $i");
+ }
+ }
+ foreach my $x (qw(inv@lid inv@lid/ i/v/a l/i/d/)) {
+ my $res = $cb->(GET("/$x"));
+ is($res->code, 404, "404 on $x");
+ }
+});
+
+done_testing();
--
EW
^ permalink raw reply related [relevance 4%]
* [PATCH 33/37] config: each_inbox iteration preserves config order
2019-01-21 20:52 5% [PATCH 00/37] viewvcs: diff highlighting and more Eric Wong
@ 2019-01-21 20:52 7% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2019-01-21 20:52 UTC (permalink / raw)
To: meta
For cross-inbox Message-ID resolution; having some sort of
stable ordering makes the most sense. Relying on the
order of the config file seems most natural and allows us
to avoid introducing yet another configuration knob.
---
lib/PublicInbox/Config.pm | 34 +++++++++++++++++++++++++---------
t/config.t | 19 +++++++++++++++++++
2 files changed, 44 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index cead7fc..ccfc114 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -90,13 +90,22 @@ sub lookup_name ($$) {
sub each_inbox {
my ($self, $cb) = @_;
- my %seen;
- foreach my $k (keys %$self) {
- $k =~ m!\Apublicinbox\.([^/]+)\.mainrepo\z! or next;
- next if $seen{$1};
- $seen{$1} = 1;
- my $ibx = lookup_name($self, $1) or next;
- $cb->($ibx);
+ if (my $section_order = $self->{-section_order}) {
+ foreach my $section (@$section_order) {
+ next if $section !~ m!\Apublicinbox\.([^/]+)\z!;
+ $self->{"publicinbox.$1.mainrepo"} or next;
+ my $ibx = lookup_name($self, $1) or next;
+ $cb->($ibx);
+ }
+ } else {
+ my %seen;
+ foreach my $k (keys %$self) {
+ $k =~ m!\Apublicinbox\.([^/]+)\.mainrepo\z! or next;
+ next if $seen{$1};
+ $seen{$1} = 1;
+ my $ibx = lookup_name($self, $1) or next;
+ $cb->($ibx);
+ }
}
}
@@ -137,7 +146,7 @@ sub default_file {
sub git_config_dump {
my ($file) = @_;
- my ($in, $out);
+ my (%section_seen, @section_order);
my @cmd = (qw/git config/, "--file=$file", '-l');
my $cmd = join(' ', @cmd);
my $fh = popen_rd(\@cmd) or die "popen_rd failed for $file: $!\n";
@@ -146,8 +155,14 @@ sub git_config_dump {
while (defined(my $line = <$fh>)) {
chomp $line;
my ($k, $v) = split(/=/, $line, 2);
- my $cur = $rv{$k};
+ my ($section) = ($k =~ /\A(\S+)\.[^\.]+\z/);
+ unless (defined $section_seen{$section}) {
+ $section_seen{$section} = 1;
+ push @section_order, $section;
+ }
+
+ my $cur = $rv{$k};
if (defined $cur) {
if (ref($cur) eq "ARRAY") {
push @$cur, $v;
@@ -159,6 +174,7 @@ sub git_config_dump {
}
}
close $fh or die "failed to close ($cmd) pipe: $?";
+ $rv{-section_order} = \@section_order;
\%rv;
}
diff --git a/t/config.t b/t/config.t
index 5f0a95b..7531fd7 100644
--- a/t/config.t
+++ b/t/config.t
@@ -150,4 +150,23 @@ for my $s (@valid) {
ok(PublicInbox::Config::valid_inbox_name($s), "$d name accepted");
}
+{
+ my $f = "$tmpdir/ordered";
+ open my $fh, '>', $f or die "open: $!";
+ my @expect;
+ foreach my $i (0..3) {
+ push @expect, "$i";
+ print $fh <<"" or die "print: $!";
+[publicinbox "$i"]
+ mainrepo = /path/to/$i.git
+ address = $i\@example.com
+
+ }
+ close $fh or die "close: $!";
+ my $cfg = PublicInbox::Config->new($f);
+ my @result;
+ $cfg->each_inbox(sub { push @result, $_[0]->{name} });
+ is_deeply(\@result, \@expect);
+}
+
done_testing();
--
EW
^ permalink raw reply related [relevance 7%]
* [PATCH 00/37] viewvcs: diff highlighting and more
@ 2019-01-21 20:52 5% Eric Wong
2019-01-21 20:52 7% ` [PATCH 33/37] config: each_inbox iteration preserves config order Eric Wong
0 siblings, 1 reply; 3+ results
From: Eric Wong @ 2019-01-21 20:52 UTC (permalink / raw)
To: meta
Still working on VCS integration and I'm not comfortable deploying
this on the main public-inbox.org because of performance/fairness
concerns, yet.
But, perfect is the enemy of good and I figure it's worth
publishing at the moment. It's also on a Tor mirror:
http://hjrcffqmbrq6wope.onion/meta/
http://hjrcffqmbrq6wope.onion/git/
It looks great to me in Netsurf and dillo :>
People with machines powerful enough to run Firefox
(or Tor Browser Bundle) can use "View -> Page Style" to adjust
colors.
Performance considerations:
* diff highlighting alone adds 10-20% overhead to message rendering
Maybe I can speed it up with some less-readable Perl...
* blob reconstruction is horribly unfair to other clients at the
moment. Fixing this is a priority for me.
I haven't hooked up highlight to blob viewing, yet; but that's
coming; too.
Thinking about it more, the blob lookups is so specific to git
that I'm not sure other VCSes can be supported...
The following changes since commit 55db8a2a51c13aec813ac56bbaac1505791fd262:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TODO: autolinkify that(!)
t/git.t: do not pass "-b" to git-repack(1) (2019-01-18 22:00:33 +0000)
are available in the Git repository at:
https://public-inbox.org/ viewvcs
for you to fetch changes up to c440c879d38e67f62bdbb74f616dc84d20899c33:
t/check-www-inbox: trap SIGINT for File::Temp destruction (2019-01-21 06:53:35 +0000)
----------------------------------------------------------------
Eric Wong (37):
view: disable bold in topic display
hval: force monospace for <form> elements, too
t/perf-msgview: add test to check msg_html performance
solver: initial Perl implementation
git: support multiple URL endpoints
git: add git_quote
git: check saves error on disambiguation
solver: various bugfixes and cleanups
view: wire up diff and vcs viewers with solver
git: disable abbreviations with cat-file hints
solver: operate directly on git index
view: enable naming hints for raw blob downloads
git: support 'ambiguous' result from --batch-check
solver: more verbose blob resolution
solver: break up patch application steps
solver: switch patch application to use a callback
solver: simplify control flow for initial loop
solver: break @todo loop into a callback
solver: note the synchronous nature of index preparation
solver: add a TODO note about making this fully evented
view: enforce trailing slash for /$INBOX/$OID/s/ endpoints
solver: restore diagnostics and deal with CRLF
www: admin-configurable CSS via "publicinbox.css"
$INBOX/_/text/color/ and sample user-side CSS
viewdiff: support diff-highlighting w/o coderepo
viewdiff: cleanup state transitions a bit
viewdiff: quote attributes for Atom feed
t/check-www-inbox: use xmlstarlet to validate Atom if available
viewdiff: do not link to 0{7,40} blobs (again)
viewvcs: disable white-space prewrap in blob view
solver: force quoted-printable bodies to LF
solver: remove extra "^index $OID..$OID" line
config: each_inbox iteration preserves config order
t/check-www-inbox: warn on missing Content-Type
highlight: initial wrapper and PSGI service
hval: split out escape sequences to a separate table
t/check-www-inbox: trap SIGINT for File::Temp destruction
Documentation/design_www.txt | 6 +-
MANIFEST | 15 +
Makefile.PL | 3 +
TODO | 2 -
contrib/css/216dark.css | 26 ++
contrib/css/216light.css | 25 ++
contrib/css/README | 41 +++
examples/highlight.psgi | 13 +
examples/public-inbox.psgi | 2 +-
lib/PublicInbox/Config.pm | 96 +++++-
lib/PublicInbox/Git.pm | 87 ++++-
lib/PublicInbox/HlMod.pm | 126 ++++++++
lib/PublicInbox/Hval.pm | 38 +--
lib/PublicInbox/SolverGit.pm | 454 +++++++++++++++++++++++++++
lib/PublicInbox/UserContent.pm | 78 +++++
lib/PublicInbox/View.pm | 51 ++-
lib/PublicInbox/ViewDiff.pm | 161 ++++++++++
lib/PublicInbox/ViewVCS.pm | 110 +++++++
lib/PublicInbox/WWW.pm | 152 ++++++++-
lib/PublicInbox/WwwHighlight.pm | 73 +++++
lib/PublicInbox/WwwStream.pm | 4 +-
lib/PublicInbox/WwwText.pm | 35 +++
script/public-inbox-httpd | 2 +-
t/check-www-inbox.perl | 26 +-
t/config.t | 19 ++
t/git.t | 7 +-
t/hl_mod.t | 54 ++++
t/perf-msgview.t | 50 +++
t/solve/0001-simple-mod.patch | 20 ++
t/solve/0002-rename-with-modifications.patch | 37 +++
t/solver_git.t | 91 ++++++
t/view.t | 2 +
32 files changed, 1841 insertions(+), 65 deletions(-)
create mode 100644 contrib/css/216dark.css
create mode 100644 contrib/css/216light.css
create mode 100644 contrib/css/README
create mode 100644 examples/highlight.psgi
create mode 100644 lib/PublicInbox/HlMod.pm
create mode 100644 lib/PublicInbox/SolverGit.pm
create mode 100644 lib/PublicInbox/UserContent.pm
create mode 100644 lib/PublicInbox/ViewDiff.pm
create mode 100644 lib/PublicInbox/ViewVCS.pm
create mode 100644 lib/PublicInbox/WwwHighlight.pm
create mode 100644 t/hl_mod.t
create mode 100644 t/perf-msgview.t
create mode 100644 t/solve/0001-simple-mod.patch
create mode 100644 t/solve/0002-rename-with-modifications.patch
create mode 100644 t/solver_git.t
Eric Wong (37):
view: disable bold in topic display
hval: force monospace for <form> elements, too
t/perf-msgview: add test to check msg_html performance
solver: initial Perl implementation
git: support multiple URL endpoints
git: add git_quote
git: check saves error on disambiguation
solver: various bugfixes and cleanups
view: wire up diff and vcs viewers with solver
git: disable abbreviations with cat-file hints
solver: operate directly on git index
view: enable naming hints for raw blob downloads
git: support 'ambiguous' result from --batch-check
solver: more verbose blob resolution
solver: break up patch application steps
solver: switch patch application to use a callback
solver: simplify control flow for initial loop
solver: break @todo loop into a callback
solver: note the synchronous nature of index preparation
solver: add a TODO note about making this fully evented
view: enforce trailing slash for /$INBOX/$OID/s/ endpoints
solver: restore diagnostics and deal with CRLF
www: admin-configurable CSS via "publicinbox.css"
$INBOX/_/text/color/ and sample user-side CSS
viewdiff: support diff-highlighting w/o coderepo
viewdiff: cleanup state transitions a bit
viewdiff: quote attributes for Atom feed
t/check-www-inbox: use xmlstarlet to validate Atom if available
viewdiff: do not link to 0{7,40} blobs (again)
viewvcs: disable white-space prewrap in blob view
solver: force quoted-printable bodies to LF
solver: remove extra "^index $OID..$OID" line
config: each_inbox iteration preserves config order
t/check-www-inbox: warn on missing Content-Type
highlight: initial wrapper and PSGI service
hval: split out escape sequences to a separate table
t/check-www-inbox: trap SIGINT for File::Temp destruction
Documentation/design_www.txt | 6 +-
MANIFEST | 15 +
Makefile.PL | 3 +
TODO | 2 -
contrib/css/216dark.css | 26 ++
contrib/css/216light.css | 25 +
contrib/css/README | 41 ++
examples/highlight.psgi | 13 +
examples/public-inbox.psgi | 2 +-
lib/PublicInbox/Config.pm | 96 +++-
lib/PublicInbox/Git.pm | 87 +++-
lib/PublicInbox/HlMod.pm | 126 +++++
lib/PublicInbox/Hval.pm | 38 +-
lib/PublicInbox/SolverGit.pm | 454 +++++++++++++++++++
lib/PublicInbox/UserContent.pm | 78 ++++
lib/PublicInbox/View.pm | 51 ++-
lib/PublicInbox/ViewDiff.pm | 161 +++++++
lib/PublicInbox/ViewVCS.pm | 110 +++++
lib/PublicInbox/WWW.pm | 152 ++++++-
lib/PublicInbox/WwwHighlight.pm | 73 +++
lib/PublicInbox/WwwStream.pm | 4 +-
lib/PublicInbox/WwwText.pm | 35 ++
script/public-inbox-httpd | 2 +-
t/check-www-inbox.perl | 26 +-
t/config.t | 19 +
t/git.t | 7 +-
t/hl_mod.t | 54 +++
t/perf-msgview.t | 50 ++
t/solve/0001-simple-mod.patch | 20 +
t/solve/0002-rename-with-modifications.patch | 37 ++
t/solver_git.t | 91 ++++
t/view.t | 2 +
32 files changed, 1841 insertions(+), 65 deletions(-)
create mode 100644 contrib/css/216dark.css
create mode 100644 contrib/css/216light.css
create mode 100644 contrib/css/README
create mode 100644 examples/highlight.psgi
create mode 100644 lib/PublicInbox/HlMod.pm
create mode 100644 lib/PublicInbox/SolverGit.pm
create mode 100644 lib/PublicInbox/UserContent.pm
create mode 100644 lib/PublicInbox/ViewDiff.pm
create mode 100644 lib/PublicInbox/ViewVCS.pm
create mode 100644 lib/PublicInbox/WwwHighlight.pm
create mode 100644 t/hl_mod.t
create mode 100644 t/perf-msgview.t
create mode 100644 t/solve/0001-simple-mod.patch
create mode 100644 t/solve/0002-rename-with-modifications.patch
create mode 100644 t/solver_git.t
^ permalink raw reply [relevance 5%]
Results 1-3 of 3 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2019-01-09 11:43 [RFC 0/2] support for /~/$MESSAGE_ID endpoint Eric Wong
2019-01-09 11:43 ` [RFC 2/2] www: add /~/$MESSAGE_ID global redirector endpoint Eric Wong
2019-01-27 2:06 ` Eric Wong
2019-01-28 13:50 ` Konstantin Ryabitsev
2019-02-01 9:00 ` Eric Wong
2019-02-01 18:31 4% ` [PATCH v2] newswww: add /$MESSAGE_ID " Eric Wong
2019-01-21 20:52 5% [PATCH 00/37] viewvcs: diff highlighting and more Eric Wong
2019-01-21 20:52 7% ` [PATCH 33/37] config: each_inbox iteration preserves config order Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).