From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-2.9 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=unavailable version=3.3.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 4C04C20A8A for ; Mon, 2 May 2016 18:01:30 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/5] nntp: append Archived-At and List-Archive headers Date: Mon, 2 May 2016 18:01:23 +0000 Message-Id: <20160502180125.21277-4-e@80x24.org> In-Reply-To: <20160502180125.21277-1-e@80x24.org> References: <20160502180125.21277-1-e@80x24.org> List-Id: For readers using NNTP, we should do our best to advertise the clonable HTTP/HTTPS URLs and the message permalink URL for ease-of-referencing messages, since we don't want the NNTP server and it's sequential article numbers to be relied on. --- lib/PublicInbox/NNTP.pm | 27 +++++++++++++++++++++++++-- lib/PublicInbox/NNTPD.pm | 3 ++- lib/PublicInbox/NewsGroup.pm | 11 ++++++++++- t/nntp.t | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index a4cf25e..3e0faaf 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -15,6 +15,7 @@ use Email::MIME; use Data::Dumper qw(Dumper); use POSIX qw(strftime); use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC); +use URI::Escape qw(uri_escape_utf8); use constant { r501 => '501 command syntax error', r221 => '221 Header follows', @@ -426,6 +427,29 @@ sub cmd_quit ($) { undef; } +sub header_append ($$$) { + my ($hdr, $k, $v) = @_; + my @v = $hdr->header($k); + foreach (@v) { + return if $v eq $_; + } + $hdr->header_set($k, @v, $v); +} + +sub set_nntp_headers { + my ($hdr, $ng, $n, $mid) = @_; + + # clobber some + $hdr->header_set('Newsgroups', $ng->{name}); + $hdr->header_set('Xref', xref($ng, $n)); + header_append($hdr, 'List-Post', "{address}>"); + if (my $url = $ng->{url}) { + $mid = uri_escape_utf8($mid); + header_append($hdr, 'Archived-At', "<$url$mid/>"); + header_append($hdr, 'List-Archive', "<$url>"); + } +} + sub art_lookup ($$$) { my ($self, $art, $set_headers) = @_; my $ng = $self->{ng}; @@ -468,8 +492,7 @@ found: return $err unless $s; my $lines; if ($set_headers) { - $s->header_set('Newsgroups', $ng->{name}); - $s->header_set('Xref', xref($ng, $n)); + set_nntp_headers($s->header_obj, $ng, $n, $mid); $lines = $s->body =~ tr!\n!\n!; # must be last diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm index 85109ea..2c84fb3 100644 --- a/lib/PublicInbox/NNTPD.pm +++ b/lib/PublicInbox/NNTPD.pm @@ -30,11 +30,12 @@ sub refresh_groups () { my $git_dir = $pi_config->{$k}; my $addr = $pi_config->{"publicinbox.$g.address"}; my $ngname = $pi_config->{"publicinbox.$g.newsgroup"}; + my $url = $pi_config->{"publicinbox.$g.url"}; if (defined $ngname) { next if ($ngname eq ''); # disabled $g = $ngname; } - my $ng = PublicInbox::NewsGroup->new($g, $git_dir, $addr); + my $ng = PublicInbox::NewsGroup->new($g, $git_dir, $addr, $url); my $old_ng = $self->{groups}->{$g}; # Reuse the old one if possible since it can hold diff --git a/lib/PublicInbox/NewsGroup.pm b/lib/PublicInbox/NewsGroup.pm index adac919..98a3595 100644 --- a/lib/PublicInbox/NewsGroup.pm +++ b/lib/PublicInbox/NewsGroup.pm @@ -13,12 +13,21 @@ require PublicInbox::Search; require PublicInbox::Git; sub new { - my ($class, $name, $git_dir, $address) = @_; + my ($class, $name, $git_dir, $address, $url) = @_; + + # first email address is preferred $address = $address->[0] if ref($address); + if ($url) { + # assume protocol-relative URLs which start with '//' means + # the server supports both HTTP and HTTPS, favor HTTPS. + $url = "https:$url" if $url =~ m!\A//!; + $url .= '/' if $url !~ m!/\z!; + } my $self = bless { name => $name, git_dir => $git_dir, address => $address, + url => $url, }, $class; $self->{domain} = ($address =~ /\@(\S+)\z/) ? $1 : 'localhost'; $self; diff --git a/t/nntp.t b/t/nntp.t index 388620e..5513c7b 100644 --- a/t/nntp.t +++ b/t/nntp.t @@ -11,6 +11,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian Danga::Socket)) { } use_ok 'PublicInbox::NNTP'; +use_ok 'PublicInbox::NewsGroup'; { sub quote_str { @@ -95,4 +96,37 @@ use_ok 'PublicInbox::NNTP'; } } +{ # test setting NNTP headers in HEAD and ARTICLE requests + require Email::MIME; + my $u = 'https://example.com/a/'; + my $ng = PublicInbox::NewsGroup->new('test', 'test.git', + 'a@example.com', '//example.com/a'); + is($ng->{url}, $u, 'URL expanded'); + my $mid = 'a@b'; + my $mime = Email::MIME->new("Message-ID: <$mid>\r\n\r\n"); + PublicInbox::NNTP::set_nntp_headers($mime->header_obj, $ng, 1, $mid); + is_deeply([ $mime->header('Message-ID') ], [ "<$mid>" ], + 'Message-ID unchanged'); + is_deeply([ $mime->header('Archived-At') ], [ "<${u}a%40b/>" ], + 'Archived-At: set'); + is_deeply([ $mime->header('List-Archive') ], [ "<$u>" ], + 'List-Archive: set'); + is_deeply([ $mime->header('List-Post') ], [ '' ], + 'List-Post: set'); + is_deeply([ $mime->header('Newsgroups') ], [ 'test' ], + 'Newsgroups: set'); + is_deeply([ $mime->header('Xref') ], [ 'example.com test:1' ], + 'Xref: set'); + + $ng->{url} = 'http://mirror.example.com/m/'; + PublicInbox::NNTP::set_nntp_headers($mime->header_obj, $ng, 2, $mid); + is_deeply([ $mime->header('Message-ID') ], [ "<$mid>" ], + 'Message-ID unchanged'); + is_deeply([ $mime->header('Archived-At') ], + [ "<${u}a%40b/>", '' ], + 'Archived-At: appended'); + is_deeply([ $mime->header('Xref') ], [ 'example.com test:2' ], + 'Old Xref: clobbered'); +} + done_testing();