diff options
| -rw-r--r-- | lib/PublicInbox/LeiMirror.pm | 13 | ||||
| -rw-r--r-- | lib/PublicInbox/Msgmap.pm | 9 | ||||
| -rw-r--r-- | lib/PublicInbox/WwwText.pm | 3 | ||||
| -rw-r--r-- | t/lei-mirror.t | 31 | ||||
| -rw-r--r-- | t/psgi_v2.t | 10 |
5 files changed, 55 insertions, 11 deletions
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 5cfa6fea..4be8f70a 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -96,15 +96,15 @@ sub _get_txt { # non-fatal my $path = $uri->path; chop($path) eq '/' or die "BUG: $uri not canonicalized"; $uri->path("$path/$endpoint"); - my $cmd = $self->{curl}->for_uri($lei, $uri, '--compressed'); - my $ce = "$self->{dst}/$file"; - my $ft = File::Temp->new(TEMPLATE => "$file-XXXX", - UNLINK => 1, DIR => $self->{dst}); - my $opt = { 0 => $lei->{0}, 1 => $ft, 2 => $lei->{2} }; + my $ft = File::Temp->new(TEMPLATE => "$file-XXXX", DIR => $self->{dst}); + my $f = $ft->filename; + my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} }; + my $cmd = $self->{curl}->for_uri($lei, $uri, + qw(--compressed -R -o), $f); my $cerr = run_reap($lei, $cmd, $opt); return "$uri missing" if ($cerr >> 8) == 22; return "# @$cmd failed (non-fatal)" if $cerr; - my $f = $ft->filename; + my $ce = "$self->{dst}/$file"; rename($f, $ce) or return "rename($f, $ce): $! (non-fatal)"; $ft->unlink_on_destroy(0); undef; # success @@ -122,6 +122,7 @@ sub _try_config { my $err = _get_txt($self, qw(_/text/config/raw inbox.config.example)); return $self->{lei}->err($err) if $err; my $f = "$self->{dst}/inbox.config.example"; + chmod((stat($f))[2] & 0444, $f) or die "chmod(a-w, $f): $!"; my $cfg = PublicInbox::Config->git_config_dump($f, $self->{lei}->{2}); my $ibx = $self->{ibx} = {}; for my $sec (grep(/\Apublicinbox\./, @{$cfg->{-section_order}})) { diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index 94a0cbeb..e71f16f8 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -32,8 +32,15 @@ sub new_file { if ($rw) { $dbh->begin_work; create_tables($dbh); - $self->created_at(time) unless $self->created_at; + unless ($self->created_at) { + my $t; + if (blessed($ibx) && + -f "$ibx->{inboxdir}/inbox.config.example") { + $t = (stat(_))[9]; # mtime set by "curl -R" + } + $self->created_at($t // time); + } $self->num_highwater(max($self)); $dbh->commit; } diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm index bb9a0a0f..8b929f74 100644 --- a/lib/PublicInbox/WwwText.pm +++ b/lib/PublicInbox/WwwText.pm @@ -8,6 +8,7 @@ use v5.10.1; use PublicInbox::Linkify; use PublicInbox::WwwStream; use PublicInbox::Hval qw(ascii_html prurl); +use HTTP::Date qw(time2str); use URI::Escape qw(uri_escape_utf8); use PublicInbox::GzipFilter qw(gzf_maybe); our $QP_URL = 'https://xapian.org/docs/queryparser.html'; @@ -171,6 +172,8 @@ sub inbox_config ($$$) { my ($ctx, $hdr, $txt) = @_; my $ibx = $ctx->{ibx}; push @$hdr, 'Content-Disposition', 'inline; filename=inbox.config'; + my $t = eval { $ibx->mm->created_at }; + push(@$hdr, 'Last-Modified', time2str($t)) if $t; my $name = dq_escape($ibx->{name}); my $inboxdir = '/path/to/top-level-inbox'; my $base_url = $ibx->base_url($ctx->{env}); diff --git a/t/lei-mirror.t b/t/lei-mirror.t index de5246b6..b449e0b4 100644 --- a/t/lei-mirror.t +++ b/t/lei-mirror.t @@ -3,8 +3,9 @@ # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> use strict; use v5.10.1; use PublicInbox::TestCommon; use PublicInbox::Inbox; -require_mods(qw(-httpd lei)); +require_mods(qw(-httpd lei DBD::SQLite)); require_cmd('curl'); +require PublicInbox::Msgmap; my $sock = tcp_server(); my ($tmpdir, $for_destroy) = tmpdir(); my $http = 'http://'.tcp_host_port($sock); @@ -12,25 +13,40 @@ my ($ro_home, $cfg_path) = setup_public_inboxes; my $cmd = [ qw(-httpd -W0 ./t/lei-mirror.psgi), "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ]; my $td = start_script($cmd, { PI_CONFIG => $cfg_path }, { 3 => $sock }); +my %created; test_lei({ tmpdir => $tmpdir }, sub { my $home = $ENV{HOME}; my $t1 = "$home/t1-mirror"; + my $mm_orig = "$ro_home/t1/public-inbox/msgmap.sqlite3"; + $created{v1} = PublicInbox::Msgmap->new_file($mm_orig)->created_at; lei_ok('add-external', $t1, '--mirror', "$http/t1/", \'--mirror v1'); - ok(-f "$t1/public-inbox/msgmap.sqlite3", 't1-mirror indexed'); + my $mm_dup = "$t1/public-inbox/msgmap.sqlite3"; + ok(-f $mm_dup, 't1-mirror indexed'); is(PublicInbox::Inbox::try_cat("$t1/description"), "mirror of $http/t1/\n", 'description set'); ok(-f "$t1/Makefile", 'convenience Makefile added (v1)'); + ok(-f "$t1/inbox.config.example", 'inbox.config.example downloaded'); + is((stat(_))[9], $created{v1}, + 'inbox.config.example mtime is ->created_at'); + is((stat(_))[2] & 0222, 0, 'inbox.config.example not writable'); + my $tb = PublicInbox::Msgmap->new_file($mm_dup)->created_at; + is($tb, $created{v1}, 'created_at matched in mirror'); lei_ok('ls-external'); like($lei_out, qr!\Q$t1\E!, 't1 added to ls-externals'); my $t2 = "$home/t2-mirror"; + $mm_orig = "$ro_home/t2/msgmap.sqlite3"; + $created{v2} = PublicInbox::Msgmap->new_file($mm_orig)->created_at; lei_ok('add-external', $t2, '--mirror', "$http/t2/", \'--mirror v2'); - ok(-f "$t2/msgmap.sqlite3", 't2-mirror indexed'); + $mm_dup = "$t2/msgmap.sqlite3"; + ok(-f $mm_dup, 't2-mirror indexed'); ok(-f "$t2/description", 't2 description'); ok(-f "$t2/Makefile", 'convenience Makefile added (v2)'); is(PublicInbox::Inbox::try_cat("$t2/description"), "mirror of $http/t2/\n", 'description set'); + $tb = PublicInbox::Msgmap->new_file($mm_dup)->created_at; + is($tb, $created{v2}, 'created_at matched in v2 mirror'); lei_ok('ls-external'); like($lei_out, qr!\Q$t2\E!, 't2 added to ls-externals'); @@ -150,6 +166,15 @@ SKIP: { ok(unlink("$d/t1/manifest.js.gz"), 'manifest created'); my $after = [ glob("$d/t1/*") ]; is_deeply($before, $after, 'no new files created'); + + ok(run_script([qw(-index -Lbasic), "$d/t1"]), 'index v1'); + ok(run_script([qw(-index -Lbasic), "$d/t2"]), 'index v2'); + my $f = "$d/t1/public-inbox/msgmap.sqlite3"; + my $ca = PublicInbox::Msgmap->new_file($f)->created_at; + is($ca, $created{v1}, 'clone + index v1 synced ->created_at'); + $f = "$d/t2/msgmap.sqlite3"; + $ca = PublicInbox::Msgmap->new_file($f)->created_at; + is($ca, $created{v2}, 'clone + index v1 synced ->created_at'); } ok($td->kill, 'killed -httpd'); diff --git a/t/psgi_v2.t b/t/psgi_v2.t index 1f190708..e0570682 100644 --- a/t/psgi_v2.t +++ b/t/psgi_v2.t @@ -9,7 +9,7 @@ use PublicInbox::Eml; use PublicInbox::Config; use PublicInbox::MID qw(mids); require_mods(qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test - URI::Escape Plack::Builder)); + URI::Escape Plack::Builder HTTP::Date)); use_ok($_) for (qw(HTTP::Request::Common Plack::Test)); use_ok 'PublicInbox::WWW'; my ($tmpdir, $for_destroy) = tmpdir(); @@ -113,6 +113,14 @@ $im->done; my $client1 = sub { my ($cb) = @_; + $res = $cb->(GET('/v2test/_/text/config/raw')); + my $lm = $res->header('Last-Modified'); + ok($lm, 'Last-Modified set w/ ->mm'); + $lm = HTTP::Date::str2time($lm); + is($lm, $ibx->mm->created_at, + 'Last-Modified for text/config/raw matches ->created_at'); + delete $ibx->{mm}; + $res = $cb->(GET("/v2test/$third/raw")); $raw = $res->content; like($raw, qr/^hello ghosts$/m, 'got third message'); |