diff options
Diffstat (limited to 'lib/PublicInbox/Inbox.pm')
| -rw-r--r-- | lib/PublicInbox/Inbox.pm | 230 |
1 files changed, 102 insertions, 128 deletions
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index c0962af9..dd689221 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -1,75 +1,51 @@ -# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # # Represents a public-inbox (which may have multiple mailing addresses) package PublicInbox::Inbox; use strict; +use v5.10.1; use PublicInbox::Git; use PublicInbox::MID qw(mid2path); use PublicInbox::Eml; use List::Util qw(max); use Carp qw(croak); +use PublicInbox::Compat qw(uniqstr); -# Long-running "git-cat-file --batch" processes won't notice -# unlinked packs, so we need to restart those processes occasionally. -# Xapian and SQLite file handles are mostly stable, but sometimes an -# admin will attempt to replace them atomically after compact/vacuum -# and we need to be prepared for that. -my $cleanup_timer; -my $CLEANUP = {}; # string(inbox) -> inbox - -sub git_cleanup ($) { - my ($self) = @_; - my $git = $self->{git} // return undef; - # normal inboxes have low startup cost and there may be many, so - # keep process+pipe counts in check. ExtSearch may have high startup - # cost (e.g. ->ALL) and but likely one per-daemon, so cleanup only - # if there's unlinked files - my $live = $self->isa(__PACKAGE__) ? $git->cleanup - : $git->cleanup_if_unlinked; - delete($self->{git}) unless $live; - $live; +# in case DBs get replaced (Xapcmd does it for v1) +sub check_inodes ($) { + for (qw(over mm)) { $_[0]->{$_}->check_inodes if $_[0]->{$_} } } -# returns true if further checking is required -sub cleanup_shards { $_[0]->{search} ? $_[0]->{search}->cleanup_shards : undef } - -sub cleanup_task () { - $cleanup_timer = undef; - my $next = {}; - for my $ibx (values %$CLEANUP) { - my $again = git_cleanup($ibx); - $ibx->cleanup_shards and $again = 1; - for my $git (@{$ibx->{-repo_objs}}) { - $again = 1 if $git->cleanup; - } - check_inodes($ibx); - $next->{"$ibx"} = $ibx if $again; - } - $CLEANUP = $next; - $cleanup_timer //= PublicInbox::DS::later(\&cleanup_task); +# search/over/mm hold onto FDs and description+cloneurl may get updated. +# creating long-lived allocations in the same phase as short-lived +# allocations also leads to fragmentation, so we don't want some stuff +# living too long. +sub do_cleanup { + my ($ibx) = @_; + my ($srch) = delete @$ibx{qw(search over mm description cloneurl)}; + $srch //= $ibx; # extsearch + delete @$srch{qw(xdb qp)}; } sub _cleanup_later ($) { # no need to require DS, here, if it were enabled another # module would've require'd it, already - if (eval { PublicInbox::DS::in_loop() }) { - $cleanup_timer //= PublicInbox::DS::later(\&cleanup_task); - $CLEANUP->{"$_[0]"} = $_[0]; # $self - } + eval { PublicInbox::DS::in_loop() } and + PublicInbox::DS::add_uniq_timer($_[0]+0, 30, \&do_cleanup, @_) } sub _set_limiter ($$$) { my ($self, $pi_cfg, $pfx) = @_; my $lkey = "-${pfx}_limiter"; - $self->{$lkey} ||= do { + $self->{$lkey} //= do { # full key is: publicinbox.$NAME.httpbackendmax my $mkey = $pfx.'max'; my $val = $self->{$mkey} or return; my $lim; if ($val =~ /\A[0-9]+\z/) { - require PublicInbox::Qspawn; - $lim = PublicInbox::Qspawn::Limiter->new($val); + require PublicInbox::Limiter; + $lim = PublicInbox::Limiter->new($val); } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) { $lim = $pi_cfg->limiter($val); warn "$mkey limiter=$val not found\n" if !$lim; @@ -94,12 +70,8 @@ sub new { delete $opts->{feedmax}; } # allow any combination of multi-line or comma-delimited hide entries - my $hide = {}; - if (defined(my $h = $opts->{hide})) { - foreach my $v (@$h) { - $hide->{$_} = 1 foreach (split(/\s*,\s*/, $v)); - } - $opts->{-hide} = $hide; + for $v (@{delete($opts->{hide}) // []}) { + $opts->{-'hide_'.$_} = 1 for split(/\s*,\s*/, $v); } bless $opts, $class; } @@ -129,7 +101,6 @@ sub git { my $g = PublicInbox::Git->new($git_dir); my $lim = $self->{-httpbackend_limiter}; $g->{-httpbackend_limiter} = $lim if $lim; - _cleanup_later($self); $g; }; } @@ -140,7 +111,7 @@ sub max_git_epoch { my $cur = $self->{-max_git_epoch}; my $changed; if (!defined($cur) || ($changed = git($self)->alternates_changed)) { - git_cleanup($self) if $changed; + $self->{git}->cleanup if $changed; my $gits = "$self->{inboxdir}/git"; if (opendir my $dh, $gits) { my $max = max(map { @@ -152,27 +123,29 @@ sub max_git_epoch { $cur; } +sub mm_file { + my ($self) = @_; + my $d = $self->{inboxdir}; + ($self->version >= 2 ? $d : "$d/public-inbox").'/msgmap.sqlite3'; +} + sub mm { my ($self, $req) = @_; $self->{mm} //= eval { require PublicInbox::Msgmap; - my $dir = $self->{inboxdir}; - if ($self->version >= 2) { - PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3"); - } else { - PublicInbox::Msgmap->new($dir); - } + _cleanup_later($self); + PublicInbox::Msgmap->new_file($self); } // ($req ? croak("E: $@") : undef); } sub search { my ($self) = @_; - my $srch = $self->{search} //= eval { + $self->{search} // eval { _cleanup_later($self); require PublicInbox::Search; - PublicInbox::Search->new($self); + my $srch = PublicInbox::Search->new($self); + (eval { $srch->xdb }) ? ($self->{search} = $srch) : undef; }; - (eval { $srch->xdb }) ? $srch : undef; } # isrch is preferred for read-only interfaces if available since it @@ -181,47 +154,32 @@ sub isrch { $_[0]->{isrch} // search($_[0]) } sub over { my ($self, $req) = @_; - $self->{over} //= eval { - my $srch = $self->{search} //= do { - _cleanup_later($self); + $self->{over} // eval { + my $srch = $self->{search} // do { require PublicInbox::Search; PublicInbox::Search->new($self); }; + _cleanup_later($self); my $over = PublicInbox::Over->new("$srch->{xpfx}/over.sqlite3"); $over->dbh; # may fail - $over; + $self->{over} = $over; } // ($req ? croak("E: $@") : undef); } -sub try_cat { - my ($path) = @_; - open(my $fh, '<', $path) or return ''; - local $/; - <$fh> // ''; -} - -sub cat_desc ($) { - my $desc = try_cat($_[0]); - local $/ = "\n"; - chomp $desc; - utf8::decode($desc); - $desc =~ s/\s+/ /smg; - $desc eq '' ? undef : $desc; -} - sub description { my ($self) = @_; - ($self->{description} //= cat_desc("$self->{inboxdir}/description")) // + ($self->{description} //= + PublicInbox::Git::cat_desc("$self->{inboxdir}/description")) // '($INBOX_DIR/description missing)'; } sub cloneurl { my ($self) = @_; - ($self->{cloneurl} //= do { - my $s = try_cat("$self->{inboxdir}/cloneurl"); - my @urls = split(/\s+/s, $s); - scalar(@urls) ? \@urls : undef - }) // []; + $self->{cloneurl} // do { + my @urls = split(/\s+/s, + PublicInbox::IO::try_cat "$self->{inboxdir}/cloneurl"); + scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef; + } // []; } sub base_url { @@ -232,20 +190,20 @@ sub base_url { $url .= '/' if $url !~ m!/\z!; return $url .= $self->{name} . '/'; } - # called from a non-PSGI environment (e.g. NNTP/POP3): - $self->{-base_url} ||= do { - my $url = $self->{url} // return undef; - $url = $url->[0] // return undef; - # expand protocol-relative URLs to HTTPS if we're - # not inside a web server - $url = "https:$url" if $url =~ m!\A//!; - $url .= '/' if $url !~ m!/\z!; - $url; - }; -} - + # called from a non-PSGI environment or cross-inbox environment + # where multiple inboxes can have different domains + my $url = $self->{url} // return undef; + $url = $url->[0] // return undef; + # expand protocol-relative URLs to HTTPS if we're + # not inside a web server + substr($url, 0, 0, 'https:') if substr($url, 0, 2) eq '//'; + $url .= '/' if substr($url, -1, 1) ne '/'; + $url; +} + +# imapserver, nntpserver configs are used here: sub _x_url ($$$) { - my ($self, $x, $ctx) = @_; # $x is "nntp" or "imap" + my ($self, $x, $ctx) = @_; # $x is "imap" or "nntp" # no checking for nntp_usable here, we can point entirely # to non-local servers or users run by a different user my $ns = $self->{"${x}server"} // @@ -267,7 +225,7 @@ sub _x_url ($$$) { if ($group) { $u .= '/' if $u !~ m!/\z!; $u .= $group; - } else { # n.b. IMAP uses "newsgroup" + } else { # n.b. IMAP and POP3 use "newsgroup" warn <<EOM; publicinbox.$self->{name}.${x}mirror=$_ missing newsgroup name EOM @@ -277,23 +235,41 @@ EOM # nntp://news.example.com/alt.example push @m, $u; } - - # List::Util::uniq requires Perl 5.26+, maybe we - # can use it by 2030 or so - my %seen; - @urls = grep { !$seen{$_}++ } (@urls, @m); + @urls = uniqstr @urls, @m; } \@urls; } # my ($self, $ctx) = @_; -sub nntp_url { $_[0]->{-nntp_url} //= _x_url($_[0], 'nntp', $_[1]) } sub imap_url { $_[0]->{-imap_url} //= _x_url($_[0], 'imap', $_[1]) } +sub nntp_url { $_[0]->{-nntp_url} //= _x_url($_[0], 'nntp', $_[1]) } + +sub pop3_url { + my ($self, $ctx) = @_; + $self->{-pop3_url} //= do { + my $ps = $self->{'pop3server'} // + $ctx->{www}->{pi_cfg}->get_all('publicinbox.pop3server'); + my $group = $self->{newsgroup}; + my @urls; + ($ps && $group) and + @urls = map { m!\Apop3?s?://! ? $_ : "pop3://$_" } @$ps; + if (my $mi = $self->{'pop3mirror'}) { + my @m = map { m!\Apop3?s?://! ? $_ : "pop3://$_" } @$mi; + @urls = uniqstr @urls, @m; + } + my $n = 0; + for (@urls) { $n += s!/+\z!! } + warn <<EOM if $n; +W: pop3server and/or pop3mirror URLs should not end with trailing slash `/' +EOM + \@urls; + } +} sub nntp_usable { my ($self) = @_; my $ret = mm($self) && over($self); - $self->{mm} = $self->{over} = $self->{search} = undef; + delete @$self{qw(mm over search)}; $ret; } @@ -308,17 +284,15 @@ sub msg_by_smsg ($$) { # ghosts may have undef smsg (from SearchThread.node) or # no {blob} field - return unless defined $smsg; - defined(my $blob = $smsg->{blob}) or return; - - $self->git->cat_file($blob); + $smsg // return; + $self->git->cat_file($smsg->{blob} // return); } sub smsg_eml { my ($self, $smsg) = @_; my $bref = msg_by_smsg($self, $smsg) or return; my $eml = PublicInbox::Eml->new($bref); - $smsg->populate($eml) unless exists($smsg->{num}); # v1 w/o SQLite + $smsg->{num} // $smsg->populate($eml); $eml; } @@ -328,7 +302,7 @@ sub smsg_by_mid ($$) { my $smsg; if (my $mm = $self->mm) { # favor the Message-ID we used for the NNTP article number: - defined(my $num = $mm->num_for($mid)) or return; + my $num = $mm->num_for($mid) // return; $smsg = $over->get_art($num); } else { my ($id, $prev); @@ -343,11 +317,6 @@ sub msg_by_mid ($$) { $smsg ? msg_by_smsg($self, $smsg) : msg_by_path($self, mid2path($mid)); } -sub recent { - my ($self, $opts, $after, $before) = @_; - $self->over->recent($opts, $after, $before); -} - sub modified { my ($self) = @_; if (my $over = $self->over) { @@ -364,7 +333,7 @@ sub modified { # (pathname is NOT public, but prefix is used for Xapian queries) sub altid_map ($) { my ($self) = @_; - $self->{-altid_map} //= eval { + eval { require PublicInbox::AltId; my $altid = $self->{altid} or return {}; my %h = map {; @@ -386,17 +355,10 @@ sub unsubscribe_unlock { delete $self->{unlock_subs}->{$ident}; } -sub check_inodes ($) { - my ($self) = @_; - for (qw(over mm)) { # TODO: search - $self->{$_}->check_inodes if $self->{$_}; - } -} - # called by inotify sub on_unlock { my ($self) = @_; - check_inodes($self); + check_inodes($self); # DB files may be replaced while holding lock my $subs = $self->{unlock_subs} or return; for my $obj (values %$subs) { eval { $obj->on_inbox_unlock($self) }; @@ -408,6 +370,16 @@ sub uidvalidity { $_[0]->{uidvalidity} //= eval { $_[0]->mm->created_at } } sub eidx_key { $_[0]->{newsgroup} // $_[0]->{inboxdir} } +# only used by NNTP, so we need ->mm anyways +sub art_min { $_[0]->{-art_min} //= eval { $_[0]->mm(1)->min } } + +# used by IMAP, too, which tries to avoid ->mm (but ->{mm} is likely +# faster since it's smaller iff available) +sub art_max { + $_[0]->{-art_max} //= eval { $_[0]->{mm}->max } // + eval { $_[0]->over(1)->max }; +} + sub mailboxid { # rfc 8474, 8620, 8621 my ($self, $imap_slice) = @_; my $pfx = defined($imap_slice) ? $self->{newsgroup} : $self->{name}; @@ -420,4 +392,6 @@ sub mailboxid { # rfc 8474, 8620, 8621 sprintf('-%x', uidvalidity($self) // 0) } +sub thing_type { 'public inbox' } + 1; |