about summary refs log tree commit homepage
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/PublicInbox/Config.pm2
-rw-r--r--lib/PublicInbox/Inbox.pm9
-rw-r--r--lib/PublicInbox/NNTP.pm2
-rw-r--r--lib/PublicInbox/SearchIdxPart.pm7
-rw-r--r--lib/PublicInbox/Spawn.pm47
-rw-r--r--lib/PublicInbox/V2Writable.pm5
-rw-r--r--lib/PublicInbox/WWW.pm16
-rw-r--r--lib/PublicInbox/WwwListing.pm104
-rw-r--r--lib/PublicInbox/WwwStream.pm11
9 files changed, 174 insertions, 29 deletions
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 631c7880..09f9179b 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -389,7 +389,7 @@ sub _fill {
         }
         # TODO: more arrays, we should support multi-value for
         # more things to encourage decentralization
-        foreach my $k (qw(address altid nntpmirror coderepo)) {
+        foreach my $k (qw(address altid nntpmirror coderepo hide)) {
                 if (defined(my $v = $self->{"$pfx.$k"})) {
                         $ibx->{$k} = _array($v);
                 }
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 0d28dd04..286555f6 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -95,6 +95,15 @@ sub new {
         if (defined $dir && -f "$dir/inbox.lock") {
                 $opts->{version} = 2;
         }
+
+        # allow any combination of multi-line or comma-delimited hide entries
+        my $hide = {};
+        if (defined(my $h = $opts->{hide})) {
+                foreach my $v (@$h) {
+                        $hide->{$_} = 1 foreach (split(/\s*,\s*/, $v));
+                }
+                $opts->{-hide} = $hide;
+        }
         bless $opts, $class;
 }
 
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index f756e92c..5c5df7b0 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -122,7 +122,7 @@ sub args_ok ($$) {
 sub process_line ($$) {
         my ($self, $l) = @_;
         my ($req, @args) = split(/\s+/, $l);
-        return unless defined($req);
+        return 1 unless defined($req); # skip blank line
         $req = lc($req);
         $req = eval {
                 no strict 'refs';
diff --git a/lib/PublicInbox/SearchIdxPart.pm b/lib/PublicInbox/SearchIdxPart.pm
index 7fe2120a..51d81a0a 100644
--- a/lib/PublicInbox/SearchIdxPart.pm
+++ b/lib/PublicInbox/SearchIdxPart.pm
@@ -48,8 +48,15 @@ sub spawn_worker {
 sub partition_worker_loop ($$$$) {
         my ($self, $r, $part, $bnote) = @_;
         $0 = "pi-v2-partition[$part]";
+        my $current_info = '';
+        my $warn_cb = $SIG{__WARN__} || sub { print STDERR @_ };
+        local $SIG{__WARN__} = sub {
+                chomp $current_info;
+                $warn_cb->("[$part] $current_info: ", @_);
+        };
         $self->begin_txn_lazy;
         while (my $line = $r->getline) {
+                $current_info = $line;
                 if ($line eq "commit\n") {
                         $self->commit_txn_lazy;
                 } elsif ($line eq "close\n") {
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index 7b0f3bdd..66b916df 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -26,22 +26,35 @@ my $vfork_spawn = <<'VFORK_SPAWN';
 #include <sys/time.h>
 #include <sys/resource.h>
 #include <unistd.h>
-#include <alloca.h>
-#include <signal.h>
-#include <assert.h>
+#include <stdlib.h>
 
-#define AV_ALLOCA(av, max) alloca((max = (av_len((av)) + 1)) * sizeof(char *))
+/* some platforms need alloca.h, but some don't */
+#if defined(__GNUC__) && !defined(alloca)
+#  define alloca(sz) __builtin_alloca(sz)
+#endif
 
-static void av2c_copy(char **dst, AV *src, I32 max)
-{
-        I32 i;
+#include <signal.h>
+#include <assert.h>
 
-        for (i = 0; i < max; i++) {
-                SV **sv = av_fetch(src, i, 0);
-                dst[i] = sv ? SvPV_nolen(*sv) : 0;
-        }
-        dst[max] = 0;
-}
+/*
+ * From the av_len apidoc:
+ *   Note that, unlike what the name implies, it returns
+ *   the highest index in the array, so to get the size of
+ *   the array you need to use "av_len(av) + 1".
+ *   This is unlike "sv_len", which returns what you would expect.
+ */
+#define AV2C_COPY(dst, src) do { \
+        I32 i; \
+        I32 top_index = av_len(src); \
+        I32 real_len = top_index + 1; \
+        I32 capa = real_len + 1; \
+        dst = alloca(capa * sizeof(char *)); \
+        for (i = 0; i < real_len; i++) { \
+                SV **sv = av_fetch(src, i, 0); \
+                dst[i] = SvPV_nolen(*sv); \
+        } \
+        dst[real_len] = 0; \
+} while (0)
 
 static void *deconst(const char *s)
 {
@@ -86,15 +99,11 @@ int pi_fork_exec(int in, int out, int err,
         const char *filename = SvPV_nolen(file);
         pid_t pid;
         char **argv, **envp;
-        I32 max;
         sigset_t set, old;
         int ret, errnum;
 
-        argv = AV_ALLOCA(cmd, max);
-        av2c_copy(argv, cmd, max);
-
-        envp = AV_ALLOCA(env, max);
-        av2c_copy(envp, env, max);
+        AV2C_COPY(argv, cmd);
+        AV2C_COPY(envp, env);
 
         ret = sigfillset(&set);
         assert(ret == 0 && "BUG calling sigfillset");
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 6829a343..87e8f3eb 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -72,6 +72,7 @@ sub new {
                 im => undef, #  PublicInbox::Import
                 parallel => 1,
                 transact_bytes => 0,
+                current_info => '',
                 xpfx => $xpfx,
                 over => PublicInbox::OverIdx->new("$xpfx/over.sqlite3", 1),
                 lock_path => "$dir/inbox.lock",
@@ -949,8 +950,10 @@ sub index_sync {
                 my $fh = $self->{reindex_pipe} = $git->popen(@cmd, $range);
                 my $cmt;
                 while (<$fh>) {
+                        chomp;
+                        $self->{current_info} = "$i.git $_";
                         if (/\A$x40$/o && !defined($cmt)) {
-                                chomp($cmt = $_);
+                                $cmt = $_;
                         } elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\tm$/o) {
                                 $self->reindex_oid($mm_tmp, $D, $git, $1,
                                                 $regen, $reindex);
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 6e69001c..1f3ca157 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -11,7 +11,7 @@
 # - Must not rely on static content
 # - UTF-8 is only for user-content, 7-bit US-ASCII for us
 package PublicInbox::WWW;
-use 5.008;
+use 5.010_001;
 use strict;
 use warnings;
 use bytes (); # only for bytes::length
@@ -68,8 +68,9 @@ sub call {
         } split(/[&;]+/, $env->{QUERY_STRING});
         $ctx->{qp} = \%qp;
 
-        # not using $env->{PATH_INFO} here since that's already decoded
+        # avoiding $env->{PATH_INFO} here since that's already decoded
         my ($path_info) = ($env->{REQUEST_URI} =~ path_re($env));
+        $path_info //= $env->{PATH_INFO};
         my $method = $env->{REQUEST_METHOD};
 
         if ($method eq 'POST') {
@@ -87,7 +88,7 @@ sub call {
 
         # top-level indices and feeds
         if ($path_info eq '/') {
-                r404();
+                www_listing($self)->call($env);
         } elsif ($path_info =~ m!$INBOX_RE\z!o) {
                 invalid_inbox($ctx, $1) || r301($ctx, $1);
         } elsif ($path_info =~ m!$INBOX_RE(?:/|/index\.html)?\z!o) {
@@ -157,6 +158,7 @@ sub preload {
         if (ref($self)) {
                 $self->cgit;
                 $self->stylesheets_prepare($_) for ('', '../', '../../');
+                $self->www_listing;
         }
 }
 
@@ -489,6 +491,14 @@ sub cgit {
         }
 }
 
+sub www_listing {
+        my ($self) = @_;
+        $self->{www_listing} ||= do {
+                require PublicInbox::WwwListing;
+                PublicInbox::WwwListing->new($self);
+        }
+}
+
 sub get_attach {
         my ($ctx, $idx, $fn) = @_;
         require PublicInbox::WwwAttach;
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
new file mode 100644
index 00000000..e8dad4b8
--- /dev/null
+++ b/lib/PublicInbox/WwwListing.pm
@@ -0,0 +1,104 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Provide an HTTP-accessible listing of inboxes.
+# Used by PublicInbox::WWW
+package PublicInbox::WwwListing;
+use strict;
+use warnings;
+use PublicInbox::Hval qw(ascii_html);
+use PublicInbox::Linkify;
+use PublicInbox::View;
+
+sub list_all ($$) {
+        my ($self, undef) = @_;
+        my @list;
+        $self->{pi_config}->each_inbox(sub {
+                my ($ibx) = @_;
+                push @list, $ibx unless $ibx->{-hide}->{www};
+        });
+        \@list;
+}
+
+sub list_match_domain ($$) {
+        my ($self, $env) = @_;
+        my @list;
+        my $host = $env->{HTTP_HOST} // $env->{SERVER_NAME};
+        $host =~ s/:\d+\z//;
+        my $re = qr!\A(?:https?:)?//\Q$host\E(?::\d+)?/!i;
+        $self->{pi_config}->each_inbox(sub {
+                my ($ibx) = @_;
+                push @list, $ibx if !$ibx->{-hide}->{www} && $ibx->{url} =~ $re;
+        });
+        \@list;
+}
+
+sub list_404 ($$) { [] }
+
+# TODO: +cgit
+my %VALID = (
+        all => *list_all,
+        'match=domain' => *list_match_domain,
+        404 => *list_404,
+);
+
+sub new {
+        my ($class, $www) = @_;
+        my $k = 'publicinbox.wwwListing';
+        my $pi_config = $www->{pi_config};
+        my $v = $pi_config->{lc($k)} // 404;
+        bless {
+                pi_config => $pi_config,
+                style => $www->style("\0"),
+                list_cb => $VALID{$v} || do {
+                        warn <<"";
+`$v' is not a valid value for `$k'
+$k be one of `all', `match=domain', or `404'
+
+                        *list_404;
+                },
+        }, $class;
+}
+
+sub ibx_entry {
+        my ($mtime, $ibx, $env) = @_;
+        my $ts = PublicInbox::View::fmt_ts($mtime);
+        my $url = PublicInbox::Hval::prurl($env, $ibx->{url});
+        my $tmp = <<"";
+* $ts - $url
+  ${\$ibx->description}
+
+        if (defined(my $info_url = $ibx->{info_url})) {
+                $tmp .= "\n$info_url";
+        }
+        $tmp;
+}
+
+# not really a stand-alone PSGI app, but maybe it could be...
+sub call {
+        my ($self, $env) = @_;
+        my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ];
+        my $list = $self->{list_cb}->($self, $env);
+        my $code = 404;
+        my $title = 'public-inbox';
+        my $out = '';
+        if (@$list) {
+                # Swartzian transform since ->modified is expensive
+                @$list = sort {
+                        $b->[0] <=> $a->[0]
+                } map { [ $_->modified, $_ ] } @$list;
+
+                $code = 200;
+                $title .= ' - listing';
+                my $tmp = join("\n", map { ibx_entry(@$_, $env) } @$list);
+                my $l = PublicInbox::Linkify->new;
+                $l->linkify_1($tmp);
+                $out = '<pre>'.$l->linkify_2(ascii_html($tmp)).'</pre><hr>';
+        }
+        $out = "<html><head><title>$title</title></head><body>" . $out;
+        $out .= '<pre>'. PublicInbox::WwwStream::code_footer($env) .
+                '</pre></body></html>';
+        [ $code, $h, [ $out ] ]
+}
+
+1;
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index ea7aaad0..f6c50496 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -10,7 +10,6 @@ package PublicInbox::WwwStream;
 use strict;
 use warnings;
 use PublicInbox::Hval qw(ascii_html);
-use URI;
 our $TOR_URL = 'https://www.torproject.org/';
 our $CODE_URL = 'https://public-inbox.org/';
 our $PROJECT = 'public-inbox';
@@ -70,6 +69,12 @@ sub _html_top ($) {
                 "</head><body>". $top . $tip;
 }
 
+sub code_footer ($) {
+        my ($env) = @_;
+        my $u = PublicInbox::Hval::prurl($env, $CODE_URL);
+        qq(AGPL code for this site: git clone <a\nhref="$u">$u</a> $PROJECT)
+}
+
 sub _html_end {
         my ($self) = @_;
         my $urls = 'Archives are clonable:';
@@ -134,12 +139,10 @@ EOF
                 $urls .= "\n note: .onion URLs require Tor: ";
                 $urls .= qq[<a\nhref="$TOR_URL">$TOR_URL</a>];
         }
-        my $url = PublicInbox::Hval::prurl($ctx->{env}, $CODE_URL);
         '<hr><pre>'.join("\n\n",
                 $desc,
                 $urls,
-                'AGPL code for this site: '.
-                qq(git clone <a\nhref="$url">$url</a> $PROJECT)
+                code_footer($ctx->{env})
         ).'</pre></body></html>';
 }