about summary refs log tree commit homepage
diff options
context:
space:
mode:
-rw-r--r--INSTALL150
-rw-r--r--lib/PublicInbox/ExtMsg.pm2
-rw-r--r--lib/PublicInbox/Msgmap.pm2
-rw-r--r--lib/PublicInbox/NNTP.pm13
-rw-r--r--lib/PublicInbox/SearchView.pm16
-rw-r--r--t/nntpd.t2
-rw-r--r--t/psgi_search.t10
7 files changed, 135 insertions, 60 deletions
diff --git a/INSTALL b/INSTALL
index 11d844cf..87aa6961 100644
--- a/INSTALL
+++ b/INSTALL
@@ -8,60 +8,117 @@ if they want to import mail into their personal inboxes.
 TODO: this still needs to be documented better,
 also see the scripts/ and sa_config/ directories in the source tree
 
-It should also be possible to use public-inbox with only IMAP
-(or even POP(!)) access to a mailbox.
-
-standard MakeMaker installation (Perl)
---------------------------------------
-
-        perl Makefile.PL
-        make
-        make test
-        make install # root permissions may be needed
-
 Requirements
 ------------
 
-* git
-* Perl and several modules:    (Debian package name)
-  - Date::Parse                libtimedate-perl
-  - Email::MIME                libemail-mime-perl
-  - Email::MIME::ContentType   libemail-mime-contenttype-perl
-  - Encode::MIME::Header       perl
+public-inbox requires a number of other packages to access its full
+functionality.  The core tools are, of course:
 
-Optional components:
+* Git
+* Perl
+* SQLite (needed for Xapian use)
+
+To accept incoming mail into a public inbox, you'll likely want:
 
 * MTA - postfix is recommended (for public-inbox-mda)
 * SpamAssassin (spamc/spamd)   (for public-inbox-watch/public-inbox-mda)
 
-Optional Perl modules:
-
-  - Plack[1]                   libplack-perl
-  - URI::Escape[1]             liburi-perl
-  - Search::Xapian[2][3]       libsearch-xapian-perl
-  - IO::Compress::Gzip[3]      perl-modules (or libio-compress-perl)
-  - DBI[3]                     libdbi-perl
-  - DBD::SQLite[2][3]          libdbd-sqlite3-perl
-  - Danga::Socket[4]           libdanga-socket-perl
-  - Net::Server[5]             libnet-server-perl
-  - Filesys::Notify::Simple[6] libfilesys-notify-simple-perl
-  - Inline::C[7]               libinline-c-perl
-  - Plack::Middleware::ReverseProxy[8] libplack-middleware-reverseproxy-perl
-  - Plack::Middleware::Deflater[8] libplack-middleware-deflater-perl
-
-[1] - Optional, needed for serving/generating Atom and HTML pages
-[2] - Optional, only required for NNTP server
-[3] - Optional, needed for gzipped mbox support over HTTP
-[4] - Optional, needed for bundled HTTP and NNTP servers
-[5] - Optional, needed for standalone daemonization of HTTP+NNTP servers
-[6] - Optional, needed for public-inbox-watch Maildir watcher
-[7] - Optional, allows speeds up spawning on Linux (see public-inbox-daemon(8))
-[8] - Optional, recommended for PSGI interface
+Beyond that, there is a long list of Perl modules required, starting with:
+
+* Date::Parse                   deb: libdatetime-perl
+                                rpm: perl-Time-ParseDate
+
+* Email::MIME                   deb: libemail-mime-perl
+                                rpm: perl-Email-MIME
+
+* Email::MIME::ContentType      deb: libemail-mime-contenttype-perl
+                                rpm: perl-Email-MIME-ContentType
+
+* Encode::MIME::Header          deb: libencode-perl
+                                rpm: perl-Encode
+
+Where "deb" indicates package names for Debian-derived distributions and
+"rpm" is for RPM-based distributions (only known to work on Fedora).
+
+Numerous optional modules are likely to be useful as well:
+
+  - Plack                      deb: libplack-perl
+                               rpm: perl-Plack, perl-Plack-Test,
+                               (for HTML/Atom generation)
+
+  - URI::Escape                deb: liburi-perl
+                               rpm: perl-URI
+                               (for HTML/Atom generation)
+
+  - Search::Xapian             deb: libsearch-xapian-perl
+                               rpm: perl-Search-Xapian
+                               (for NNTP service or gzipped mbox over HTTP)
+
+  - IO::Compress::Gzip         deb: perl-modules (or libio-compress-perl)
+                               rpm: perl-PerlIO-gzip
+                               (for gzipped mbox over HTTP)
+
+  - DBI                        deb: libdbi-perl
+                               rpm: perl-DBI
+                               (for gzipped mbox over HTTP)
+
+  - DBD::SQLite                deb: libdbd-sqlite3-perl
+                               rpm: perl-DBD-SQLite
+                               (for NNTP service or gzipped mbox over HTTP)
+
+  - Danga::Socket              deb: libdanga-socket-perl
+                               rpm: perl-Danga-Socket
+                               (for bundled HTTP and NNTP servers)
+
+  - Net::Server                deb: libnet-server-perl
+                               rpm: perl-Net-Server
+                               (for HTTP/NNTP servers as standalone daemons)
+
+  - Filesys::Notify::Simple    deb: libfilesys-notify-simple-perl
+                               rpm: perl-Filesys-Notify-Simple
+                               (for public-inbox-watch)
+
+  - Inline::C[7]               deb: libinline-c-perl
+                               (speeds up spawning on Linux
+                                (see public-inbox-daemon(8))
+
+  - Plack::Middleware::ReverseProxy
+
+                               deb: libplack-middleware-reverseproxy-perl
+                               rpm: perl-Plack-Middleware-ReverseProxy
+                               (ensures redirects are correct when running
+                                behind nginx or Varnish)
+
+  - Plack::Middleware::Deflater
+
+                               deb: libplack-middleware-deflater-perl
+                               rpm: perl-Plack-Middleware-Deflater
+                               (saves bandwidth on responses)
+
+
+On Fedora systems, you'll probably also end up wanting
+perl-Test-HTTP-Server-Simple, perl-Devel-Peek, and perl-IPC-Run to run the
+test suite.  On Debian systems, libxml-feed-perl and libipc-run-perl(*)
+will aid in running the test suite (XML::Feed and IPC::Run respectively,
+on CPAN).
+
+(*) we hope to drop this dependency someday
+
+standard MakeMaker installation (Perl)
+--------------------------------------
+
+Once the dependencies are installed, you should be able to build and
+install the system (into /usr/local) with:
+
+        perl Makefile.PL
+        make
+        make test
+        make install # root permissions may be needed
 
 When installing Search::Xapian, make sure the underlying Xapian
 installation is not affected by an index corruption bug:
 
-        https://bugs.debian.org/808610
+        https://bugs.debian.org/808610
 
 For Debian 8.x (jessie), this means using Debian 8.5 or later.
 
@@ -69,13 +126,8 @@ public-inbox will never store unregeneratable data in Xapian
 or any other search database we might use; Xapian corruption
 will not destroy critical data.
 
-Optional Perl modules (for developers):
-
-  - XML::Feed[9]               libxml-feed-perl
-  - IPC::Run[10]               libipc-run-perl
-
-[9] - Optional, for testing Atom feeds
-[10] - Optional, for some tests (we hope to drop this dependency someday)
+See the public-inbox-overview(7) man page for the next steps once the
+installation is complete.
 
 Copyright
 ---------
diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm
index a6f516df..04cb4062 100644
--- a/lib/PublicInbox/ExtMsg.pm
+++ b/lib/PublicInbox/ExtMsg.pm
@@ -17,7 +17,7 @@ our @EXT_URL = (
         # leading "//" denotes protocol-relative (http:// or https://)
         '//marc.info/?i=%s',
         '//www.mail-archive.com/search?l=mid&q=%s',
-        'http://mid.gmane.org/%s',
+        'nntp://news.gmane.org/%s',
         'https://lists.debian.org/msgid-search/%s',
         '//docs.FreeBSD.org/cgi/mid.cgi?db=mid&id=%s',
         'https://www.w3.org/mid/%s',
diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index 3237a5ed..ec3d4f9d 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -78,7 +78,7 @@ sub meta_accessor {
         $prev = $dbh->selectrow_array($sql, undef, $key);
 
         if (defined $prev) {
-                $sql = 'UPDATE meta SET val = ? WHERE key = ? LIMIT 1';
+                $sql = 'UPDATE meta SET val = ? WHERE key = ?';
                 $dbh->do($sql, undef, $value, $key);
         } else {
                 $sql = 'INSERT INTO meta (key,val) VALUES (?,?)';
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index ace56e7a..cdbd8e98 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -115,6 +115,7 @@ sub args_ok ($$) {
 sub process_line ($$) {
         my ($self, $l) = @_;
         my ($req, @args) = split(/\s+/, $l);
+        return unless defined($req);
         $req = lc($req);
         $req = eval {
                 no strict 'refs';
@@ -943,11 +944,13 @@ sub event_write {
 sub event_read {
         my ($self) = @_;
         use constant LINE_MAX => 512; # RFC 977 section 2.3
-        my $r = 1;
 
-        my $buf = $self->read(LINE_MAX) or return $self->close;
-        $self->{rbuf} .= $$buf;
-        while ($r > 0 && $self->{rbuf} =~ s/\A\s*([^\r\n]+)\r?\n//) {
+        if (index($self->{rbuf}, "\n") < 0) {
+                my $buf = $self->read(LINE_MAX) or return $self->close;
+                $self->{rbuf} .= $$buf;
+        }
+        my $r = 1;
+        while ($r > 0 && $self->{rbuf} =~ s/\A\s*([^\r\n]*)\r?\n//) {
                 my $line = $1;
                 return $self->close if $line =~ /[[:cntrl:]]/s;
                 my $t0 = now();
@@ -967,7 +970,7 @@ sub event_read {
 sub watch_read {
         my ($self, $bool) = @_;
         my $rv = $self->SUPER::watch_read($bool);
-        if ($bool && $self->{rbuf} ne '') {
+        if ($bool && index($self->{rbuf}, "\n") >= 0) {
                 # Force another read if there is a pipelined request.
                 # We don't know if the socket has anything for us to read,
                 # and we must double-check again by the time the timer fires
diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index d038dfca..5d500c1b 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -22,6 +22,7 @@ sub mbox_results {
         my ($ctx) = @_;
         my $q = PublicInbox::SearchQuery->new($ctx->{qp});
         my $x = $q->{x};
+        require PublicInbox::Mbox;
         return PublicInbox::Mbox::mbox_all($ctx, $q->{'q'}) if $x eq 'm';
         sres_top_html($ctx);
 }
@@ -35,7 +36,7 @@ sub sres_top_html {
         my $code = 200;
         # double the limit for expanded views:
         my $opts = {
-                limit => $LIM,
+                limit => $q->{l},
                 offset => $q->{o},
                 mset => 1,
                 relevance => $q->{r},
@@ -181,6 +182,7 @@ sub search_nav_bot {
         my ($mset, $q) = @_;
         my $total = $mset->get_matches_estimated;
         my $o = $q->{o};
+        my $l = $q->{l};
         my $end = $o + $mset->size;
         my $beg = $o + 1;
         my $rv = '</pre><hr><pre id=t>';
@@ -190,15 +192,15 @@ sub search_nav_bot {
         } else {
                 $rv .= "No more results, only $total";
         }
-        my $n = $o + $LIM;
+        my $n = $o + $l;
 
         if ($n < $total) {
-                my $qs = $q->qs_html(o => $n);
+                my $qs = $q->qs_html(o => $n, l => $l);
                 $rv .= qq{  <a\nhref="?$qs"\nrel=next>next</a>}
         }
         if ($o > 0) {
                 $rv .= $n < $total ? '/' : '       ';
-                my $p = $o - $LIM;
+                my $p = $o - $l;
                 my $qs = $q->qs_html(o => ($p > 0 ? $p : 0));
                 $rv .= qq{<a\nhref="?$qs"\nrel=prev>prev</a>};
         }
@@ -305,10 +307,13 @@ sub new {
         my ($class, $qp) = @_;
 
         my $r = $qp->{r};
+        my ($l) = (($qp->{l} || '') =~ /(\d+)/);
+        $l = $LIM if !$l || $l > $LIM;
         bless {
                 q => $qp->{'q'},
                 x => $qp->{x} || '',
                 o => (($qp->{o} || '0') =~ /(\d+)/),
+                l => $l,
                 r => (defined $r && $r ne '0'),
         }, $class;
 }
@@ -331,6 +336,9 @@ sub qs_html {
         if (my $o = $self->{o}) { # ignore o == 0
                 $qs .= "&amp;o=$o";
         }
+        if (my $l = $self->{l}) {
+                $qs .= "&amp;l=$l";
+        }
         if (my $r = $self->{r}) {
                 $qs .= "&amp;r";
         }
diff --git a/t/nntpd.t b/t/nntpd.t
index c6e34ed3..3698f98b 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -147,6 +147,8 @@ EOF
         is($buf, "201 server ready - post via email\r\n", 'got greeting');
         $s->autoflush(1);
 
+        ok(syswrite($s, "   \r\n"), 'wrote spaces');
+        ok(syswrite($s, "\r\n"), 'wrote nothing');
         syswrite($s, "NEWGROUPS\t19990424 000000 \033GMT\007\r\n");
         is(0, sysread($s, $buf, 4096), 'GOT EOF on cntrl');
 
diff --git a/t/psgi_search.t b/t/psgi_search.t
index 60a44bde..2f033016 100644
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -62,6 +62,16 @@ test_psgi(sub { $www->call(@_) }, sub {
         is('%C3%86var', (keys %uniq)[0], 'matches original query');
         ok(index($html, 'by &#198;var Arnfj&#246;r&#240; Bjarmason') >= 0,
                 "displayed Ævar's name properly in HTML");
+
+        my $warn = [];
+        local $SIG{__WARN__} = sub { push @$warn, @_ };
+        $res = $cb->(GET('/test/?q=s:test&l=5e'));
+        is($res->code, 200, 'successful search result');
+        is_deeply([], $warn, 'no warnings from non-numeric comparison');
+
+        $res = $cb->(POST('/test/?q=s:bogus&x=m'));
+        is($res->code, 404, 'failed search result gives 404');
+        is_deeply([], $warn, 'no warnings');
 });
 
 done_testing();