about summary refs log tree commit homepage
diff options
context:
space:
mode:
-rw-r--r--.gitignore1
-rw-r--r--Documentation/.gitignore2
-rw-r--r--Documentation/include.mk25
-rw-r--r--Documentation/public-inbox-config.pod151
-rw-r--r--Documentation/public-inbox-daemon.pod171
-rw-r--r--Documentation/public-inbox-httpd.pod40
-rw-r--r--Documentation/public-inbox-index.pod97
-rw-r--r--Documentation/public-inbox-mda.pod27
-rw-r--r--Documentation/public-inbox-nntpd.pod53
-rw-r--r--Documentation/public-inbox-overview.pod108
-rw-r--r--Documentation/public-inbox-watch.pod121
-rw-r--r--HACKING4
-rw-r--r--MANIFEST8
-rw-r--r--lib/PublicInbox/WWW.pod56
14 files changed, 849 insertions, 15 deletions
diff --git a/.gitignore b/.gitignore
index 3b333a52..6a44471f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,5 +9,6 @@
 *.1
 *.5
 *.7
+*.8
 *.html
 *.gz
diff --git a/Documentation/.gitignore b/Documentation/.gitignore
index 8ba4186a..107ad36f 100644
--- a/Documentation/.gitignore
+++ b/Documentation/.gitignore
@@ -1 +1 @@
-/public-inbox-mda.txt
+/public-inbox-*.txt
diff --git a/Documentation/include.mk b/Documentation/include.mk
index 9d2c3b09..5154d4b6 100644
--- a/Documentation/include.mk
+++ b/Documentation/include.mk
@@ -15,37 +15,58 @@ PODTEXT = pod2text
 PODTEXT_OPTS = --stderr
 podtext = $(PODTEXT) $(PODTEXT_OPTS)
 
+# MakeMaker only seems to support manpage sections 1 and 3...
 m1 =
 m1 += public-inbox-mda
+m1 += public-inbox-httpd
+m1 += public-inbox-nntpd
+m1 += public-inbox-watch
 m5 =
+m5 += public-inbox-config
 m7 =
+m7 += public-inbox-overview
+m8 =
+m8 += public-inbox-daemon
 
 man1 := $(addsuffix .1, $(m1))
 man5 := $(addsuffix .5, $(m5))
 man7 := $(addsuffix .7, $(m7))
+man8 := $(addsuffix .8, $(m8))
 
 all:: man html
 
-man: $(man1) $(man5) $(man7)
+man: $(man1) $(man5) $(man7) $(man8)
 
+prefix ?= $(PREFIX)
 prefix ?= $(HOME)
 mandir ?= $(prefix)/share/man
 man1dir = $(mandir)/man1
 man5dir = $(mandir)/man5
 man7dir = $(mandir)/man7
+man8dir = $(mandir)/man8
 
 install-man: man
         test -z "$(man1)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man1dir)
         test -z "$(man5)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man5dir)
         test -z "$(man7)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man7dir)
+        test -z "$(man8)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man8dir)
         test -z "$(man1)" || $(INSTALL) -m 644 $(man1) $(DESTDIR)$(man1dir)
         test -z "$(man5)" || $(INSTALL) -m 644 $(man5) $(DESTDIR)$(man5dir)
         test -z "$(man7)" || $(INSTALL) -m 644 $(man7) $(DESTDIR)$(man7dir)
+        test -z "$(man8)" || $(INSTALL) -m 644 $(man8) $(DESTDIR)$(man8dir)
+
+doc_install :: install-man
 
 %.1 %.5 %.7 %.8 : Documentation/%.pod
         $(podman) -s $(subst .,,$(suffix $@)) $< $@+ && mv $@+ $@
 
-mantxt = $(addprefix Documentation/, $(addsuffix .txt, $(m1)))
+manuals :=
+manuals += $(m1)
+manuals += $(m5)
+manuals += $(m7)
+manuals += $(m8)
+
+mantxt = $(addprefix Documentation/, $(addsuffix .txt, $(manuals)))
 docs += $(mantxt)
 
 all :: $(mantxt)
diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod
new file mode 100644
index 00000000..00376457
--- /dev/null
+++ b/Documentation/public-inbox-config.pod
@@ -0,0 +1,151 @@
+=head1 NAME
+
+public-inbox-config - public-inbox config file description
+
+=head1 SYNOPSIS
+
+~/.public-inbox/config
+
+=head1 DESCRIPTION
+
+The public-inbox config file is parseable by L<git-config(1)>.
+This is a global configuration file for mapping/discovering
+all public-inboxes used by a particular user.
+
+=head1 CONFIGURATION FILE
+
+=head2 EXAMPLE
+
+        [publicinbox "test"]
+                mainrepo = /home/user/path/to/test.git
+                ; multiple addresses are supported
+                address = test@example.com
+                ; address = alternate@example.com
+                url = http://example.com/test
+                newsgroup = inbox.test
+
+=head2 VARIABLES
+
+=over 8
+
+=item publicinbox.<name>.address
+
+The email address of the public-inbox.  May be specified
+more than once for merging multiple mailing lists (or migrating
+to new addresses).  This must be specified at least once,
+the first value will be considered the primary address for
+informational purposes.
+
+Default: none, required
+
+=item publicinbox.<name>.mainrepo
+
+The absolute path to the git repository which hosts the
+public-inbox.  This must be specified once.
+
+Default: none, required
+
+=item publicinbox.<name>.url
+
+The primary URL for hosting the HTTP/HTTPS archives.
+Additional HTTP/HTTPS URLs may be specified via
+C<$GIT_DIR/cloneurl> as documented in L<gitweb(1)>
+
+Default: none, optional
+
+=item publicinbox.<name>.newsgroup
+
+The NNTP group name for use with L<public-inbox-nntpd(8)>.  This
+may be any newsgroup name with hierarchies delimited by '.'.
+For example, the newsgroup for L<mailto:meta@public-inbox.org>
+is: C<inbox.comp.mail.public-inbox.meta>
+
+Omitting this for the given inbox will prevent the group from
+being read by L<public-inbox-nntpd(1)>
+
+Default: none, optional
+
+=item publicinbox.<name>.watch
+
+A location for L<public-inbox-watch(1)> to watch.  Currently,
+only C<maildir:> paths are supported:
+
+        [publicinbox "test"]
+                watch = maildir:/path/to/maildirs/.INBOX.test/
+
+Default: none; only for L<public-inbox-watch(1)> users
+
+=item publicinbox.<name>.watchheader
+
+        [publicinbox "test"]
+                watchheader = List-Id:<test.example.com>
+
+Default: none; only for L<public-inbox-watch(1)> users
+
+=item publicinbox.<name>.nntpmirror
+
+This may be the full NNTP URL of an independently-run mirror.
+For example, the https://public-inbox.org/meta/ inbox is
+mirrored by Gmane at
+C<nntp://news.gmane.org/gmane.mail.public-inbox.general>
+
+Default: none
+
+=item publicinboxwatch.spamcheck
+
+This may be set to C<spamc> to enable the use of SpamAssassin
+L<spamc(1)> for filtering spam before it is imported into git
+history.  Other spam filtering backends may be supported in
+the future.
+
+Default: none
+
+=item publicinboxwatch.watchspam
+
+This may be set to C<spamc> to enable the use of SpamAssassin
+L<spamc(1)> for filtering spam before it is imported into git
+history.  Other spam filtering backends may be supported in
+the future.  This only affects L<public-inbox-watch(1)>.
+
+Default: none
+
+=item publicinbox.nntpserver
+
+Set this to point to the address of the L<public-inbox-nntpd(1)>
+instance.  This is used to advertise the existence of the NNTP
+presnce in the L<PublicInbox::WWW> HTML interface.
+
+Multiple values are allowed for servers with multiple
+addresses or mirrors.
+
+Default: none
+
+=back
+
+=head1 ENVIRONMENT
+
+=over 8
+
+=item PI_CONFIG
+
+Used to override the default "~/.public-inbox/config" value.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2016 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<git(1)>, L<git-config(1)>, L<public-inbox-daemon(8)>,
+L<public-inbox-mda(1)>, L<public-inbox-watch(1)>
diff --git a/Documentation/public-inbox-daemon.pod b/Documentation/public-inbox-daemon.pod
new file mode 100644
index 00000000..42beda69
--- /dev/null
+++ b/Documentation/public-inbox-daemon.pod
@@ -0,0 +1,171 @@
+=head1 NAME
+
+public-inbox-daemon - common usage for public-inbox network daemons
+
+=head1 SYNOPSIS
+
+        public-inbox-httpd
+        public-inbox-nntpd
+
+=head1 DESCRIPTION
+
+This manual describes common options and behavior for
+public-inbox network daemons.  Network daemons for public-inbox
+provide read-only NNTP and HTTP access to public-inboxes.  Write
+access to a public-inbox repository will never be required to
+run these.
+
+These daemons are implemented with a common core using
+non-blocking sockets and optimized for fairness; even with
+thousands of connected clients over slow links.
+
+They also provide graceful shutdown/upgrade support to avoid
+breaking existing connections during software upgrades.
+
+These daemons may also utilize multiple pre-forked worker
+processes to take advantage of multiple CPUs.
+
+Native TLS (Transport Layer Security) support is planned.
+
+=head1 OPTIONS
+
+=over
+
+=item -l, --listen ADDRESS
+
+This takes an absolute path to a Unix socket or HOST:PORT
+to listen on.  For example, to listen to TCP connections on
+port 119, use: C<-l 0.0.0.0:119>.  This may also point to
+a Unix socket (C<-l /path/to/http.sock>) for a reverse proxy
+like L<nginx(1)> to use.
+
+May be specified multiple times to allow listening on multiple
+sockets.
+
+Default: server-dependent unless socket activation is used with
+L<systemd(1)> or similar (see L<systemd.socket(5)>).
+
+=item -1, --stdout PATH
+
+Specify an appendable path to redirect stdout descriptor (1) to.
+Using this is preferable to setting up the redirect externally
+(e.g. E<gt>E<gt>/path/to/log in shell) since it allows
+SIGUSR1 to be handled (see L<SIGNALS/SIGNALS> below).
+
+Default: /dev/null
+
+=item -2, --stderr PATH
+
+Like C<--stdout>, but for the stderr descriptor (2).
+
+=item -W, --worker-processes
+
+Set the number of worker processes.
+
+Normally, this should match the number of CPUs on the system to
+take full advantage of the hardware.  However, users of
+memory-constrained systems may want to lower this.
+
+Setting this to zero (C<-W0>) disables the master/worker split;
+saving some memory but removing the ability to use SIGTTIN
+to increase worker processes or have the worker restarted by
+the master on crashes.
+
+Default: 1
+
+=back
+
+=head1 SIGNALS
+
+Most of our signal handling behavior is copied from L<nginx(1)>
+and/or L<starman(1)>; so it is possible to reuse common scripts
+for managing them.
+
+=over 8
+
+=item SIGUSR1
+
+Reopens log files pointed to by --stdout and --stderr options.
+
+=item SIGUSR2
+
+Spawn a new process with the intention to replace the running one.
+See L</UPGRADING> below.
+
+=item SIGHUP
+
+Reload config files associated with the process.
+(FIXME: not tested for -httpd, yet)
+
+=item SIGTTIN
+
+Increase the number of running workers processes by one.
+
+=item SIGTTOU
+
+Decrease the number of running worker processes by one.
+
+=item SIGWINCH
+
+Stop all running worker processes.   SIGHUP or SIGTTIN
+may be used to restart workers.
+
+=item SIGQUIT
+
+Gracefully terminate the running process.
+
+=back
+
+SIGTTOU, SIGTTIN, SIGWINCH all have no effect when worker
+processes are disabled with C<-W0> on the command-line.
+
+=head1 ENVIRONMENT
+
+=over 8
+
+=item PI_CONFIG
+
+The default config file, normally "~/.public-inbox/config".
+See L<public-inbox-config(5)>
+
+=item LISTEN_FDS, LISTEN_PID
+
+Used by systemd (and compatible) installations for socket
+activation.  See L<systemd.socket(5)> and L<sd_listen_fds(3)>.
+
+=back
+
+=head1 UPGRADING
+
+There are two ways to upgrade a running process.
+
+Users of process management systems with socket activation
+(L<systemd(1)> or similar) may rely on multiple instances For
+systemd, this means using two (or more) '@' instances for each
+service (e.g. C<SERVICENAME@INSTANCE>) as documented in
+L<systemd.unit(5)>.
+
+Users of traditional SysV init may use SIGUSR2 to spawn
+a replacement process and gracefully terminate the old
+process using SIGQUIT.
+
+In either case, the old process will not truncate running
+responses; so responses to expensive requests do not get
+interrupted and lost.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2013-2016 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<public-inbox-httpd(1)>, L<public-inbox-nntpd(1)>
diff --git a/Documentation/public-inbox-httpd.pod b/Documentation/public-inbox-httpd.pod
new file mode 100644
index 00000000..8605d747
--- /dev/null
+++ b/Documentation/public-inbox-httpd.pod
@@ -0,0 +1,40 @@
+=head1 NAME
+
+public-inbox-httpd - PSGI server optimized for public-inbox
+
+=head1 SYNOPSIS
+
+B<public-inbox-httpd> [OPTIONS] [/path/to/myapp.psgi]
+
+=head1 DESCRIPTION
+
+public-inbox-httpd is a PSGI/Plack server supporting HTTP/1.1
+and HTTP/1.0.  It uses options and environment variables common
+to all L<public-inbox-daemon(8)> implementations in addition to
+the PSGI file.
+
+If a PSGI file is not specified, L<PublicInbox::WWW> is
+loaded with a default middleware stack consisting of
+L<Plack::Middleware::Deflater>,
+L<Plack::Middleware::ReverseProxy>, and
+L<Plack::Middleware::Head>
+
+This may point to a PSGI file for supporting generic PSGI apps.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2013-2016 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<git(1)>, L<git-config(1)>, L<public-inbox-daemon(8)>,
+L<Plack>
diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod
new file mode 100644
index 00000000..554346de
--- /dev/null
+++ b/Documentation/public-inbox-index.pod
@@ -0,0 +1,97 @@
+=head1 NAME
+
+public-inbox-index - create and update search indices
+
+=head1 SYNOPSIS
+
+public-inbox-index GIT_DIR
+
+=head1 DESCRIPTION
+
+public-inbox-index creates and updates the search and NNTP
+article number database used by the read-only public-inbox HTTP
+and NNTP interfaces.  Currently, this requires L<Search::Xapian>
+and L<DBD::SQlite> and L<DBI> Perl modules.
+
+Once the initial indices are created by public-inbox-index,
+L<public-inbox-mda(1)> and L<public-inbox-watch(1)> will
+automatically maintain them.
+
+Running this manually to update indices is only required if
+relying on L<git-fetch(1)> to mirror an existing public-inbox.
+
+Having a search and article number database is essential to
+running the NNTP interface, and strongly recommended for the
+HTTP interface as it provides thread grouping in addition
+to normal search functionality.
+
+=head1 FILES
+
+All public-inbox-specific files are contained within the
+C<$GIT_DIR/public-inbox/> directory.  All files are expected to
+grow in size as more messages are archived, so using compaction
+commands (e.g. L<xapian-compact(1)>) is not recommended unless
+the list is no longer active.
+
+=item $GIT_DIR/public-inbox/msgmap.sqlite3
+
+The stable NNTP article number to Message-ID mapping is
+stored in an SQLite3 database.
+
+This is required for users of L<public-inbox-nntpd(1)>, but
+users of the L<PublicInbox::WWW> interface will find it
+useful for attempting recovery from copy-paste truncations of
+URLs containing long Message-IDs.
+
+Avoid removing this file and regenerating it; it may cause
+existing NNTP readers to lose sync and miss (or duplicate)
+messages.
+
+This file is relatively small, and typically less than 5%
+of the space of the mail stored in a packed git repository.
+
+=item $GIT_DIR/public-inbox/xapian*
+
+The database used by L<Search::Xapian>.  This directory name is
+followed by a number indicating the index schema version this
+installation of public-inbox uses.
+
+These directories may be safely deleted or removed in full
+while the NNTP and HTTP interfaces are no longer accessing
+them.
+
+In addition to providing a search interface for the HTTP
+interface, the Xapian database is used to group and combine
+related messages into threads.  For NNTP servers, it also
+provides a cache of metadata and header information often
+requested by NNTP clients.
+
+This directory is large, often two to three times the size of
+the objects stored in a packed git repository.
+
+=head1 ENVIRONMENT
+
+=over 8
+
+=item PI_CONFIG
+
+Used to override the default "~/.public-inbox/config" value.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2016 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<Search::Xapian>, L<DBD::SQLite>
diff --git a/Documentation/public-inbox-mda.pod b/Documentation/public-inbox-mda.pod
index 18fcdd08..3a43a1ce 100644
--- a/Documentation/public-inbox-mda.pod
+++ b/Documentation/public-inbox-mda.pod
@@ -1,6 +1,6 @@
 =head1 NAME
 
-public-inbox-mda - mail delivery for public-inbox
+public-inbox-mda - mail delivery agent for public-inbox
 
 =head1 SYNOPSIS
 
@@ -9,7 +9,8 @@ B<public-inbox-mda> E<lt>MESSAGE
 =head1 DESCRIPTION
 
 Mail Delivery Agent (MDA) for public-inbox installations.
-Each system user may have their own public-inbox instances
+Each system user may have their own public-inbox instances.
+This may be invoked via L<procmail(1)> or similar tools.
 
 =head1 ENVIRONMENT
 
@@ -17,33 +18,37 @@ Each system user may have their own public-inbox instances
 
 =item ORIGINAL_RECIPIENT
 
-the original recipient email address, from Postfix
+The original recipient email address, set by the MTA.  Postfix
+sets it by default, untested on other MTAs.
 
 =item PI_CONFIG
 
-config file. default: ~/.public-inbox/config
+Per-user config file parseable by L<git-config(1)>.
+See L<public-inbox-config(5)>.
+
+Default: ~/.public-inbox/config
 
 =item PI_EMERGENCY
 
-emergency destination.  default: ~/.public-inbox/emergency/
+emergency Maildir destination.
 
-=back
+Default: ~/.public-inbox/emergency/
 
-=head1 PI_CONFIG FILE
+=back
 
-This is a config file parseable by L<git-config(1)>.
 
 =head1 CONTACT
 
-All feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
 
-The mail archives are hosted at L<http://public-inbox.org/meta/>
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
 
 =head1 COPYRIGHT
 
 Copyright 2013-2016 all contributors L<mailto:meta@public-inbox.org>
 
-License: AGPL-3.0+ L<http://www.gnu.org/licenses/agpl-3.0.txt>
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
 =head1 SEE ALSO
 
diff --git a/Documentation/public-inbox-nntpd.pod b/Documentation/public-inbox-nntpd.pod
new file mode 100644
index 00000000..2f9dbabf
--- /dev/null
+++ b/Documentation/public-inbox-nntpd.pod
@@ -0,0 +1,53 @@
+=head1 NAME
+
+public-inbox-nntpd - NNTP server for sharing public-inbox
+
+=head1 SYNOPSIS
+
+B<public-inbox-nntpd> [OPTIONS]
+
+=head1 DESCRIPTION
+
+public-inbox-nntpd provides a read-only NNTP daemon for
+public-inbox.  It uses options and environment variables common
+to all L<public-inbox-daemon(8)> implementations.
+
+The default configuration will never require write access
+tto the directory where the public-inbox is stored, so it
+may be run as a different user than the user running
+L<public-inbox-watch(1)>, L<public-inbox-mda(1)>, or
+L<git-fetch(1)>.
+
+=head1 CONFIGURATION
+
+These configuration knobs should be used in the
+L<public-inbox-config(5)>
+
+=over 8
+
+=item publicinbox.<name>.newsgroup
+
+=item publicinbox.nntpserver
+
+=back
+
+See L<public-inbox-config(5)> for documentation on them.
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>,
+L<nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta>,
+L<nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta>
+
+=head1 COPYRIGHT
+
+Copyright 2013-2016 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<git(1)>, L<git-config(1)>, L<public-inbox-daemon(8)>,
+L<public-inbox-config(5)>
diff --git a/Documentation/public-inbox-overview.pod b/Documentation/public-inbox-overview.pod
new file mode 100644
index 00000000..2c97f876
--- /dev/null
+++ b/Documentation/public-inbox-overview.pod
@@ -0,0 +1,108 @@
+=head1 NAME
+
+public-inbox-overview - an overview of public-inbox
+
+=head1 DESCRIPTION
+
+public-inbox consists of many parts which may be used
+independently or in conjunction of each other for:
+
+=over 4
+
+=item 1
+
+Mirroring existing public-inboxes.
+
+=item 2
+
+Mirroring mailing lists directly.
+
+=item 3
+
+Hosting standalone.
+
+=back
+
+=head2 Mirroring existing public-inboxes
+
+Mirroring existing public-inboxes is the easiest way to get
+started.  Your mirror will remain dependent on the REMOTE_URL
+you are mirroring and you only need to use two new commands in
+addition to common L<git(1)> commands.
+
+        git clone --mirror REMOTE_URL /path/to/repo.git
+
+        # The following should create the necessary entry in
+        # ~/.public-inbox/config
+        public-inbox-init NAME /path/to/repo.git MY_URL LIST_ADDRESS
+
+        # Optional but strongly recommended for hosting HTTP
+        # (and required for NNTP)
+        # enable search (requires Search::Xapian and DBD::SQLite)
+        public-inbox-index /path/to/repo.git
+
+        # Periodically update the repo with the following commands
+        # to update the git repo and index new messages:
+        cd /path/to/repo.git && git fetch && public-inbox-index
+
+See L</"Hosting public-inboxes"> below for info on how to expose
+your mirror to other readers.
+
+=head2 Mirroring mailing lists directly
+
+Mirroring existing mailing lists may be done by any reader
+of a mailing list using L<public-inbox-watch(1)>.
+
+        # This will create a new git repository:
+        public-inbox-init NAME /path/to/repo.git MY_URL LIST_ADDRESS
+
+Then, see the L<public-inbox-watch(1)> manual for configuring
+C<watch>, C<watchheader>, and the optional C<spamcheck> and
+C<watchspam> entries.
+
+You will need to leave L<public-inbox-watch(1)> running to
+keep the mailbox up-to-date as messages are delivered to
+the mailing list.
+
+Running L<public-inbox-index(1)> to create search indices
+is recommended.  L<public-inbox-watch(1)> will automatically
+maintain the indices if they were created by
+L<public-inbox-index(1)>
+
+        public-inbox-index /path/to/repo.git
+
+=head2 Hosting standalone
+
+Using L<public-inbox-init(1)> to initialize the inbox as in the
+other methods is recommended.  See L<public-inbox-mda(1)> for
+more details; but this also requires MTA-specific knowledge.
+
+=head2 Hosting public-inboxes
+
+Since public-inboxes are git repositories, they may be served to
+remote clients via L<git-daemon(1)> as well as specialized HTTP
+and NNTP daemons distributed with public-inbox.
+
+See L<public-inbox-httpd(1)> and L<public-inbox-nntpd(1)>
+for more information on using these daemons.
+
+Hosting a public-inbox over HTTP or NNTP will never require
+write access to any files in the git repository, including
+the search indices or article number map database.
+
+Users familiar with PSGI and L<Plack> may also use
+L<PublicInbox::WWW> with the preferred server instead of
+L<public-inbox-httpd(1)>
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2016 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
diff --git a/Documentation/public-inbox-watch.pod b/Documentation/public-inbox-watch.pod
new file mode 100644
index 00000000..404303e8
--- /dev/null
+++ b/Documentation/public-inbox-watch.pod
@@ -0,0 +1,121 @@
+=head1 NAME
+
+public-inbox-watch - mailbox watcher for public-inbox
+
+=head1 SYNOPSIS
+
+B<public-inbox-watch>
+
+In ~/.public-inbox/config:
+
+        [publicinbox "test"]
+                ; generic public-inbox-config keys:
+                address = test@example.com
+                url = http://example.com/test
+                mainrepo = /path/to/test.example.com.git
+
+                ; config keys specific to public-inbox-watch:
+                watch = maildir:/path/to/maildirs/.INBOX.test/
+                watchheader = List-Id:<test.example.com>
+
+        [publicinboxwatch]
+                ; optional, enable use of spamc(1) for checking:
+                spamcheck = spamc
+
+                ; optional, emails marked as read which appear
+                ; here will be trained as spam and deleted from
+                ; the mainrepos of any public-inboxes which are
+                ; configured for watch.
+                ; This is global for all publicinbox.* sections
+                watchspam = maildir:/path/to/maildirs/.INBOX.spam
+
+=head1 DESCRIPTION
+
+public-inbox-watch allows watching a mailbox (currently only
+Maildir) for the arrival of new messages and automatically
+importing them into a public-inbox (git) repository.
+public-inbox-watch is useful in situations when a user wishes to
+mirror an existing mailing list, but has no access to run
+L<public-inbox-mda(1)> on a server.  Unlike public-inbox-mda
+which is invoked once per-message, public-inbox-watch is a
+persistent process, making it faster for after-the-fact imports
+of large Maildirs.
+
+Upon startup, it scans the mailbox for new messages to be
+imported while it was not running.
+
+Currently, only Maildirs are supported and the
+L<Filesys::Notify::Simple> Perl module is required.
+
+For now, IMAP users should use tools such as L<mbsync(1)>
+or L<offlineimap(1)> to bidirectionally sync their IMAP
+folders to Maildirs for public-inbox-watch.
+
+public-inbox-watch should be run inside a L<screen(1)> session
+or as a L<systemd(1)> service.  Errors are emitted to stderr.
+
+=head1 OPTIONS
+
+public-inbox-watch takes no command-line options.
+
+=head1 CONFIGURATION
+
+These configuration knobs should be used in the
+L<public-inbox-config(5)>
+
+=over 8
+
+=item publicinbox.<name>.watch
+
+=item publicinbox.<name>.watchheader
+
+=item publicinboxwatch.spamcheck
+
+=item publicinboxwatch.watchspam
+
+=back
+
+See L<public-inbox-config(5)> for documentation on them.
+
+=head1 SIGNALS
+
+=over 8
+
+=item SIGHUP
+
+Reload the config file (default: ~/.public-inbox/config)
+
+=item SIGUSR1
+
+Rescan all watched mailboxes.  This is done automatically after
+startup.
+
+=back
+
+=head1 ENVIRONMENT
+
+=over 8
+
+=item PI_CONFIG
+
+config file. default: ~/.public-inbox/config
+See L<public-inbox-config(5)>
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright 2016 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<public-inbox-config(5)>
diff --git a/HACKING b/HACKING
index 3c2235a4..3435775c 100644
--- a/HACKING
+++ b/HACKING
@@ -36,12 +36,14 @@ In general, we favor mature and well-tested old things rather than
 the shiny new.
 
 Avoid relying on compiled modules too much.  Even if it is Free,
-compiled code makes packages more expensive to audit, build, and
+compiled code makes packages more expensive to audit, build,
 distribute and verify.  public-inbox itself will only be implemented
 in scripting languages (currently Perl 5).
 
 Performance should be reasonably good for server administrators, too,
 and we will sacrifice features to achieve predictable performance.
+Encouraging folks to self-host will be easier with lower hardware
+requirements.
 
 See design_www.txt and design_notes.txt in the Documentation/
 directory for design decisions made during development.
diff --git a/MANIFEST b/MANIFEST
index 306945a2..c39fa261 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -5,7 +5,14 @@ Documentation/dc-dlvr-spam-flow.txt
 Documentation/design_notes.txt
 Documentation/design_www.txt
 Documentation/include.mk
+Documentation/public-inbox-config.pod
+Documentation/public-inbox-daemon.pod
+Documentation/public-inbox-httpd.pod
+Documentation/public-inbox-index.pod
 Documentation/public-inbox-mda.pod
+Documentation/public-inbox-nntpd.pod
+Documentation/public-inbox-overview.pod
+Documentation/public-inbox-watch.pod
 Documentation/txt2pre
 HACKING
 INSTALL
@@ -79,6 +86,7 @@ lib/PublicInbox/Thread.pm
 lib/PublicInbox/Unsubscribe.pm
 lib/PublicInbox/View.pm
 lib/PublicInbox/WWW.pm
+lib/PublicInbox/WWW.pod
 lib/PublicInbox/WatchMaildir.pm
 lib/PublicInbox/WwwAttach.pm
 lib/PublicInbox/WwwStream.pm
diff --git a/lib/PublicInbox/WWW.pod b/lib/PublicInbox/WWW.pod
new file mode 100644
index 00000000..a1d33a3b
--- /dev/null
+++ b/lib/PublicInbox/WWW.pod
@@ -0,0 +1,56 @@
+=head1 NAME
+
+PublicInbox::WWW - PSGI interface for public-inbox
+
+=head1 SYNOPSIS
+
+In your .psgi file:
+
+        use PublicInbox::WWW;
+
+        my $www = PublicInbox::WWW->new;
+        builder {
+                enable 'Head';
+                mount '/inboxes' => sub { $www->call(@_) };
+        };
+
+=head1 DESCRIPTION
+
+The PSGI web interface for public-inbox.
+
+Using this directly is not needed unless you wish to customize
+your public-inbox PSGI deployment or are using a PSGI server
+other than L<public-inbox-httpd(1)>.
+
+While this PSGI application works with all PSGI/Plack web
+servers such as L<starman(1)>, L<starlet(1)> or L<twiggy(1)>;
+PublicInbox::WWW takes advantage of currently-undocumented APIs
+of L<public-inbox-httpd(1)> to improve fairness when serving
+large responses for thread views and git clones.
+
+=head1 ENVIRONMENT
+
+=over 8
+
+=item PI_CONFIG
+
+Used to override the default "~/.public-inbox/config" value.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/>
+and L<http://hjrcffqmbrq6wope.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016 all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<http://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<http://plackperl.org/>, L<Plack>, L<public-inbox-httpd(1)>