From 66c6699142d95cce9e5c9d49e0498161a776cc22 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 7 Sep 2016 00:47:15 +0000 Subject: doc: new docs for user-level commands Hopefully more folks can download and run public-inbox, nowadays. --- Documentation/.gitignore | 2 +- Documentation/include.mk | 25 ++++- Documentation/public-inbox-config.pod | 151 ++++++++++++++++++++++++++++ Documentation/public-inbox-daemon.pod | 171 ++++++++++++++++++++++++++++++++ Documentation/public-inbox-httpd.pod | 40 ++++++++ Documentation/public-inbox-index.pod | 97 ++++++++++++++++++ Documentation/public-inbox-mda.pod | 27 +++-- Documentation/public-inbox-nntpd.pod | 53 ++++++++++ Documentation/public-inbox-overview.pod | 108 ++++++++++++++++++++ Documentation/public-inbox-watch.pod | 121 ++++++++++++++++++++++ 10 files changed, 781 insertions(+), 14 deletions(-) create mode 100644 Documentation/public-inbox-config.pod create mode 100644 Documentation/public-inbox-daemon.pod create mode 100644 Documentation/public-inbox-httpd.pod create mode 100644 Documentation/public-inbox-index.pod create mode 100644 Documentation/public-inbox-nntpd.pod create mode 100644 Documentation/public-inbox-overview.pod create mode 100644 Documentation/public-inbox-watch.pod (limited to 'Documentation') diff --git a/Documentation/.gitignore b/Documentation/.gitignore index 8ba4186a..107ad36f 100644 --- a/Documentation/.gitignore +++ b/Documentation/.gitignore @@ -1 +1 @@ -/public-inbox-mda.txt +/public-inbox-*.txt diff --git a/Documentation/include.mk b/Documentation/include.mk index 9d2c3b09..5154d4b6 100644 --- a/Documentation/include.mk +++ b/Documentation/include.mk @@ -15,37 +15,58 @@ PODTEXT = pod2text PODTEXT_OPTS = --stderr podtext = $(PODTEXT) $(PODTEXT_OPTS) +# MakeMaker only seems to support manpage sections 1 and 3... m1 = m1 += public-inbox-mda +m1 += public-inbox-httpd +m1 += public-inbox-nntpd +m1 += public-inbox-watch m5 = +m5 += public-inbox-config m7 = +m7 += public-inbox-overview +m8 = +m8 += public-inbox-daemon man1 := $(addsuffix .1, $(m1)) man5 := $(addsuffix .5, $(m5)) man7 := $(addsuffix .7, $(m7)) +man8 := $(addsuffix .8, $(m8)) all:: man html -man: $(man1) $(man5) $(man7) +man: $(man1) $(man5) $(man7) $(man8) +prefix ?= $(PREFIX) prefix ?= $(HOME) mandir ?= $(prefix)/share/man man1dir = $(mandir)/man1 man5dir = $(mandir)/man5 man7dir = $(mandir)/man7 +man8dir = $(mandir)/man8 install-man: man test -z "$(man1)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man1dir) test -z "$(man5)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man5dir) test -z "$(man7)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man7dir) + test -z "$(man8)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man8dir) test -z "$(man1)" || $(INSTALL) -m 644 $(man1) $(DESTDIR)$(man1dir) test -z "$(man5)" || $(INSTALL) -m 644 $(man5) $(DESTDIR)$(man5dir) test -z "$(man7)" || $(INSTALL) -m 644 $(man7) $(DESTDIR)$(man7dir) + test -z "$(man8)" || $(INSTALL) -m 644 $(man8) $(DESTDIR)$(man8dir) + +doc_install :: install-man %.1 %.5 %.7 %.8 : Documentation/%.pod $(podman) -s $(subst .,,$(suffix $@)) $< $@+ && mv $@+ $@ -mantxt = $(addprefix Documentation/, $(addsuffix .txt, $(m1))) +manuals := +manuals += $(m1) +manuals += $(m5) +manuals += $(m7) +manuals += $(m8) + +mantxt = $(addprefix Documentation/, $(addsuffix .txt, $(manuals))) docs += $(mantxt) all :: $(mantxt) diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod new file mode 100644 index 00000000..00376457 --- /dev/null +++ b/Documentation/public-inbox-config.pod @@ -0,0 +1,151 @@ +=head1 NAME + +public-inbox-config - public-inbox config file description + +=head1 SYNOPSIS + +~/.public-inbox/config + +=head1 DESCRIPTION + +The public-inbox config file is parseable by L. +This is a global configuration file for mapping/discovering +all public-inboxes used by a particular user. + +=head1 CONFIGURATION FILE + +=head2 EXAMPLE + + [publicinbox "test"] + mainrepo = /home/user/path/to/test.git + ; multiple addresses are supported + address = test@example.com + ; address = alternate@example.com + url = http://example.com/test + newsgroup = inbox.test + +=head2 VARIABLES + +=over 8 + +=item publicinbox..address + +The email address of the public-inbox. May be specified +more than once for merging multiple mailing lists (or migrating +to new addresses). This must be specified at least once, +the first value will be considered the primary address for +informational purposes. + +Default: none, required + +=item publicinbox..mainrepo + +The absolute path to the git repository which hosts the +public-inbox. This must be specified once. + +Default: none, required + +=item publicinbox..url + +The primary URL for hosting the HTTP/HTTPS archives. +Additional HTTP/HTTPS URLs may be specified via +C<$GIT_DIR/cloneurl> as documented in L + +Default: none, optional + +=item publicinbox..newsgroup + +The NNTP group name for use with L. This +may be any newsgroup name with hierarchies delimited by '.'. +For example, the newsgroup for L +is: C + +Omitting this for the given inbox will prevent the group from +being read by L + +Default: none, optional + +=item publicinbox..watch + +A location for L to watch. Currently, +only C paths are supported: + + [publicinbox "test"] + watch = maildir:/path/to/maildirs/.INBOX.test/ + +Default: none; only for L users + +=item publicinbox..watchheader + + [publicinbox "test"] + watchheader = List-Id: + +Default: none; only for L users + +=item publicinbox..nntpmirror + +This may be the full NNTP URL of an independently-run mirror. +For example, the https://public-inbox.org/meta/ inbox is +mirrored by Gmane at +C + +Default: none + +=item publicinboxwatch.spamcheck + +This may be set to C to enable the use of SpamAssassin +L for filtering spam before it is imported into git +history. Other spam filtering backends may be supported in +the future. + +Default: none + +=item publicinboxwatch.watchspam + +This may be set to C to enable the use of SpamAssassin +L for filtering spam before it is imported into git +history. Other spam filtering backends may be supported in +the future. This only affects L. + +Default: none + +=item publicinbox.nntpserver + +Set this to point to the address of the L +instance. This is used to advertise the existence of the NNTP +presnce in the L HTML interface. + +Multiple values are allowed for servers with multiple +addresses or mirrors. + +Default: none + +=back + +=head1 ENVIRONMENT + +=over 8 + +=item PI_CONFIG + +Used to override the default "~/.public-inbox/config" value. + +=back + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2016 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L, L, L, +L, L diff --git a/Documentation/public-inbox-daemon.pod b/Documentation/public-inbox-daemon.pod new file mode 100644 index 00000000..42beda69 --- /dev/null +++ b/Documentation/public-inbox-daemon.pod @@ -0,0 +1,171 @@ +=head1 NAME + +public-inbox-daemon - common usage for public-inbox network daemons + +=head1 SYNOPSIS + + public-inbox-httpd + public-inbox-nntpd + +=head1 DESCRIPTION + +This manual describes common options and behavior for +public-inbox network daemons. Network daemons for public-inbox +provide read-only NNTP and HTTP access to public-inboxes. Write +access to a public-inbox repository will never be required to +run these. + +These daemons are implemented with a common core using +non-blocking sockets and optimized for fairness; even with +thousands of connected clients over slow links. + +They also provide graceful shutdown/upgrade support to avoid +breaking existing connections during software upgrades. + +These daemons may also utilize multiple pre-forked worker +processes to take advantage of multiple CPUs. + +Native TLS (Transport Layer Security) support is planned. + +=head1 OPTIONS + +=over + +=item -l, --listen ADDRESS + +This takes an absolute path to a Unix socket or HOST:PORT +to listen on. For example, to listen to TCP connections on +port 119, use: C<-l 0.0.0.0:119>. This may also point to +a Unix socket (C<-l /path/to/http.sock>) for a reverse proxy +like L to use. + +May be specified multiple times to allow listening on multiple +sockets. + +Default: server-dependent unless socket activation is used with +L or similar (see L). + +=item -1, --stdout PATH + +Specify an appendable path to redirect stdout descriptor (1) to. +Using this is preferable to setting up the redirect externally +(e.g. EE/path/to/log in shell) since it allows +SIGUSR1 to be handled (see L below). + +Default: /dev/null + +=item -2, --stderr PATH + +Like C<--stdout>, but for the stderr descriptor (2). + +=item -W, --worker-processes + +Set the number of worker processes. + +Normally, this should match the number of CPUs on the system to +take full advantage of the hardware. However, users of +memory-constrained systems may want to lower this. + +Setting this to zero (C<-W0>) disables the master/worker split; +saving some memory but removing the ability to use SIGTTIN +to increase worker processes or have the worker restarted by +the master on crashes. + +Default: 1 + +=back + +=head1 SIGNALS + +Most of our signal handling behavior is copied from L +and/or L; so it is possible to reuse common scripts +for managing them. + +=over 8 + +=item SIGUSR1 + +Reopens log files pointed to by --stdout and --stderr options. + +=item SIGUSR2 + +Spawn a new process with the intention to replace the running one. +See L below. + +=item SIGHUP + +Reload config files associated with the process. +(FIXME: not tested for -httpd, yet) + +=item SIGTTIN + +Increase the number of running workers processes by one. + +=item SIGTTOU + +Decrease the number of running worker processes by one. + +=item SIGWINCH + +Stop all running worker processes. SIGHUP or SIGTTIN +may be used to restart workers. + +=item SIGQUIT + +Gracefully terminate the running process. + +=back + +SIGTTOU, SIGTTIN, SIGWINCH all have no effect when worker +processes are disabled with C<-W0> on the command-line. + +=head1 ENVIRONMENT + +=over 8 + +=item PI_CONFIG + +The default config file, normally "~/.public-inbox/config". +See L + +=item LISTEN_FDS, LISTEN_PID + +Used by systemd (and compatible) installations for socket +activation. See L and L. + +=back + +=head1 UPGRADING + +There are two ways to upgrade a running process. + +Users of process management systems with socket activation +(L or similar) may rely on multiple instances For +systemd, this means using two (or more) '@' instances for each +service (e.g. C) as documented in +L. + +Users of traditional SysV init may use SIGUSR2 to spawn +a replacement process and gracefully terminate the old +process using SIGQUIT. + +In either case, the old process will not truncate running +responses; so responses to expensive requests do not get +interrupted and lost. + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2013-2016 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L, L diff --git a/Documentation/public-inbox-httpd.pod b/Documentation/public-inbox-httpd.pod new file mode 100644 index 00000000..8605d747 --- /dev/null +++ b/Documentation/public-inbox-httpd.pod @@ -0,0 +1,40 @@ +=head1 NAME + +public-inbox-httpd - PSGI server optimized for public-inbox + +=head1 SYNOPSIS + +B [OPTIONS] [/path/to/myapp.psgi] + +=head1 DESCRIPTION + +public-inbox-httpd is a PSGI/Plack server supporting HTTP/1.1 +and HTTP/1.0. It uses options and environment variables common +to all L implementations in addition to +the PSGI file. + +If a PSGI file is not specified, L is +loaded with a default middleware stack consisting of +L, +L, and +L + +This may point to a PSGI file for supporting generic PSGI apps. + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2013-2016 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L, L, L, +L diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod new file mode 100644 index 00000000..554346de --- /dev/null +++ b/Documentation/public-inbox-index.pod @@ -0,0 +1,97 @@ +=head1 NAME + +public-inbox-index - create and update search indices + +=head1 SYNOPSIS + +public-inbox-index GIT_DIR + +=head1 DESCRIPTION + +public-inbox-index creates and updates the search and NNTP +article number database used by the read-only public-inbox HTTP +and NNTP interfaces. Currently, this requires L +and L and L Perl modules. + +Once the initial indices are created by public-inbox-index, +L and L will +automatically maintain them. + +Running this manually to update indices is only required if +relying on L to mirror an existing public-inbox. + +Having a search and article number database is essential to +running the NNTP interface, and strongly recommended for the +HTTP interface as it provides thread grouping in addition +to normal search functionality. + +=head1 FILES + +All public-inbox-specific files are contained within the +C<$GIT_DIR/public-inbox/> directory. All files are expected to +grow in size as more messages are archived, so using compaction +commands (e.g. L) is not recommended unless +the list is no longer active. + +=item $GIT_DIR/public-inbox/msgmap.sqlite3 + +The stable NNTP article number to Message-ID mapping is +stored in an SQLite3 database. + +This is required for users of L, but +users of the L interface will find it +useful for attempting recovery from copy-paste truncations of +URLs containing long Message-IDs. + +Avoid removing this file and regenerating it; it may cause +existing NNTP readers to lose sync and miss (or duplicate) +messages. + +This file is relatively small, and typically less than 5% +of the space of the mail stored in a packed git repository. + +=item $GIT_DIR/public-inbox/xapian* + +The database used by L. This directory name is +followed by a number indicating the index schema version this +installation of public-inbox uses. + +These directories may be safely deleted or removed in full +while the NNTP and HTTP interfaces are no longer accessing +them. + +In addition to providing a search interface for the HTTP +interface, the Xapian database is used to group and combine +related messages into threads. For NNTP servers, it also +provides a cache of metadata and header information often +requested by NNTP clients. + +This directory is large, often two to three times the size of +the objects stored in a packed git repository. + +=head1 ENVIRONMENT + +=over 8 + +=item PI_CONFIG + +Used to override the default "~/.public-inbox/config" value. + +=back + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2016 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L, L diff --git a/Documentation/public-inbox-mda.pod b/Documentation/public-inbox-mda.pod index 18fcdd08..3a43a1ce 100644 --- a/Documentation/public-inbox-mda.pod +++ b/Documentation/public-inbox-mda.pod @@ -1,6 +1,6 @@ =head1 NAME -public-inbox-mda - mail delivery for public-inbox +public-inbox-mda - mail delivery agent for public-inbox =head1 SYNOPSIS @@ -9,7 +9,8 @@ B EMESSAGE =head1 DESCRIPTION Mail Delivery Agent (MDA) for public-inbox installations. -Each system user may have their own public-inbox instances +Each system user may have their own public-inbox instances. +This may be invoked via L or similar tools. =head1 ENVIRONMENT @@ -17,33 +18,37 @@ Each system user may have their own public-inbox instances =item ORIGINAL_RECIPIENT -the original recipient email address, from Postfix +The original recipient email address, set by the MTA. Postfix +sets it by default, untested on other MTAs. =item PI_CONFIG -config file. default: ~/.public-inbox/config +Per-user config file parseable by L. +See L. + +Default: ~/.public-inbox/config =item PI_EMERGENCY -emergency destination. default: ~/.public-inbox/emergency/ +emergency Maildir destination. -=back +Default: ~/.public-inbox/emergency/ -=head1 PI_CONFIG FILE +=back -This is a config file parseable by L. =head1 CONTACT -All feedback welcome via plain-text mail to L +Feedback welcome via plain-text mail to L -The mail archives are hosted at L +The mail archives are hosted at L +and L =head1 COPYRIGHT Copyright 2013-2016 all contributors L -License: AGPL-3.0+ L +License: AGPL-3.0+ L =head1 SEE ALSO diff --git a/Documentation/public-inbox-nntpd.pod b/Documentation/public-inbox-nntpd.pod new file mode 100644 index 00000000..2f9dbabf --- /dev/null +++ b/Documentation/public-inbox-nntpd.pod @@ -0,0 +1,53 @@ +=head1 NAME + +public-inbox-nntpd - NNTP server for sharing public-inbox + +=head1 SYNOPSIS + +B [OPTIONS] + +=head1 DESCRIPTION + +public-inbox-nntpd provides a read-only NNTP daemon for +public-inbox. It uses options and environment variables common +to all L implementations. + +The default configuration will never require write access +tto the directory where the public-inbox is stored, so it +may be run as a different user than the user running +L, L, or +L. + +=head1 CONFIGURATION + +These configuration knobs should be used in the +L + +=over 8 + +=item publicinbox..newsgroup + +=item publicinbox.nntpserver + +=back + +See L for documentation on them. + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L, +L, +L + +=head1 COPYRIGHT + +Copyright 2013-2016 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L, L, L, +L diff --git a/Documentation/public-inbox-overview.pod b/Documentation/public-inbox-overview.pod new file mode 100644 index 00000000..2c97f876 --- /dev/null +++ b/Documentation/public-inbox-overview.pod @@ -0,0 +1,108 @@ +=head1 NAME + +public-inbox-overview - an overview of public-inbox + +=head1 DESCRIPTION + +public-inbox consists of many parts which may be used +independently or in conjunction of each other for: + +=over 4 + +=item 1 + +Mirroring existing public-inboxes. + +=item 2 + +Mirroring mailing lists directly. + +=item 3 + +Hosting standalone. + +=back + +=head2 Mirroring existing public-inboxes + +Mirroring existing public-inboxes is the easiest way to get +started. Your mirror will remain dependent on the REMOTE_URL +you are mirroring and you only need to use two new commands in +addition to common L commands. + + git clone --mirror REMOTE_URL /path/to/repo.git + + # The following should create the necessary entry in + # ~/.public-inbox/config + public-inbox-init NAME /path/to/repo.git MY_URL LIST_ADDRESS + + # Optional but strongly recommended for hosting HTTP + # (and required for NNTP) + # enable search (requires Search::Xapian and DBD::SQLite) + public-inbox-index /path/to/repo.git + + # Periodically update the repo with the following commands + # to update the git repo and index new messages: + cd /path/to/repo.git && git fetch && public-inbox-index + +See L below for info on how to expose +your mirror to other readers. + +=head2 Mirroring mailing lists directly + +Mirroring existing mailing lists may be done by any reader +of a mailing list using L. + + # This will create a new git repository: + public-inbox-init NAME /path/to/repo.git MY_URL LIST_ADDRESS + +Then, see the L manual for configuring +C, C, and the optional C and +C entries. + +You will need to leave L running to +keep the mailbox up-to-date as messages are delivered to +the mailing list. + +Running L to create search indices +is recommended. L will automatically +maintain the indices if they were created by +L + + public-inbox-index /path/to/repo.git + +=head2 Hosting standalone + +Using L to initialize the inbox as in the +other methods is recommended. See L for +more details; but this also requires MTA-specific knowledge. + +=head2 Hosting public-inboxes + +Since public-inboxes are git repositories, they may be served to +remote clients via L as well as specialized HTTP +and NNTP daemons distributed with public-inbox. + +See L and L +for more information on using these daemons. + +Hosting a public-inbox over HTTP or NNTP will never require +write access to any files in the git repository, including +the search indices or article number map database. + +Users familiar with PSGI and L may also use +L with the preferred server instead of +L + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2016 all contributors L + +License: AGPL-3.0+ L diff --git a/Documentation/public-inbox-watch.pod b/Documentation/public-inbox-watch.pod new file mode 100644 index 00000000..404303e8 --- /dev/null +++ b/Documentation/public-inbox-watch.pod @@ -0,0 +1,121 @@ +=head1 NAME + +public-inbox-watch - mailbox watcher for public-inbox + +=head1 SYNOPSIS + +B + +In ~/.public-inbox/config: + + [publicinbox "test"] + ; generic public-inbox-config keys: + address = test@example.com + url = http://example.com/test + mainrepo = /path/to/test.example.com.git + + ; config keys specific to public-inbox-watch: + watch = maildir:/path/to/maildirs/.INBOX.test/ + watchheader = List-Id: + + [publicinboxwatch] + ; optional, enable use of spamc(1) for checking: + spamcheck = spamc + + ; optional, emails marked as read which appear + ; here will be trained as spam and deleted from + ; the mainrepos of any public-inboxes which are + ; configured for watch. + ; This is global for all publicinbox.* sections + watchspam = maildir:/path/to/maildirs/.INBOX.spam + +=head1 DESCRIPTION + +public-inbox-watch allows watching a mailbox (currently only +Maildir) for the arrival of new messages and automatically +importing them into a public-inbox (git) repository. +public-inbox-watch is useful in situations when a user wishes to +mirror an existing mailing list, but has no access to run +L on a server. Unlike public-inbox-mda +which is invoked once per-message, public-inbox-watch is a +persistent process, making it faster for after-the-fact imports +of large Maildirs. + +Upon startup, it scans the mailbox for new messages to be +imported while it was not running. + +Currently, only Maildirs are supported and the +L Perl module is required. + +For now, IMAP users should use tools such as L +or L to bidirectionally sync their IMAP +folders to Maildirs for public-inbox-watch. + +public-inbox-watch should be run inside a L session +or as a L service. Errors are emitted to stderr. + +=head1 OPTIONS + +public-inbox-watch takes no command-line options. + +=head1 CONFIGURATION + +These configuration knobs should be used in the +L + +=over 8 + +=item publicinbox..watch + +=item publicinbox..watchheader + +=item publicinboxwatch.spamcheck + +=item publicinboxwatch.watchspam + +=back + +See L for documentation on them. + +=head1 SIGNALS + +=over 8 + +=item SIGHUP + +Reload the config file (default: ~/.public-inbox/config) + +=item SIGUSR1 + +Rescan all watched mailboxes. This is done automatically after +startup. + +=back + +=head1 ENVIRONMENT + +=over 8 + +=item PI_CONFIG + +config file. default: ~/.public-inbox/config +See L + +=back + +=head1 CONTACT + +Feedback welcome via plain-text mail to L + +The mail archives are hosted at L +and L + +=head1 COPYRIGHT + +Copyright 2016 all contributors L + +License: AGPL-3.0+ L + +=head1 SEE ALSO + +L -- cgit v1.2.3-24-ge0c7