From 6a414a4087a59ad8c62cbef30984632ea31ced23 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 19 Apr 2014 19:19:06 +0000 Subject: various documentation updates We have an HTML homepage, OMG! --- .gitignore | 4 +++ Documentation/design_notes.txt | 24 ++++++++-------- Documentation/design_www.txt | 6 ++++ Documentation/include.mk | 65 ++++++++++++++++++++++++++++++++++++++++++ Documentation/txt2pre | 26 +++++++++++++++++ INSTALL | 21 ++++++++++---- Makefile.PL | 15 +--------- README | 6 +--- 8 files changed, 130 insertions(+), 37 deletions(-) create mode 100644 Documentation/include.mk create mode 100755 Documentation/txt2pre diff --git a/.gitignore b/.gitignore index 0c5ad528..c748e7ca 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,8 @@ /Makefile /blib /cover_db +*.1 +*.5 +*.7 +*.html *.gz diff --git a/Documentation/design_notes.txt b/Documentation/design_notes.txt index 286f35c0..d7313cb6 100644 --- a/Documentation/design_notes.txt +++ b/Documentation/design_notes.txt @@ -1,5 +1,5 @@ -Design notes ------------- +public-inbox design notes +------------------------- Challenges to running normal mailing lists ------------------------------------------ @@ -14,7 +14,6 @@ confused users who cannot unsubscribe. Use existing infrastructure --------------------------- - * public-inbox can coexist with existing mailing lists, any subscriber to the existing mailing list can begin delivering messages to public-inbox-mda(1) @@ -35,7 +34,6 @@ Use existing infrastructure Why email? ---------- - * Freedom from proprietary services, tools and APIs. Communicating with developers and users of Free Software should not rely on proprietary tools or services. @@ -53,9 +51,9 @@ Why email? There is no need to ask the NSA for backups of your mail archives :) * git, one of the most widely-used version control systems, includes many - tools for for email: git-format-patch(1), git-send-email(1), git-am(1). - Furthermore, the development of git itself is based on the git mailing - list. + tools for for email, including: git-format-patch(1), git-send-email(1), + git-am(1), git-imap-send(1). Furthermore, the development of git itself + is based on the git mailing list. * Email is already the de-facto form of communication in many Free Software communities.. @@ -66,7 +64,6 @@ Why email? Why git? -------- - * git is distributed and robust while being both fast and space-efficient with text data. NNTP was considered, but does not support delta-compression and places no guarantees on data/transport @@ -78,7 +75,6 @@ Why git? Why perl 5? ----------- - * Perl 5 is widely available on modern *nix systems with good a history of backwards and forward compatibility. @@ -87,7 +83,6 @@ Why perl 5? Laziness -------- - * Stick to dependencies available in Debian main, this should make it easier for potential users to install, and easier for distro maintainers to pick up. @@ -115,7 +110,6 @@ Laziness Web notes --------- - * Getting users to install/run ssoma (or any new tool) is difficult. The web views must be easily read/cache/mirror-able. @@ -128,9 +122,15 @@ Web notes * Raw text endpoint allows users to write client-side JS endpoints without hosting the data themselves (or on a different server). +What sucks about public-inbox +----------------------------- +* Lack of push notification. On the other hand, feeds seem popular. + +* some (mostly GUI) mail clients cannot set In-Reply-To headers + properly without the original message. + Scalability notes ----------------- - Even with shallow clone, storing the history of large/busy mailing lists may place much burden on subscribers and servers. However, having a single (or few) refs representing the entire history of a list is good diff --git a/Documentation/design_www.txt b/Documentation/design_www.txt index 0f63fdf2..1cd0d96e 100644 --- a/Documentation/design_www.txt +++ b/Documentation/design_www.txt @@ -1,15 +1,21 @@ URL naming ---------- +### Unstable endpoints /$LISTNAME/?r=$GIT_COMMIT -> HTML only /$LISTNAME/index.atom.xml -> Atom feed /$LISTNAME/all.atom.xml -> Atom feed, includes replies + +### Stable endpoints /$LISTNAME/m/$MESSAGE_ID.html -> HTML content (short quotes) /$LISTNAME/m/$MESSAGE_ID.txt -> raw original /$LISTNAME/m/$MESSAGE_ID -> 301 to .html version /$LISTNAME/f/$MESSAGE_ID.html -> HTML content (full quotes) /$LISTNAME/f/$MESSAGE_ID -> 301 to .html version /$LISTNAME/f/$MESSAGE_ID.txt -> 301 to m/$MESSAGE_ID.txt + +TODO (these might be expensive) +------------------------------- /$LISTNAME/t/$MESSAGE_ID.html -> HTML content of thread /$LISTNAME/t/$MESSAGE_ID.mbox -> mbox content of thread diff --git a/Documentation/include.mk b/Documentation/include.mk new file mode 100644 index 00000000..776413e6 --- /dev/null +++ b/Documentation/include.mk @@ -0,0 +1,65 @@ +# Copyright (C) 2013, Eric Wong and all contributors +# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +all:: + +RSYNC = rsync +RSYNC_DEST = public-inbox.org:/srv/public-inbox/ +docs := README COPYING INSTALL $(shell git ls-files 'Documentation/*.txt') +INSTALL = install +PANDOC = pandoc +PANDOC_OPTS = -f markdown --email-obfuscation=none +pandoc = $(PANDOC) $(PANDOC_OPTS) + +m1 = +m1 += public-inbox-mda +m5 = +m7 = + +man1 := $(addsuffix .1, $(m1)) +man5 := $(addsuffix .5, $(m5)) +man7 := $(addsuffix .7, $(m7)) + +all:: man html + +man: $(man1) $(man5) $(man7) + +prefix ?= $(HOME) +mandir ?= $(prefix)/share/man +man1dir = $(mandir)/man1 +man5dir = $(mandir)/man5 +man7dir = $(mandir)/man7 + +install-man: man + test -z "$(man1)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man1dir) + test -z "$(man5)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man5dir) + test -z "$(man7)" || $(INSTALL) -d -m 755 $(DESTDIR)$(man7dir) + test -z "$(man1)" || $(INSTALL) -m 644 $(man1) $(DESTDIR)$(man1dir) + test -z "$(man5)" || $(INSTALL) -m 644 $(man5) $(DESTDIR)$(man5dir) + test -z "$(man7)" || $(INSTALL) -m 644 $(man7) $(DESTDIR)$(man7dir) +%.1 %.5 %.7 : Documentation/%.txt + $(pandoc) -s -t man < $< > $@+ && mv $@+ $@ + +txt2pre = ./Documentation/txt2pre < $< > $@+ && touch -r $< $@+ && mv $@+ $@ +txt = INSTALL README COPYING + +INSTALL.html: INSTALL + $(txt2pre) +index.html: README + $(txt2pre) + +docs_html := index.html INSTALL.html +html: $(docs_html) +gz_docs := $(addsuffix .gz, $(docs) $(docs_html)) +rsync_docs := $(gz_docs) $(docs) $(txt) $(docs_html) +%.gz: % + gzip -9 --rsyncable < $< > $@+ + touch -r $< $@+ + mv $@+ $@ + +gz-doc: $(gz_docs) +rsync-doc: + git set-file-times $(docs) $(txt) + $(MAKE) gz-doc + $(RSYNC) --chmod=Fugo=r -av $(rsync_docs) $(RSYNC_DEST) +clean-doc: + $(RM) $(man1) $(man5) $(man7) $(gz_docs) $(docs_html) diff --git a/Documentation/txt2pre b/Documentation/txt2pre new file mode 100755 index 00000000..f84f9c06 --- /dev/null +++ b/Documentation/txt2pre @@ -0,0 +1,26 @@ +#!/usr/bin/env perl +# Copyright (C) 2014, Eric Wong and all contributors +# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) +# +# Stupid script to make HTML from preformatted, utf-8 text versions, +# only generating links for http(s). Markdown does too much +# and requires indentation to output preformatted text. +use strict; +use warnings; +use CGI qw/escapeHTML/; +use Encode qw/encode/; +my $str = eval { local $/; <> }; +$str = escapeHTML($str); +$str = encode('us-ascii', $str, Encode::HTMLCREF); +my ($title) = ($str =~ /\A([^\n]+)/); + +# temporarily swap > for escape so our s!! to add href works. +# there's probably a way to do this with only a single s!! ... +$str =~ s!>!\e!g; +$str =~ s!\b(https?://[\w+\+\&\?\.\%\;/-]+)!$1!g; +$str =~ s!\e!>!g; # swap escapes back to > + +print '', + '', + "$title", + "\n
",  $str , '
'; diff --git a/INSTALL b/INSTALL index 8e512a63..bbc91533 100644 --- a/INSTALL +++ b/INSTALL @@ -1,6 +1,15 @@ -Note: this still needs to be documented better, +public-inbox (server-side) installation +--------------------------------------- + +This is for folks who want to setup their own public-inbox instance. +Clients should see http://ssoma.public-inbox.org/INSTALL instead. + +TODO: this still needs to be documented better, also see the scripts/ and sa_config/ directories in the source tree +It should also be possible to use public-inbox with only IMAP +(or even POP(!)) access to a mailbox. + standard MakeMaker installation (Perl) -------------------------------------- @@ -17,7 +26,7 @@ Requirements (server MDA) * MTA - postfix is recommended * lynx (for converting HTML messages to text) * Perl and several modules: (Debian package name) - - CGI perl-modules[1] + - CGI[1] perl-modules[2] - Date::Parse libtimedate-perl - Email::Address libemail-address-perl - Email::Filter libemail-filter-perl @@ -26,12 +35,12 @@ Requirements (server MDA) - Encode::MIME::Header perl - File::Path::Expand libfile-path-expand-perl - IPC::Run libipc-run-perl - - Mail::Thread (2.5+) libmail-thread-perl - - URI::Escape liburi-perl - - XML::Atom::SimpleFeed libxml-atom-simplefeed-perl + - Mail::Thread (2.5+)[1] libmail-thread-perl + - URI::Escape[1] liburi-perl + - XML::Atom::SimpleFeed[1] libxml-atom-simplefeed-perl [1] - Only required for serving/generating Atom and HTML pages. - Keep in mind this will be split into a separate Debian package +[2] - Keep in mind this will be split into a separate Debian package when CGI.pm is dropped from the Perl standard library. Plack/PSGI, FastCGI, and mod_perl2 will all be supported. diff --git a/Makefile.PL b/Makefile.PL index 245bb887..3dfafe7b 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -33,20 +33,7 @@ WriteMakefile( sub MY::postamble { <<'EOF'; -RSYNC_DEST = public-inbox.org:/srv/public-inbox/ -docs = README COPYING INSTALL $(shell git ls-files Documentation/ '*.txt') -gz_docs = $(addsuffix .gz, $(docs)) -%.gz: % - gzip -9 --rsyncable < $< > $@+ - touch -r $< $@+ - mv $@+ $@ - -gz-docs: $(gz_docs) -rsync-docs: - git set-file-times $(docs) - $(MAKE) gz-docs - rsync --chmod=Fugo=r -av $(gz_docs) $(docs) $(RSYNC_DEST) - +-include Documentation/include.mk N = $(shell echo $$(( $$(nproc 2>/dev/null || echo 2) + 1))) check:: pure_all prove -lv -j$(N) diff --git a/README b/README index f12c1cba..342b847a 100644 --- a/README +++ b/README @@ -1,6 +1,5 @@ public-inbox - an "archives first" approach to mailing lists ------------------------------------------------------------ - public-inbox implements the sharing of an email inbox via git to complement or replace traditional mailing lists. @@ -20,17 +19,14 @@ to run their own instances with minimal overhead. Features -------- - * anybody may participate via plain-text email * stores email in git, readers have a complete archive of the inbox * Atom feed allows casual readers to follow via feed reader * Mail user-agent (MUA) users may use Maildir, mbox(5) and/or IMAP locally * uses only well-documented and easy-to-implement data formats - Implementation -------------- - public-inbox uses ssoma[1], Some Sort Of Mail Archiver which implements no policy of its own. By exposing an inbox via git, readers may follow the inbox without subscribing via SMTP and have easy access to old @@ -95,7 +91,7 @@ subscription. This also makes it easier to rope in folks of tangentially related projects we depend on (e.g. git developers on git@vger.kernel.org). -You can subscribe via ssoma(1), LISTNAME is a name of your choosing: +You may subscribe via ssoma(1), LISTNAME is a name of your choosing: URL=git://git.public-inbox.org/public-inbox LISTNAME=public-inbox -- cgit v1.2.3-24-ge0c7