diff options
| -rw-r--r-- | AUTHORS | 2 | ||||
| -rw-r--r-- | Documentation/dc-dlvr-spam-flow.txt | 2 | ||||
| -rw-r--r-- | Documentation/include.mk | 9 | ||||
| -rw-r--r-- | Documentation/public-inbox-mda.pod | 25 | ||||
| -rw-r--r-- | Documentation/public-inbox-overview.pod | 68 | ||||
| -rw-r--r-- | README | 4 | ||||
| -rw-r--r-- | lib/PublicInbox/WwwStream.pm | 5 | ||||
| -rwxr-xr-x | script/public-inbox-mda | 7 | ||||
| -rw-r--r-- | t/mda.t | 15 |
9 files changed, 99 insertions, 38 deletions
@@ -1,6 +1,6 @@ This list only includes major contributors. -See history in git (via `git clone https://public-inbox.org/public-inbox') +See history in git (via `git clone https://public-inbox.org/public-inbox.git') for a full history of the project. * Eric Wong <e@80x24.org> (BDFL) diff --git a/Documentation/dc-dlvr-spam-flow.txt b/Documentation/dc-dlvr-spam-flow.txt index 81aba766..d151d272 100644 --- a/Documentation/dc-dlvr-spam-flow.txt +++ b/Documentation/dc-dlvr-spam-flow.txt @@ -45,4 +45,4 @@ delivery path as well as removing the message from the git tree. * spamc / spamd - SpamAssassin: http://spamassassin.apache.org/ * report-spam / dc-dlvr - distributed with public-inbox in the scripts/ - directory: git clone https://public-inbox.org/ public-inbox + directory: git clone https://public-inbox.org/public-inbox.git diff --git a/Documentation/include.mk b/Documentation/include.mk index 4af8d455..1460604d 100644 --- a/Documentation/include.mk +++ b/Documentation/include.mk @@ -13,7 +13,6 @@ MAN = man # this is "xml" on FreeBSD and maybe some other distros: XMLSTARLET = xmlstarlet -AWK = awk # same as pod2text COLUMNS = 76 @@ -89,11 +88,9 @@ doc_install :: install-man %.1 %.5 %.7 %.8 : Documentation/%.pod $(podman) -s $(subst .,,$(suffix $@)) $< $@+ && mv $@+ $@ -# n.b. not sure if our usage of man(1) is portable or not, so not -# enabled by default "check" target: -# check :: check-man -check_man = @echo CHECK80 $<;COLS=80 $(MAN) ./$^ | \ - $(AWK) 'length>80{print;err=1}END{exit(err)}' >&2 +check :: check-man +check_man = @echo CHECK80 $<;COLUMNS=80 $(MAN) ./$^ | \ + $(AWK) '{gsub(/\b./,"")}length>80{print;err=1}END{exit(err)}' >&2 %.1.cols : %.1; $(check_man) %.5.cols : %.5; $(check_man) diff --git a/Documentation/public-inbox-mda.pod b/Documentation/public-inbox-mda.pod index 921b7a15..01bb8476 100644 --- a/Documentation/public-inbox-mda.pod +++ b/Documentation/public-inbox-mda.pod @@ -16,6 +16,31 @@ By default, it relies on L<spamc(1)> for filtering mail, but may be disabled via L<public-inbox-config(5)/publicinboxmda.spamcheck> +=head1 OPTIONS + +=over 8 + +=item --no-precheck + +By default, public-inbox-mda does some simple checks before +invoking L<spamc(1)> since it is intended to receive mail before +it goes to a mailing list. + +However, some users prefer to use public-inbox-mda to mirror +mailing lists. This option exists to support those users. + +Using this option, the following prechecks are disabled: + + * multiple Message-IDs + * non-existent Message-IDs + * Message-IDs longer than 244 characters long + * From: header shorter than 3 characters + * Subject: header shorter than 2 characters + * unusable Date: headers + * inbox address specified in To: or Cc: header + +=back + =head1 ENVIRONMENT =over 8 diff --git a/Documentation/public-inbox-overview.pod b/Documentation/public-inbox-overview.pod index 46060d0c..91d4a88e 100644 --- a/Documentation/public-inbox-overview.pod +++ b/Documentation/public-inbox-overview.pod @@ -4,7 +4,7 @@ public-inbox-overview - an overview of public-inbox =head1 DESCRIPTION -public-inbox consists of many parts which may be used +public-inbox consists of many pieces which may be used independently or in conjunction of each other for: =over 4 @@ -15,11 +15,11 @@ Mirroring existing public-inboxes. =item 2 -Mirroring mailing lists directly. +Mirroring mailing lists. =item 3 -Hosting standalone. +Hosting standalone inboxes. =back @@ -30,35 +30,52 @@ started. Your mirror will remain dependent on the REMOTE_URL you are mirroring and you only need to use two new commands in addition to common L<git(1)> commands. - git clone --mirror REMOTE_URL /path/to/repo.git +Instructions are different depending on whether the inbox +is L<public-inbox-v1-format(5)> or L<public-inbox-v2-format(5)>. +See the "Archives are clonable:" part of the WWW interface +of a given inbox for cloning instructions specific to +that inbox. The instructions are roughly: - # The following should create the necessary entry in - # ~/.public-inbox/config - public-inbox-init NAME /path/to/repo.git MY_URL LIST_ADDRESS + # for v1 inboxes: + git clone --mirror URL INBOX_DIR - # Optional but strongly recommended for hosting HTTP - # (and required for NNTP) - # enable search (requires Search::Xapian and DBD::SQLite) - public-inbox-index /path/to/repo.git + # for v2 inboxes (each epoch needs to be cloned): + git clone --mirror URL/EPOCH INBOX_DIR/git/EPOCH.git - # Periodically update the repo with the following commands - # to update the git repo and index new messages: - cd /path/to/repo.git && git fetch && public-inbox-index + # The following should create the necessary entry in + # ~/.public-inbox/config, use "-V2" only for v2 inboxes: + public-inbox-init [-V2] NAME INBOX_DIR MY_URL LIST_ADDRESS -See L</"Hosting public-inboxes"> below for info on how to expose + # Optional but strongly recommended for hosting HTTP + # (and required for NNTP) + # enable overview (requires DBD::SQLite) and, if Search::Xapian is + # available, search: + public-inbox-index INBOX_DIR + + # Periodically fetch the repo using git-fetch(1) + # for v1 inboxes: + git --git-dir=INBOX_DIR fetch + + # for v2 (in most cases, only the newest epoch needs to be fetched): + git --git-dir=INBOX_DIR/git/EPOCH.git fetch + + # index new messages after fetching: + public-inbox-index INBOX_DIR + +See L</"Serving public-inboxes"> below for info on how to expose your mirror to other readers. -=head2 Mirroring mailing lists directly +=head2 Mirroring mailing lists -Mirroring existing mailing lists may be done by any reader +Mirroring mailing lists may be done by any reader of a mailing list using L<public-inbox-watch(1)>. # This will create a new git repository: - public-inbox-init NAME /path/to/repo.git MY_URL LIST_ADDRESS + public-inbox-init -V2 NAME INBOX_DIR MY_URL LIST_ADDRESS Then, see the L<public-inbox-watch(1)> manual for configuring -C<watch>, C<watchheader>, and the optional C<spamcheck> and -C<watchspam> entries. +C<watch>, C<watchheader>, C<listid> and the optional C<spamcheck> +and C<watchspam> entries. You will need to leave L<public-inbox-watch(1)> running to keep the mailbox up-to-date as messages are delivered to @@ -69,15 +86,20 @@ is recommended. L<public-inbox-watch(1)> will automatically maintain the indices if they were created by L<public-inbox-index(1)> - public-inbox-index /path/to/repo.git + public-inbox-index INBOX_DIR + +Instead of using L<public-inbox-watch(1)>, using +L<public-inbox-mda(1)> with the C<--no-precheck> option and +relying on the C<listid> directive in L<public-inbox-config(5)> +is also an option. -=head2 Hosting standalone +=head2 Hosting standalone inboxes Using L<public-inbox-init(1)> to initialize the inbox as in the other methods is recommended. See L<public-inbox-mda(1)> for more details; but this also requires MTA-specific knowledge. -=head2 Hosting public-inboxes +=head2 Serving public-inboxes Since public-inboxes are git repositories, they may be served to remote clients via L<git-daemon(1)> as well as specialized HTTP @@ -91,8 +91,8 @@ Hacking AGPL source code is available via git: - git clone https://public-inbox.org/ public-inbox - git clone git://repo.or.cz/public-inbox + git clone https://public-inbox.org/public-inbox.git + git clone https://repo.or.cz/public-inbox.git torsocks git clone http://hjrcffqmbrq6wope.onion/public-inbox See below for contact info. diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm index 6696cc35..0f4f55d0 100644 --- a/lib/PublicInbox/WwwStream.pm +++ b/lib/PublicInbox/WwwStream.pm @@ -11,8 +11,7 @@ use strict; use warnings; use PublicInbox::Hval qw(ascii_html); our $TOR_URL = 'https://www.torproject.org/'; -our $CODE_URL = 'https://public-inbox.org/'; -our $PROJECT = 'public-inbox'; +our $CODE_URL = 'https://public-inbox.org/public-inbox.git'; # noop for HTTP.pm (and any other PSGI servers) sub close {} @@ -80,7 +79,7 @@ sub _html_top ($) { sub code_footer ($) { my ($env) = @_; my $u = PublicInbox::Hval::prurl($env, $CODE_URL); - qq(AGPL code for this site: git clone <a\nhref="$u">$u</a> $PROJECT) + qq(AGPL code for this site: git clone <a\nhref="$u">$u</a>) } sub _html_end { diff --git a/script/public-inbox-mda b/script/public-inbox-mda index dea49dc5..584218b5 100755 --- a/script/public-inbox-mda +++ b/script/public-inbox-mda @@ -5,7 +5,8 @@ # Mail delivery agent for public-inbox, run from your MTA upon mail delivery use strict; use warnings; -my $usage = 'public-inbox-mda < rfc2822_message'; +my $usage = 'public-inbox-mda [OPTIONS] < rfc2822_message'; +my $precheck = grep(/\A--no-precheck\z/, @ARGV) ? 0 : 1; my ($ems, $emm); sub do_exit { @@ -55,7 +56,9 @@ $dst->{inboxdir} or do_exit(67); $dst = PublicInbox::InboxWritable->new($dst); # pre-check, MDA has stricter rules than an importer might; -do_exit(0) unless PublicInbox::MDA->precheck($simple, $dst->{address}); +if ($precheck && !PublicInbox::MDA->precheck($simple, $dst->{address})) { + do_exit(0); +} $simple = undef; my $spam_ok; if ($spamc) { @@ -293,6 +293,21 @@ EOF my $path = mid2path($mid); my $msg = `git --git-dir=$maindir cat-file blob HEAD:$path`; like($msg, qr/\Q$list_id\E/, 'delivered message w/ List-ID matches'); + + # try a message w/o precheck + $simple = Email::Simple->new(<<EOF); +To: You <you\@example.com> +List-Id: <$list_id> + +this message would not be accepted without --no-precheck +EOF + $in = $simple->as_string; + my ($out, $err) = ('', ''); + IPC::Run::run([$mda, '--no-precheck'], \$in, \$out, \$err); + is($?, 0, 'mda OK with List-Id match and --no-precheck'); + my $cur = `git --git-dir=$maindir diff HEAD~1..HEAD`; + like($cur, qr/this message would not be accepted without --no-precheck/, + '--no-precheck delivered message anyways'); } done_testing(); |