#!/usr/bin/perl -w # Copyright (C) 2015-2019 all contributors # License: AGPL-3.0+ # # Incremental (or one-shot) importer of a slrnpull news spool =begin usage export ORIGINAL_RECIPIENT=address@example.com public-inbox-init $INBOX $GIT_DIR $HTTP_URL $ORIGINAL_RECIPIENT ./import_slrnspool SLRNPULL_ROOT/news/foo/bar =cut use strict; use warnings; use PublicInbox::Config; use PublicInbox::MIME; use PublicInbox::Import; use PublicInbox::Git; sub usage { "Usage:\n".join('',grep(/\t/, `head -n 10 $0`)) } my $exit = 0; my $sighandler = sub { $exit = 1 }; $SIG{INT} = $sighandler; $SIG{TERM} = $sighandler; my $spool = shift @ARGV or die usage(); my $recipient = $ENV{ORIGINAL_RECIPIENT}; defined $recipient or die usage(); my $config = PublicInbox::Config->new; my $ibx = $config->lookup($recipient); my $git = $ibx->git; my $im; if (($ibx->{version} || 1) == 2) { require PublicInbox::V2Writable; $im = PublicInbox::V2Writable->new($ibx); $im->{parallel} = 0; # pointless to be parallel for a single message } else { $im = PublicInbox::Import->new($git, $ibx->{name}, $ibx->{-primary_address}); } $ibx->{filter} ||= 'PublicInbox::Filter::Gmane'; my $filter = $ibx->filter; sub key { "publicinbox.$ibx->{name}.importslrnspoolstate"; } sub get_min { my $f = PublicInbox::Config->default_file; my $out = $git->qx('config', "--file=$f", key($ibx)); $out ||= 0; chomp $out; $out =~ /\A[0-9]+\z/ and return $out; 0; } sub set_min { my ($num) = @_; my $f = PublicInbox::Config->default_file; my @cmd = (qw/git config/, "--file=$f", key($ibx), $num); system(@cmd) == 0 or die join(' ', @cmd). " failed: $?\n"; } my $n = get_min(); my $ok; my $max_gap = 200000; my $max = $n + $max_gap; $spool =~ s!/+\z!!; for (; $exit == 0 && $n < $max; $n++) { my $fn = "$spool/$n"; open(my $fh, '<', $fn) or next; $max = $n + $max_gap; print STDERR $fn, "\n"; my $mime = PublicInbox::MIME->new(eval { local $/; <$fh> }); $filter->scrub($mime); $im->add($mime); $ok = $n + 1; set_min($ok); } $im->done;