From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-2.9 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, T_RP_MATCHES_RCVD shortcircuit=no autolearn=unavailable version=3.3.2 X-Original-To: meta@public-inbox.org Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 4A79820247 for ; Mon, 12 Jan 2015 02:31:07 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 4/4] import_slrnspool: fork a process for each message Date: Mon, 12 Jan 2015 02:30:57 +0000 Message-Id: <1421029857-1702-5-git-send-email-e@80x24.org> X-Mailer: git-send-email 2.2.0.35.g882592c In-Reply-To: <1421029857-1702-1-git-send-email-e@80x24.org> References: <1421029857-1702-1-git-send-email-e@80x24.org> List-Id: This prevents process growth when importing large messages. Memory growth could be due to the sliding sbrk window in glibc malloc or a circular reference in the Email::* Perl code somewhere. --- scripts/import_slrnspool | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/scripts/import_slrnspool b/scripts/import_slrnspool index e55c96c..dd612b1 100755 --- a/scripts/import_slrnspool +++ b/scripts/import_slrnspool @@ -66,27 +66,41 @@ for (; $exit == 0 && $n < $max; $n++) { print STDERR $fn, "\n"; open(my $fh, '<', $fn) or next; $max = $n + $max_gap; - my $f = Email::Filter->new(data => eval { local $/; <$fh> }); - my $s = $f->simple; - # gmane rewrites Received headers, which increases spamminess - # Some older archives set Original-To - foreach my $x (qw(Received To)) { - my @h = $s->header("Original-$x"); - if (@h) { - $s->header_set($x, @h); - $s->header_set("Original-$x"); + # prevent process growth by forking a new process for each message + my $pid = fork; + die "failed to fork: $!\n" unless defined $pid; + + if ($pid == 0) { + my $f = Email::Filter->new(data => eval { local $/; <$fh> }); + close $fh; + $fh = undef; + my $s = $f->simple; + + # gmane rewrites Received headers, which increases spamminess + # Some older archives set Original-To + foreach my $x (qw(Received To)) { + my @h = $s->header("Original-$x"); + if (@h) { + $s->header_set($x, @h); + $s->header_set("Original-$x"); + } } - } - # triggers for the SA HEADER_SPAM rule - foreach my $drop (qw(Approved)) { $s->header_set($drop) } + # triggers for the SA HEADER_SPAM rule + foreach my $drop (qw(Approved)) { $s->header_set($drop) } - # appears to be an old gmane bug: - $s->header_set('connect()'); + # appears to be an old gmane bug: + $s->header_set('connect()'); - $f->exit(0); - $f->pipe(@mda); + $f->exit(0); + $f->pipe(@mda); + exit 0; + } else { + close $fh; + waitpid($pid, 0); + die "error: $?\n" if $?; + } $ok = $n + 1; set_min($cfg, $ok); } -- EW