From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 5A6F31F4B4 for ; Wed, 31 Mar 2021 07:45:51 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] lei_input: reduce IPC traffic with multiple inputs Date: Wed, 31 Mar 2021 07:45:51 +0000 Message-Id: <20210331074551.25641-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: No point in sending a command for every input when a single one will do. We'll also trigger LeiStore->done sooner in the worker rather than later. --- lib/PublicInbox/LeiConvert.pm | 11 +++++------ lib/PublicInbox/LeiImport.pm | 6 +----- lib/PublicInbox/LeiInput.pm | 9 +++++++++ lib/PublicInbox/LeiTag.pm | 6 +----- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm index fb7a2f3b..0ce49ea9 100644 --- a/lib/PublicInbox/LeiConvert.pm +++ b/lib/PublicInbox/LeiConvert.pm @@ -7,6 +7,7 @@ use strict; use v5.10.1; use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use PublicInbox::LeiOverview; +use PublicInbox::DS; # /^input_/ subs are used by PublicInbox::LeiInput @@ -32,12 +33,10 @@ sub input_maildir_cb { $self->{wcb}->(undef, { kw => $kw }, $eml); } -sub do_convert { # via wq_do +sub process_inputs { # via wq_do my ($self) = @_; - $PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid - for my $input (@{$self->{inputs}}) { - $self->input_path_url($input); - } + local $PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid + $self->SUPER::process_inputs; delete $self->{lei}->{1}; delete $self->{wcb}; # commit } @@ -55,7 +54,7 @@ sub lei_convert { # the main "lei convert" method $self->prepare_inputs($lei, \@inputs) or return; my ($op_c, $ops) = $lei->workers_start($self, 'lei_convert', 1); $lei->{cnv} = $self; - $self->wq_io_do('do_convert', []); + $self->wq_io_do('process_inputs', []); $self->wq_close(1); $op_c->op_wait_event($ops); } diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index dbf655b6..d33143ef 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -39,8 +39,6 @@ sub import_done_wait { # dwaitpid callback my ($arg, $pid) = @_; my ($imp, $lei) = @$arg; $lei->child_error($?, 'non-fatal errors during import') if $?; - my $sto = delete $lei->{sto} // return $lei->fail('BUG: {sto} gone'); - my $wait = $sto->ipc_do('done'); # PublicInbox::LeiStore::done $lei->dclose; } @@ -52,9 +50,7 @@ sub import_done { # EOF callback for main daemon sub net_merge_complete { # callback used by LeiAuth my ($self) = @_; - for my $input (@{$self->{inputs}}) { - $self->wq_io_do('input_path_url', [], $input); - } + $self->wq_io_do('process_inputs'); $self->wq_close(1); } diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm index 505b73ff..40d71f9e 100644 --- a/lib/PublicInbox/LeiInput.pm +++ b/lib/PublicInbox/LeiInput.pm @@ -5,6 +5,7 @@ package PublicInbox::LeiInput; use strict; use v5.10.1; +use PublicInbox::DS; sub check_input_format ($;$) { my ($lei, $files) = @_; @@ -165,6 +166,14 @@ $input is `eml', not --in-format=$in_fmt $self->{inputs} = $inputs; } +sub process_inputs { + my ($self) = @_; + for my $input (@{$self->{inputs}}) { + $self->input_path_url($input); + } + my $wait = $self->{lei}->{sto}->ipc_do('done') if $self->{lei}->{sto}; +} + sub input_only_atfork_child { my ($self) = @_; my $lei = $self->{lei}; diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm index 8b012b16..d572a84a 100644 --- a/lib/PublicInbox/LeiTag.pm +++ b/lib/PublicInbox/LeiTag.pm @@ -73,8 +73,6 @@ sub tag_done_wait { # dwaitpid callback my ($arg, $pid) = @_; my ($tag, $lei) = @$arg; $lei->child_error($?, 'non-fatal errors during tag') if $?; - my $sto = delete $lei->{sto}; - my $wait = $sto->ipc_do('done') if $sto; # PublicInbox::LeiStore::done $lei->dclose; } @@ -86,9 +84,7 @@ sub tag_done { # EOF callback for main daemon sub net_merge_complete { # callback used by LeiAuth my ($self) = @_; - for my $input (@{$self->{inputs}}) { - $self->wq_io_do('input_path_url', [], $input); - } + $self->wq_io_do('process_inputs'); $self->wq_close(1); }