user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH] lei_input: reduce IPC traffic with multiple inputs
Date: Wed, 31 Mar 2021 07:45:51 +0000	[thread overview]
Message-ID: <20210331074551.25641-1-e@80x24.org> (raw)

No point in sending a command for every input when a
single one will do.  We'll also trigger LeiStore->done
sooner in the worker rather than later.
---
 lib/PublicInbox/LeiConvert.pm | 11 +++++------
 lib/PublicInbox/LeiImport.pm  |  6 +-----
 lib/PublicInbox/LeiInput.pm   |  9 +++++++++
 lib/PublicInbox/LeiTag.pm     |  6 +-----
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index fb7a2f3b..0ce49ea9 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -7,6 +7,7 @@ use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
 use PublicInbox::LeiOverview;
+use PublicInbox::DS;
 
 # /^input_/ subs are used by PublicInbox::LeiInput
 
@@ -32,12 +33,10 @@ sub input_maildir_cb {
 	$self->{wcb}->(undef, { kw => $kw }, $eml);
 }
 
-sub do_convert { # via wq_do
+sub process_inputs { # via wq_do
 	my ($self) = @_;
-	$PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid
-	for my $input (@{$self->{inputs}}) {
-		$self->input_path_url($input);
-	}
+	local $PublicInbox::DS::in_loop = 0; # force synchronous dwaitpid
+	$self->SUPER::process_inputs;
 	delete $self->{lei}->{1};
 	delete $self->{wcb}; # commit
 }
@@ -55,7 +54,7 @@ sub lei_convert { # the main "lei convert" method
 	$self->prepare_inputs($lei, \@inputs) or return;
 	my ($op_c, $ops) = $lei->workers_start($self, 'lei_convert', 1);
 	$lei->{cnv} = $self;
-	$self->wq_io_do('do_convert', []);
+	$self->wq_io_do('process_inputs', []);
 	$self->wq_close(1);
 	$op_c->op_wait_event($ops);
 }
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index dbf655b6..d33143ef 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -39,8 +39,6 @@ sub import_done_wait { # dwaitpid callback
 	my ($arg, $pid) = @_;
 	my ($imp, $lei) = @$arg;
 	$lei->child_error($?, 'non-fatal errors during import') if $?;
-	my $sto = delete $lei->{sto} // return $lei->fail('BUG: {sto} gone');
-	my $wait = $sto->ipc_do('done'); # PublicInbox::LeiStore::done
 	$lei->dclose;
 }
 
@@ -52,9 +50,7 @@ sub import_done { # EOF callback for main daemon
 
 sub net_merge_complete { # callback used by LeiAuth
 	my ($self) = @_;
-	for my $input (@{$self->{inputs}}) {
-		$self->wq_io_do('input_path_url', [], $input);
-	}
+	$self->wq_io_do('process_inputs');
 	$self->wq_close(1);
 }
 
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 505b73ff..40d71f9e 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -5,6 +5,7 @@
 package PublicInbox::LeiInput;
 use strict;
 use v5.10.1;
+use PublicInbox::DS;
 
 sub check_input_format ($;$) {
 	my ($lei, $files) = @_;
@@ -165,6 +166,14 @@ $input is `eml', not --in-format=$in_fmt
 	$self->{inputs} = $inputs;
 }
 
+sub process_inputs {
+	my ($self) = @_;
+	for my $input (@{$self->{inputs}}) {
+		$self->input_path_url($input);
+	}
+	my $wait = $self->{lei}->{sto}->ipc_do('done') if $self->{lei}->{sto};
+}
+
 sub input_only_atfork_child {
 	my ($self) = @_;
 	my $lei = $self->{lei};
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 8b012b16..d572a84a 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -73,8 +73,6 @@ sub tag_done_wait { # dwaitpid callback
 	my ($arg, $pid) = @_;
 	my ($tag, $lei) = @$arg;
 	$lei->child_error($?, 'non-fatal errors during tag') if $?;
-	my $sto = delete $lei->{sto};
-	my $wait = $sto->ipc_do('done') if $sto; # PublicInbox::LeiStore::done
 	$lei->dclose;
 }
 
@@ -86,9 +84,7 @@ sub tag_done { # EOF callback for main daemon
 
 sub net_merge_complete { # callback used by LeiAuth
 	my ($self) = @_;
-	for my $input (@{$self->{inputs}}) {
-		$self->wq_io_do('input_path_url', [], $input);
-	}
+	$self->wq_io_do('process_inputs');
 	$self->wq_close(1);
 }
 

                 reply	other threads:[~2021-03-31  7:45 UTC|newest]

Thread overview: [no followups] expand[flat|nested]  mbox.gz  Atom feed

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210331074551.25641-1-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH] lei_input: reduce IPC traffic with multiple inputs' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.mail.public-inbox.meta
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.mail.public-inbox.meta
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git