From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-3.8 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B535C20078 for ; Thu, 4 Feb 2021 09:59:31 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 07/10] lei q: eliminate $not_done temporary git dir hack Date: Thu, 4 Feb 2021 00:59:27 -0900 Message-Id: <20210204095930.20278-8-e@80x24.org> In-Reply-To: <20210204095930.20278-1-e@80x24.org> References: <20210204095930.20278-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Another step towards simplifying lei internals. None of our current uses of ->wq_do involve FD passing, and the plan is only rely on FD passing between lei-daemon and lei(1). Internally, it ought to be possible for lei-daemon internal bits to be ordered properly to not need FD passing. --- lib/PublicInbox/LeiOverview.pm | 23 ++--------------------- lib/PublicInbox/LeiToMail.pm | 3 +-- lib/PublicInbox/LeiXSearch.pm | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm index 3125f015..d3df4faa 100644 --- a/lib/PublicInbox/LeiOverview.pm +++ b/lib/PublicInbox/LeiOverview.pm @@ -147,17 +147,6 @@ sub _unbless_smsg { sub ovv_atexit_child { my ($self, $lei) = @_; - if (my $l2m = $lei->{l2m}) { - # wait for ->write_mail work we submitted to lei2mail - if (my $rd = delete $l2m->{each_smsg_done}) { - read($rd, my $buf, 1); # wait for EOF - } - } - # order matters, git->{-tmp}->DESTROY must not fire until - # {each_smsg_done} hits EOF above - if (my $git = delete $self->{git}) { - $git->async_wait_all; - } if (my $bref = delete $lei->{ovv_buf}) { my $lk = $self->lock_for_scope; $lei->out($$bref); @@ -213,19 +202,11 @@ sub ovv_each_smsg_cb { # runs in wq worker usually $wcb->(undef, $smsg, $eml); }; } elsif ($l2m && $l2m->{-wq_s1}) { - # $io->[0] becomes a notification pipe that triggers EOF - # in this wq worker when all outstanding ->write_mail - # calls are complete - my $io = []; - pipe($l2m->{each_smsg_done}, $io->[0]) or die "pipe: $!"; - fcntl($io->[0], 1031, 4096) if $^O eq 'linux'; # F_SETPIPE_SZ - my $git = $ibxish->git; # (LeiXSearch|Inbox|ExtSearch)->git - $self->{git} = $git; - my $git_dir = $git->{git_dir}; + my $git_dir = $ibxish->git->{git_dir}; sub { my ($smsg, $mitem) = @_; $smsg->{pct} = get_pct($mitem) if $mitem; - $l2m->wq_do('write_mail', $io, $git_dir, $smsg); + $l2m->wq_do('write_mail', [], $git_dir, $smsg); } } elsif ($self->{fmt} =~ /\A(concat)?json\z/ && $lei->{opt}->{pretty}) { my $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},"; diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index 1f815e40..4f847221 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -490,10 +490,9 @@ sub poke_dst { sub write_mail { # via ->wq_do my ($self, $git_dir, $smsg) = @_; - my $not_done = delete $self->{0} // die 'BUG: $not_done missing'; my $git = $self->{"$$\0$git_dir"} //= PublicInbox::Git->new($git_dir); git_async_cat($git, $smsg->{blob}, \&git_to_mail, - [$self->{wcb}, $smsg, $not_done]); + [$self->{wcb}, $smsg]); } sub wq_atexit_child { diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index e7f0ef63..2dc44414 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -287,12 +287,15 @@ sub query_remote_mboxrd { $lei->{ovv}->ovv_atexit_child($lei); } -sub git { +# called by LeiOverview::each_smsg_cb +sub git { $_[0]->{git_tmp} // die 'BUG: caller did not set {git_tmp}' } + +sub git_tmp ($) { my ($self) = @_; my (%seen, @dirs); - my $tmp = File::Temp->newdir('lei_xsrch_git-XXXXXXXX', TMPDIR => 1); - for my $ibx (@{$self->{shard2ibx} // []}) { - my $d = File::Spec->canonpath($ibx->git->{git_dir}); + my $tmp = File::Temp->newdir("lei_xsearch_git.$$-XXXX", TMPDIR => 1); + for my $ibxish (locals($self)) { + my $d = File::Spec->canonpath($ibxish->git->{git_dir}); $seen{$d} //= push @dirs, "$d/objects\n" } my $git_dir = $tmp->dirname; @@ -428,6 +431,11 @@ sub do_query { # 1031: F_SETPIPE_SZ fcntl($lei->{startq}, 1031, 4096) if $^O eq 'linux'; } + if (!$lei->{opt}->{thread} && locals($self)) { # for query_mset + # lei->{git_tmp} is set for wq_wait_old so we don't + # delete until all lei2mail + lei_xsearch workers are reaped + $lei->{git_tmp} = $self->{git_tmp} = git_tmp($self); + } $self->wq_workers_start('lei_xsearch', $self->{jobs}, $lei->oldset, { lei => $lei }); my $op = delete $lei->{pkt_op_c};