user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 1/4] lei q: skip lei/store->write_prepare for JSON outputs
Date: Fri, 26 Mar 2021 09:51:23 +0000	[thread overview]
Message-ID: <20210326095126.8184-2-e@80x24.org> (raw)
In-Reply-To: <20210326095126.8184-1-e@80x24.org>

JSON outputs won't write to lei/store at all, so there's
no point in forking the store worker if it's not already
running.

LeiSearch object ($lse) is also fork-safe until it opens a
persistent FD for Xapian/SQLite so we can unconditionally
carry it across fork.
---
 lib/PublicInbox/LeiOverview.pm |  4 ++--
 lib/PublicInbox/LeiQuery.pm    | 16 ++++++++--------
 lib/PublicInbox/LeiToMail.pm   | 16 ++++++++--------
 lib/PublicInbox/LeiXSearch.pm  |  1 +
 4 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index b4d81328..96bfff24 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -223,7 +223,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
 		}
 	} elsif ($self->{fmt} =~ /\A(concat)?json\z/ && $lei->{opt}->{pretty}) {
 		my $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},";
-		my $lse = $lei->{sto}->search;
+		my $lse = $lei->{lse};
 		sub { # DIY prettiness :P
 			my ($smsg, $mitem) = @_;
 			return if $dedupe->is_smsg_dup($smsg);
@@ -247,7 +247,7 @@ sub ovv_each_smsg_cb { # runs in wq worker usually
 		}
 	} elsif ($json) {
 		my $ORS = $self->{fmt} eq 'json' ? ",\n" : "\n"; # JSONL
-		my $lse = $lei->{sto}->search;
+		my $lse = $lei->{lse};
 		sub {
 			my ($smsg, $mitem) = @_;
 			return if $dedupe->is_smsg_dup($smsg);
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 84996e7e..65aa9e87 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -25,8 +25,7 @@ sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
 	my ($self) = @_; # $_[1] = $rbuf
 	if (defined($_[1])) {
 		$_[1] eq '' and return eval {
-			my $lse = delete $self->{lse};
-			$lse->query_approxidate($lse->git,
+			$self->{lse}->query_approxidate($self->{lse}->git,
 						$self->{mset_opt}->{qstr});
 			_start_query($self);
 		};
@@ -50,11 +49,7 @@ sub lei_q {
 	# --local is enabled by default unless --only is used
 	# we'll allow "--only $LOCATION --local"
 	my $sto = $self->_lei_store(1);
-	if (($opt->{'import-remote'} //= 1) |
-			(($opt->{'import-before'} //= \1) ? 1 : 0)) {
-		$sto->write_prepare($self);
-	}
-	my $lse = $sto->search;
+	my $lse = $self->{lse} = $sto->search;
 	if ($opt->{'local'} //= scalar(@only) ? 0 : 1) {
 		$lxs->prepare_external($lse);
 	}
@@ -103,6 +98,12 @@ sub lei_q {
 		return $self->fail("`$mj' writer jobs must be >= 1");
 	}
 	PublicInbox::LeiOverview->new($self) or return;
+	if ($self->{l2m} && ($opt->{'import-remote'} //= 1) |
+				# we use \1 (a ref) to distinguish between
+				# user-supplied and default value
+				(($opt->{'import-before'} //= \1) ? 1 : 0)) {
+		$sto->write_prepare($self);
+	}
 	$self->{l2m} and $self->{l2m}->{-wq_nr_workers} = $mj // do {
 		$mj = POSIX::lround($nproc * 3 / 4); # keep some CPU for git
 		$mj <= 0 ? 1 : $mj;
@@ -131,7 +132,6 @@ sub lei_q {
 no query allowed on command-line with --stdin
 
 		require PublicInbox::InputPipe;
-		$self->{lse} = $lse; # for query_approxidate
 		PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
 		return;
 	}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 1be15707..f71f74cc 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -263,7 +263,7 @@ sub _mbox_write_cb ($$) {
 	my $atomic_append = !defined($ovv->{lock_path});
 	my $dedupe = $lei->{dedupe};
 	$dedupe->prepare_dedupe;
-	my $lse = $lei->{sto} ? $lei->{sto}->search : undef;
+	my $lse = $lei->{lse}; # may be undef
 	sub { # for git_to_mail
 		my ($buf, $smsg, $eml) = @_;
 		$eml //= PublicInbox::Eml->new($buf);
@@ -352,7 +352,7 @@ sub _maildir_write_cb ($$) {
 	my $dedupe = $lei->{dedupe};
 	$dedupe->prepare_dedupe if $dedupe;
 	my $dst = $lei->{ovv}->{dst};
-	my $lse = $lei->{sto} ? $lei->{sto}->search : undef;
+	my $lse = $lei->{lse}; # may be undef
 	sub { # for git_to_mail
 		my ($buf, $smsg, $eml) = @_;
 		$dst // return $lei->fail; # dst may be undef-ed in last run
@@ -373,7 +373,7 @@ sub _imap_write_cb ($$) {
 	my $imap_append = $lei->{net}->can('imap_append');
 	my $mic = $lei->{net}->mic_get($self->{uri});
 	my $folder = $self->{uri}->mailbox;
-	my $lse = $lei->{sto} ? $lei->{sto}->search : undef;
+	my $lse = $lei->{lse}; # may be undef
 	sub { # for git_to_mail
 		my ($bref, $smsg, $eml) = @_;
 		$mic // return $lei->fail; # dst may be undef-ed in last run
@@ -449,7 +449,7 @@ sub _pre_augment_maildir {
 sub _do_augment_maildir {
 	my ($self, $lei) = @_;
 	my $dst = $lei->{ovv}->{dst};
-	my $lse = $lei->{sto}->search if $lei->{opt}->{'import-before'};
+	my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
 	my ($mod, $shard) = @{$self->{shard_info} // []};
 	if ($lei->{opt}->{augment}) {
 		my $dedupe = $lei->{dedupe};
@@ -481,7 +481,7 @@ sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb
 sub _do_augment_imap {
 	my ($self, $lei) = @_;
 	my $net = $lei->{net};
-	my $lse = $lei->{sto}->search if $lei->{opt}->{'import-before'};
+	my $lse = $lei->{opt}->{'import-before'} ? $lei->{lse} : undef;
 	if ($lei->{opt}->{augment}) {
 		my $dedupe = $lei->{dedupe};
 		if ($dedupe && $dedupe->prepare_dedupe) {
@@ -523,9 +523,9 @@ sub _pre_augment_mbox {
 		die "seek($dst): $!\n";
 	}
 	if (!$self->{seekable}) {
-		my $ia = $lei->{opt}->{'import-before'};
+		my $imp_before = $lei->{opt}->{'import-before'};
 		die "--import-before specified but $dst is not seekable\n"
-			if $ia && !ref($ia);
+			if $imp_before && !ref($imp_before);
 		die "--augment specified but $dst is not seekable\n" if
 			$lei->{opt}->{augment};
 	}
@@ -562,7 +562,7 @@ sub _do_augment_mbox {
 		$dedupe->prepare_dedupe if $dedupe;
 	}
 	if ($opt->{'import-before'}) { # the default
-		my $lse = $lei->{sto}->search;
+		my $lse = $lei->{lse};
 		PublicInbox::MboxReader->$fmt($rd, \&_mbox_augment_kw_maybe,
 						$lei, $lse, $opt->{augment});
 		if (!$opt->{augment} and !truncate($out, 0)) {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index f64b2c62..6410e0ea 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -430,6 +430,7 @@ sub do_query {
 	$lei->{1}->autoflush(1);
 	$lei->start_pager if delete $lei->{need_pager};
 	$lei->{ovv}->ovv_begin($lei);
+	die 'BUG: xdb|over open' if $lei->{lse}->{xdb} || $lei->{lse}->{over};
 	if ($l2m) {
 		$l2m->pre_augment($lei);
 		if ($lei->{opt}->{augment} && delete $lei->{early_mua}) {

  reply	other threads:[~2021-03-26  9:51 UTC|newest]

Thread overview: 6+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-03-26  9:51 [PATCH 0/4] lei minor things Eric Wong
2021-03-26  9:51 ` Eric Wong [this message]
2021-03-26  9:51 ` [PATCH 2/4] lei: do not blindly commit to lei/store on close Eric Wong
2021-03-26  9:51 ` [PATCH 3/4] lei: support /dev/fd/[0-2] inputs and outputs in daemon Eric Wong
2021-03-26 23:07   ` [SQUASH] account for /dev/* FDs in prepare_inputs, too Eric Wong
2021-03-26  9:51 ` [PATCH 4/4] lei mark: disallow '!' in labels Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210326095126.8184-2-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 1/4] lei q: skip lei/store->write_prepare for JSON outputs' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.mail.public-inbox.meta
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.mail.public-inbox.meta
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git