user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 2/5] index: ignore some warnings, set {current_info} for v1
Date: Wed, 16 Dec 2020 23:19:03 +0000	[thread overview]
Message-ID: <20201216231906.6356-3-e@80x24.org> (raw)
In-Reply-To: <20201216231906.6356-1-e@80x24.org>

-index runs on data that's already frozen in git, so there's
no point in warning users about it.

While we're at it, set the {current_info} prefix for v1 as
we do in v2 inboxes in case new problems show up.
---
 lib/PublicInbox/Admin.pm     | 23 ++++++++++++++---------
 lib/PublicInbox/SearchIdx.pm |  6 +++++-
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index eeef2f63..3977d812 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -209,12 +209,20 @@ sub index_terminate {
 
 sub index_inbox {
 	my ($ibx, $im, $opt) = @_;
+	require PublicInbox::InboxWritable;
 	my $jobs = delete $opt->{jobs} if $opt;
 	if (my $pr = $opt->{-progress}) {
 		$pr->("indexing $ibx->{inboxdir} ...\n");
 	}
 	local %SIG = %SIG;
 	setup_signals(\&index_terminate, $ibx);
+	my $warn_cb = $SIG{__WARN__} // sub { print STDERR @_ };
+	my $idx = { current_info => $ibx->{inboxdir} };
+	my $warn_ignore = PublicInbox::InboxWritable->can('warn_ignore');
+	local $SIG{__WARN__} = sub {
+		return if $warn_ignore->(@_);
+		$warn_cb->($idx->{current_info}, ': ', @_);
+	};
 	if (ref($ibx) && $ibx->version == 2) {
 		eval { require PublicInbox::V2Writable };
 		die "v2 requirements not met: $@\n" if $@;
@@ -226,21 +234,18 @@ sub index_inbox {
 			} else {
 				my $n = $v2w->{shards};
 				if ($jobs < ($n + 1) && !$opt->{reshard}) {
-					warn
-"Unable to respect --jobs=$jobs on index, inbox was created with $n shards\n";
+					warn <<EOM;
+Unable to respect --jobs=$jobs on index, inbox was created with $n shards
+EOM
 				}
 			}
 		}
-		my $warn_cb = $SIG{__WARN__} || sub { print STDERR @_ };
-		local $SIG{__WARN__} = sub {
-			$warn_cb->($v2w->{current_info}, ': ', @_);
-		};
-		$v2w->index_sync($opt);
+		$idx = $v2w;
 	} else {
 		require PublicInbox::SearchIdx;
-		my $s = PublicInbox::SearchIdx->new($ibx, 1);
-		$s->index_sync($opt);
+		$idx = PublicInbox::SearchIdx->new($ibx, 1);
 	}
+	$idx->index_sync($opt);
 }
 
 sub progress_prepare ($) {
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index c6d2a0e8..b731f698 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -617,6 +617,7 @@ sub index_both { # git->cat_async callback
 	$size += crlf_adjust($$bref);
 	my $smsg = bless { bytes => $size, blob => $oid }, 'PublicInbox::Smsg';
 	my $self = $sync->{sidx};
+	local $self->{current_info} = "$self->{current_info}: $oid";
 	my $eml = PublicInbox::Eml->new($bref);
 	$smsg->{num} = index_mm($self, $eml, $oid, $sync) or
 		die "E: could not generate NNTP article number for $oid";
@@ -628,7 +629,9 @@ sub index_both { # git->cat_async callback
 sub unindex_both { # git->cat_async callback
 	my ($bref, $oid, $type, $size, $sync) = @_;
 	return if is_bad_blob($oid, $type, $size, $sync->{oid});
-	unindex_eml($sync->{sidx}, $oid, PublicInbox::Eml->new($bref));
+	my $self = $sync->{sidx};
+	local $self->{current_info} = "$self->{current_info}: $oid";
+	unindex_eml($self, $oid, PublicInbox::Eml->new($bref));
 	# may be undef if leftover
 	if (defined(my $cur_cmt = $sync->{cur_cmt})) {
 		${$sync->{latest_cmt}} = $cur_cmt;
@@ -872,6 +875,7 @@ sub _index_sync {
 	my ($self, $opt) = @_;
 	my $tip = $opt->{ref} || 'HEAD';
 	my $ibx = $self->{ibx};
+	local $self->{current_info} = "$ibx->{inboxdir}";
 	$self->{batch_bytes} = $opt->{batch_size} // $BATCH_BYTES;
 	$ibx->git->batch_prepare;
 	my $pr = $opt->{-progress};

  parent reply	other threads:[~2020-12-16 23:19 UTC|newest]

Thread overview: 6+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-12-16 23:19 [PATCH 0/5] various indexing cleanups and quieting Eric Wong
2020-12-16 23:19 ` [PATCH 1/5] inboxwritable: warn_ignore: "Bad UTF7 data escape" Eric Wong
2020-12-16 23:19 ` Eric Wong [this message]
2020-12-16 23:19 ` [PATCH 3/5] inbox: simplify v2 epoch counting Eric Wong
2020-12-16 23:19 ` [PATCH 4/5] inboxwritable: drop git_dir_n sub Eric Wong
2020-12-16 23:19 ` [PATCH 5/5] extsearchidx: no need to make InboxWritable Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20201216231906.6356-3-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).