[PATCH 0/2] lei inspect stuffs

[PATCH 0/2] lei inspect stuffs

[Eric Wong]

Some obvious fixes while working extindex stuff..

Eric Wong (2):
  lei inspect: integerize "bytes" and "lines" fields
  lei inspect: fix "mid:" prefix, expand to Xapian

 MANIFEST                      |  1 +
 lib/PublicInbox/LeiInspect.pm | 69 +++++++++++++++++++++--------------
 t/lei-inspect.t               | 14 +++++++
 3 files changed, 56 insertions(+), 28 deletions(-)
 create mode 100644 t/lei-inspect.t

[PATCH 1/2] lei inspect: integerize "bytes" and "lines" fields

[Eric Wong]

These are always numeric, but none of the Perl code cares;
but we want to prevent JSON from quoting them.
---
 lib/PublicInbox/LeiInspect.pm | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2158b996a3a4..f18e31c5c8f4 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -13,6 +13,12 @@ use PublicInbox::Config;
 use PublicInbox::MID qw(mids);
 use PublicInbox::NetReader qw(imap_uri nntp_uri);
 
+sub _json_prep ($) {
+	my ($smsg) = @_;
+	$smsg->{$_} += 0 for qw(bytes lines); # integerize
+	+{ %$smsg } # unbless and scalarize
+}
+
 sub inspect_blob ($$) {
 	my ($lei, $oidhex) = @_;
 	my $ent = {};
@@ -143,7 +149,7 @@ sub inspect_num ($$) {
 	}
 	if ($ibx && $ibx->over) {
 		my $smsg = $ibx->over->get_art($num);
-		$ent->{smsg} = { %$smsg } if $smsg;
+		$ent->{smsg} = _json_prep($smsg) if $smsg;
 	}
 	defined($docid) ? inspect_docid($lei, $docid, $ent) : $ent;
 }
@@ -164,7 +170,7 @@ sub inspect_mid ($$) {
 	if ($ibx && $ibx->over) {
 		my ($id, $prev);
 		while (my $smsg = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
-			push @{$ent->{smsg}}, { %$smsg }
+			push @{$ent->{smsg}}, _json_prep($smsg);
 		}
 	}
 	$ent;

[PATCH 2/2] lei inspect: fix "mid:" prefix, expand to Xapian

[Eric Wong]

This fixes inspect for uninitialized instances, and adds Xapian
("xdoc") output if available.

Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Message-ID: <20211001204943.l4yl6xvc45c5eapz@meerkat.local>
---
 MANIFEST                      |  1 +
 lib/PublicInbox/LeiInspect.pm | 59 ++++++++++++++++++++---------------
 t/lei-inspect.t               | 14 +++++++++
 3 files changed, 48 insertions(+), 26 deletions(-)
 create mode 100644 t/lei-inspect.t

diff --git a/MANIFEST b/MANIFEST
index 929f5f869c5b..74b28d2d54ab 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -450,6 +450,7 @@ t/lei-import-maildir.t
 t/lei-import-nntp.t
 t/lei-import.t
 t/lei-index.t
+t/lei-inspect.t
 t/lei-lcat.t
 t/lei-mirror.psgi
 t/lei-mirror.t
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index f18e31c5c8f4..590dfdabca56 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -78,22 +78,9 @@ sub inspect_sync_folder ($$) {
 	$ent
 }
 
-sub inspect_docid ($$;$) {
-	my ($lei, $docid, $ent) = @_;
-	require PublicInbox::Search;
-	$ent //= {};
-	my $xdb;
-	if ($xdb = delete $ent->{xdb}) { # from inspect_num
-	} elsif (defined(my $dir = $lei->{opt}->{dir})) {
-		no warnings 'once';
-		$xdb = $PublicInbox::Search::X{Database}->new($dir);
-	} else {
-		$xdb = $lei->{lse}->xdb;
-	}
-	$xdb or return $lei->fail('no Xapian DB');
-	my $doc = $xdb->get_document($docid); # raises
+sub _inspect_doc ($$) {
+	my ($ent, $doc) = @_;
 	my $data = $doc->get_data;
-	$ent->{docid} = $docid;
 	$ent->{data_length} = length($data);
 	$ent->{description} = $doc->get_description;
 	$ent->{$_} = $doc->$_ for (qw(termlist_count values_count));
@@ -119,6 +106,24 @@ sub inspect_docid ($$;$) {
 	$ent;
 }
 
+sub inspect_docid ($$;$) {
+	my ($lei, $docid, $ent) = @_;
+	require PublicInbox::Search;
+	$ent //= {};
+	my $xdb;
+	if ($xdb = delete $ent->{xdb}) { # from inspect_num
+	} elsif (defined(my $dir = $lei->{opt}->{dir})) {
+		no warnings 'once';
+		$xdb = $PublicInbox::Search::X{Database}->new($dir);
+	} elsif ($lei->{lse}) {
+		$xdb = $lei->{lse}->xdb;
+	}
+	$xdb or return $lei->fail('no Xapian DB');
+	my $doc = $xdb->get_document($docid); # raises
+	$ent->{docid} = $docid;
+	_inspect_doc($ent, $doc);
+}
+
 sub dir2ibx ($$) {
 	my ($lei, $dir) = @_;
 	if (-f "$dir/ei.lock") {
@@ -138,11 +143,9 @@ sub inspect_num ($$) {
 	my $ent = { num => $num };
 	if (defined(my $dir = $lei->{opt}->{dir})) {
 		$ibx = dir2ibx($lei, $dir) or return;
-		if ($ent->{xdb} = $ibx->xdb) {
-			my $num2docid = $lei->{lse}->can('num2docid');
-			$docid = $num2docid->($ibx, $num);
-		}
-	} else {
+		$ent->{xdb} = $ibx->xdb and # for inspect_docid
+			$docid = PublicInbox::LeiSearch::num2docid($ibx, $num);
+	} elsif ($lei->{lse}) {
 		$ibx = $lei->{lse};
 		$lei->{lse}->xdb; # set {nshard} for num2docid
 		$docid = $lei->{lse}->num2docid($num);
@@ -156,16 +159,12 @@ sub inspect_num ($$) {
 
 sub inspect_mid ($$) {
 	my ($lei, $mid) = @_;
-	my ($ibx, $over);
+	my $ibx;
 	my $ent = { mid => $mid };
 	if (defined(my $dir = $lei->{opt}->{dir})) {
-		my $num2docid = $lei->{lse}->can('num mid => [ $mid ] 2docid');
-		$ibx = dir2ibx($lei, $dir) or return;
-		# $ent->{xdb} = $ibx->xdb //
-			# return $lei->fail("no Xapian DB for $dir");
+		$ibx = dir2ibx($lei, $dir)
 	} else {
 		$ibx = $lei->{lse};
-		$lei->{lse}->xdb; # set {nshard} for num2docid
 	}
 	if ($ibx && $ibx->over) {
 		my ($id, $prev);
@@ -173,6 +172,14 @@ sub inspect_mid ($$) {
 			push @{$ent->{smsg}}, _json_prep($smsg);
 		}
 	}
+	if ($ibx && $ibx->search) {
+		my $mset = $ibx->search->mset(qq{mid:"$mid"});
+		for (sort { $a->get_docid <=> $b->get_docid } $mset->items) {
+			my $tmp = { docid => $_->get_docid };
+			_inspect_doc($tmp, $_->get_document);
+			push @{$ent->{xdoc}}, $tmp;
+		}
+	}
 	$ent;
 }
 
diff --git a/t/lei-inspect.t b/t/lei-inspect.t
new file mode 100644
index 000000000000..077d0d13e84d
--- /dev/null
+++ b/t/lei-inspect.t
@@ -0,0 +1,14 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+
+test_lei(sub {
+	my ($ro_home, $cfg_path) = setup_public_inboxes;
+	lei_ok qw(inspect --dir), "$ro_home/t1", 'mid:testmessage@example.com';
+	my $ent = json_utf8->decode($lei_out);
+	is(ref($ent->{smsg}), 'ARRAY', 'smsg array');
+	is(ref($ent->{xdoc}), 'ARRAY', 'xdoc array');
+});
+
+done_testing;