user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
* [PATCH 0/2] lei inspect: "mid:" prefix and pager
@ 2021-07-01 11:31 Eric Wong
  2021-07-01 11:31 ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
  2021-07-01 11:31 ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong
  0 siblings, 2 replies; 3+ messages in thread
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
  To: meta

Some things that came up while chasing -extindex deduplication
weirdness (and my head hurts because of that...)

Eric Wong (2):
  lei inspect: support automatic pager in output
  lei inspect: support "mid:" (and "m:") prefix

 lib/PublicInbox/LeiInspect.pm | 60 +++++++++++++++++++++++++++--------
 1 file changed, 46 insertions(+), 14 deletions(-)

^ permalink raw reply	[flat|nested] 3+ messages in thread

* [PATCH 1/2] lei inspect: support automatic pager in output
  2021-07-01 11:31 [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
@ 2021-07-01 11:31 ` Eric Wong
  2021-07-01 11:31 ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong
  1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
  To: meta

All commands which output non-trivial amounts of data to
the terminal should support this.
---
 lib/PublicInbox/LeiInspect.pm | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 30714764..9a7900c7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -155,9 +155,6 @@ sub inspect1 ($$$) {
 
 sub lei_inspect {
 	my ($lei, @argv) = @_;
-	$lei->{1}->autoflush(0);
-	my $multi = scalar(@argv) > 1;
-	$lei->out('[') if $multi;
 	$lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
 	$lei->{lse} = ($lei->{opt}->{external} // 1) ? do {
 		my $sto = $lei->_lei_store;
@@ -166,6 +163,10 @@ sub lei_inspect {
 	if ($lei->{opt}->{pretty} || -t $lei->{1}) {
 		$lei->{json}->pretty(1)->indent(2);
 	}
+	$lei->start_pager if -t $lei->{1};
+	$lei->{1}->autoflush(0);
+	my $multi = scalar(@argv) > 1;
+	$lei->out('[') if $multi;
 	while (defined(my $x = shift @argv)) {
 		inspect1($lei, $x, scalar(@argv)) or return;
 	}

^ permalink raw reply	[flat|nested] 3+ messages in thread

* [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix
  2021-07-01 11:31 [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
  2021-07-01 11:31 ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
@ 2021-07-01 11:31 ` Eric Wong
  1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2021-07-01 11:31 UTC (permalink / raw)
  To: meta

Using this to track down deduplication failures in -extindex...
---
 lib/PublicInbox/LeiInspect.pm | 53 +++++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 11 deletions(-)

diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 9a7900c7..574da7a7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -98,22 +98,29 @@ sub inspect_docid ($$;$) {
 	$ent;
 }
 
+sub dir2ibx ($$) {
+	my ($lei, $dir) = @_;
+	if (-f "$dir/ei.lock") {
+		require PublicInbox::ExtSearch;
+		PublicInbox::ExtSearch->new($dir);
+	} elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
+		require PublicInbox::Inbox; # v2, v1
+		bless { inboxdir => $dir }, 'PublicInbox::Inbox';
+	} else {
+		$lei->fail("no (indexed) inbox or extindex at $dir");
+	}
+}
+
 sub inspect_num ($$) {
 	my ($lei, $num) = @_;
 	my ($docid, $ibx);
 	my $ent = { num => $num };
 	if (defined(my $dir = $lei->{opt}->{dir})) {
-		my $num2docid = $lei->{lse}->can('num2docid');
-		if (-f "$dir/ei.lock") {
-			require PublicInbox::ExtSearch;
-			$ibx = PublicInbox::ExtSearch->new($dir);
-		} elsif (-f "$dir/inbox.lock" || -d "$dir/public-inbox") {
-			require PublicInbox::Inbox; # v2, v1
-			$ibx = bless { inboxdir => $dir }, 'PublicInbox::Inbox';
+		$ibx = dir2ibx($lei, $dir) or return;
+		if ($ent->{xdb} = $ibx->xdb) {
+			my $num2docid = $lei->{lse}->can('num2docid');
+			$docid = $num2docid->($ibx, $num);
 		}
-		$ent->{xdb} = $ibx->xdb //
-			return $lei->fail("no Xapian DB for $dir");
-		$docid = $num2docid->($ibx, $num);
 	} else {
 		$ibx = $lei->{lse};
 		$lei->{lse}->xdb; # set {nshard} for num2docid
@@ -123,7 +130,29 @@ sub inspect_num ($$) {
 		my $smsg = $ibx->over->get_art($num);
 		$ent->{smsg} = { %$smsg } if $smsg;
 	}
-	inspect_docid($lei, $docid, $ent);
+	defined($docid) ? inspect_docid($lei, $docid, $ent) : $ent;
+}
+
+sub inspect_mid ($$) {
+	my ($lei, $mid) = @_;
+	my ($ibx, $over);
+	my $ent = { mid => $mid };
+	if (defined(my $dir = $lei->{opt}->{dir})) {
+		my $num2docid = $lei->{lse}->can('num mid => [ $mid ] 2docid');
+		$ibx = dir2ibx($lei, $dir) or return;
+		# $ent->{xdb} = $ibx->xdb //
+			# return $lei->fail("no Xapian DB for $dir");
+	} else {
+		$ibx = $lei->{lse};
+		$lei->{lse}->xdb; # set {nshard} for num2docid
+	}
+	if ($ibx && $ibx->over) {
+		my ($id, $prev);
+		while (my $smsg = $ibx->over->next_by_mid($mid, \$id, \$prev)) {
+			push @{$ent->{smsg}}, { %$smsg }
+		}
+	}
+	$ent;
 }
 
 sub inspect1 ($$$) {
@@ -145,6 +174,8 @@ sub inspect1 ($$$) {
 		$ent = inspect_docid($lei, $1 + 0);
 	} elsif ($item =~ m!\Anum:([0-9]+)\z!) {
 		$ent = inspect_num($lei, $1 + 0);
+	} elsif ($item =~ m!\A(?:mid|m):(.+)\z!) {
+		$ent = inspect_mid($lei, $1);
 	} else { # TODO: more things
 		return $lei->fail("$item not understood");
 	}

^ permalink raw reply	[flat|nested] 3+ messages in thread

end of thread, other threads:[~2021-07-01 11:31 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-07-01 11:31 [PATCH 0/2] lei inspect: "mid:" prefix and pager Eric Wong
2021-07-01 11:31 ` [PATCH 1/2] lei inspect: support automatic pager in output Eric Wong
2021-07-01 11:31 ` [PATCH 2/2] lei inspect: support "mid:" (and "m:") prefix Eric Wong

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.mail.public-inbox.meta
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.mail.public-inbox.meta
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git