* [PATCH 02/12] lei inspect: support NNTP URLs
2021-09-21 7:41 7% [PATCH 00/12] lei: fix various annoyances Eric Wong
@ 2021-09-21 7:41 3% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
No reason not to support them, since there's more
public-inbox-nntpd instances than -imapd instances,
currently.
---
lib/PublicInbox/LeiInspect.pm | 48 +++++++++++++++++++++++++++-------
lib/PublicInbox/LeiMailSync.pm | 40 +++++++++++++++++++++++++++-
lib/PublicInbox/TestCommon.pm | 11 ++++++--
t/lei-import-nntp.t | 21 +++++++++++++++
4 files changed, 108 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 48da826b..ab2c98d9 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -11,6 +11,7 @@ use v5.10.1;
use parent qw(PublicInbox::IPC);
use PublicInbox::Config;
use PublicInbox::MID qw(mids);
+use PublicInbox::NetReader qw(imap_uri nntp_uri);
sub inspect_blob ($$) {
my ($lei, $oidhex) = @_;
@@ -32,13 +33,33 @@ sub inspect_imap_uid ($$) {
my $ent = {};
my $lms = $lei->lms or return $ent;
my $oidhex = $lms->imap_oid($lei, $uid_uri);
- if (ref(my $err = $oidhex)) { # art2folder error
+ if (ref(my $err = $oidhex)) { # arg2folder error
$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
}
$ent->{$$uid_uri} = $oidhex;
$ent;
}
+sub inspect_nntp_range {
+ my ($lei, $uri) = @_;
+ my ($ng, $beg, $end) = $uri->group;
+ $uri = $uri->clone;
+ $uri->group($ng);
+ my $ent = {};
+ my $ret = { "$uri" => $ent };
+ my $lms = $lei->lms or return $ret;
+ my $err = $lms->arg2folder($lei, my $folders = [ $$uri ]);
+ if ($err) {
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ }
+ $end //= $beg;
+ for my $art ($beg..$end) {
+ my $oidbin = $lms->imap_oidbin($folders->[0], $art);
+ $ent->{$art} = $oidbin ? unpack('H*', $oidbin) : undef;
+ }
+ $ret;
+}
+
sub inspect_sync_folder ($$) {
my ($lei, $folder) = @_;
my $ent = {};
@@ -161,14 +182,6 @@ sub inspect1 ($$$) {
my $ent;
if ($item =~ /\Ablob:(.+)/) {
$ent = inspect_blob($lei, $1);
- } elsif ($item =~ m!\Aimaps?://!i) {
- require PublicInbox::URIimap;
- my $uri = PublicInbox::URIimap->new($item);
- if (defined($uri->uid)) {
- $ent = inspect_imap_uid($lei, $uri);
- } else {
- $ent = inspect_sync_folder($lei, $item);
- }
} elsif ($item =~ m!\A(?:maildir|mh):!i || -d $item) {
$ent = inspect_sync_folder($lei, $item);
} elsif ($item =~ m!\Adocid:([0-9]+)\z!) {
@@ -177,6 +190,23 @@ sub inspect1 ($$$) {
$ent = inspect_num($lei, $1 + 0);
} elsif ($item =~ m!\A(?:mid|m):(.+)\z!) {
$ent = inspect_mid($lei, $1);
+ } elsif (my $iuri = imap_uri($item)) {
+ if (defined($iuri->uid)) {
+ $ent = inspect_imap_uid($lei, $iuri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ } elsif (my $nuri = nntp_uri($item)) {
+ if (defined(my $mid = $nuri->message)) {
+ $ent = inspect_mid($lei, $mid);
+ } else {
+ my ($group, $beg, $end) = $nuri->group;
+ if (defined($beg)) {
+ $ent = inspect_nntp_range($lei, $nuri);
+ } else {
+ $ent = inspect_sync_folder($lei, $item);
+ }
+ }
} else { # TODO: more things
return $lei->fail("$item not understood");
}
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index f185b585..d9b9e117 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -247,12 +247,14 @@ sub location_stats {
SELECT COUNT(name) FROM blob2name WHERE fid = ?
$ret->{'name.count'} = $row if $row;
+ my $ntype = ($folder =~ m!\A(?:nntps?|s?news)://!i) ? 'article' :
+ (($folder =~ m!\Aimaps?://!i) ? 'uid' : "TODO<$folder>");
for my $op (qw(count min max)) {
($row) = $dbh->selectrow_array(<<"", undef, $fid);
SELECT $op(uid) FROM blob2num WHERE fid = ?
$row or last;
- $ret->{"uid.$op"} = $row;
+ $ret->{"$ntype.$op"} = $row;
}
$ret;
}
@@ -369,6 +371,30 @@ sub match_imap_url {
"E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n";
}
+sub match_nntp_url ($$$) {
+ my ($self, $url, $all) = @_; # $all = [ $lms->folders ];
+ $all //= [ $self->folders ];
+ require PublicInbox::URInntps;
+ my $want = PublicInbox::URInntps->new($url)->canonical;
+ my ($s, $h, $p) = ($want->scheme, $want->host, $want->port);
+ my $ng = $want->group; # force scalar (no article ranges)
+ my @uri = map { PublicInbox::URInntps->new($_)->canonical }
+ grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$ng\E\b!, @$all);
+ my @match;
+ for my $x (@uri) {
+ next if $x->group ne $ng || $x->host ne $h || $x->port != $p;
+ # maybe user was forgotten on CLI:
+ if (defined($x->userinfo) && !defined($want->userinfo)) {
+ push @match, $x;
+ } elsif (($x->userinfo//"\0") eq ($want->userinfo//"\0")) {
+ push @match, $x;
+ }
+ }
+ return @match if wantarray;
+ scalar(@match) <= 1 ? $match[0] :
+ "E: `$url' is ambiguous:\n\t".join("\n\t", @match)."\n";
+}
+
# returns undef on failure, number on success
sub group2folders {
my ($self, $lei, $all, $folders) = @_;
@@ -428,6 +454,18 @@ sub arg2folder {
$_ = $$res;
push(@{$err->{qerr}}, <<EOM);
# using `$res' instead of `$orig'
+EOM
+ } else {
+ $lei->err($res) if defined $res;
+ push @no, $orig;
+ }
+ } elsif (m!\A(?:nntps?|s?news)://!i) {
+ my $orig = $_;
+ my $res = match_nntp_url($self, $orig, \@all);
+ if (ref $res) {
+ $_ = $$res;
+ push(@{$err->{qerr}}, <<EOM);
+# using `$res' instead of `$orig'
EOM
} else {
$lei->err($res) if defined $res;
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 0ee4b228..9e152394 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -18,7 +18,7 @@ BEGIN {
run_script start_script key2sub xsys xsys_e xqx eml_load tick
have_xapian_compact json_utf8 setup_public_inboxes create_inbox
tcp_host_port test_lei lei lei_ok $lei_out $lei_err $lei_opt
- test_httpd xbail require_cmd);
+ test_httpd xbail require_cmd is_xdeeply);
require Test::More;
my @methods = grep(!/\W/, @Test::More::EXPORT);
eval(join('', map { "*$_=\\&Test::More::$_;" } @methods));
@@ -520,6 +520,13 @@ sub json_utf8 () {
state $x = ref(PublicInbox::Config->json)->new->utf8->canonical;
}
+sub is_xdeeply ($$$) {
+ my ($x, $y, $desc) = @_;
+ my $ok = is_deeply($x, $y, $desc);
+ diag explain([$x, '!=', $y]) if !$ok;
+ $ok;
+}
+
sub test_lei {
SKIP: {
my ($cb) = pop @_;
@@ -590,7 +597,7 @@ SKIP: {
my $f = "$daemon_xrd/lei/errors.log";
open my $fh, '<', $f or BAIL_OUT "$f: $!";
my @l = <$fh>;
- is_deeply(\@l, [],
+ is_xdeeply(\@l, [],
"$t daemon XDG_RUNTIME_DIR/lei/errors.log empty");
}
}; # SKIP if missing git 2.6+ || Xapian || SQLite || json
diff --git a/t/lei-import-nntp.t b/t/lei-import-nntp.t
index df0594d4..0b080781 100644
--- a/t/lei-import-nntp.t
+++ b/t/lei-import-nntp.t
@@ -49,6 +49,15 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $end = $high - 1;
lei_ok qw(import), "$url/$high";
+ lei_ok('inspect', $url); is_xdeeply(json_utf8->decode($lei_out), {
+ $url => { 'article.count' => 1,
+ 'article.min' => $high,
+ 'article.max' => $high, }
+ }, 'inspect output for URL after single message') or diag $lei_out;
+ lei_ok('inspect', "$url/$high");
+ my $x = json_utf8->decode($lei_out);
+ like($x->{$url}->{$high}, qr/\A[a-f0-9]{40,}\z/, 'inspect shows blob');
+
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article number not stored as folder');
lei_ok qw(q z:0..); my $one = json_utf8->decode($lei_out);
@@ -57,6 +66,18 @@ test_lei({ tmpdir => $tmpdir }, sub {
local $ENV{HOME} = "$tmpdir/h3";
lei_ok qw(import), "$url/$low-$end";
+ lei_ok('inspect', $url); is_xdeeply(json_utf8->decode($lei_out), {
+ $url => { 'article.count' => $end - $low + 1,
+ 'article.min' => $low,
+ 'article.max' => $end, }
+ }, 'inspect output for URL after range') or diag $lei_out;
+ lei_ok('inspect', "$url/$low-$end");
+ $x = json_utf8->decode($lei_out);
+ is_deeply([ ($low..$end) ], [ sort { $a <=> $b } keys %{$x->{$url}} ],
+ 'inspect range shows range');
+ is(scalar(grep(/\A[a-f0-9]{40,}\z/, values %{$x->{$url}})),
+ $end - $low + 1, 'all values are git blobs');
+
lei_ok 'ls-mail-sync';
is($lei_out, "$url\n", 'article range not stored as folder');
lei_ok qw(q z:0..); my $start = json_utf8->decode($lei_out);
^ permalink raw reply related [relevance 3%]
* [PATCH 00/12] lei: fix various annoyances
@ 2021-09-21 7:41 7% Eric Wong
2021-09-21 7:41 3% ` [PATCH 02/12] lei inspect: support NNTP URLs Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-09-21 7:41 UTC (permalink / raw)
To: meta
Eric Wong (12):
lei inspect: convert to WQ worker
lei inspect: support NNTP URLs
lei_mail_sync: account for non-unique cases
lei: simplify internal arg2folder usage
lei lcat: use single queue for ordering
doc: lei-security: section for WIP auth methods
lei lcat: support NNTP URLs
lei: various completion improvements
lei q: show progress on >1s preparation phase
search: drop reopen retry message
lei q: update messages to reflect --save default
lei q: improve --limit behavior and progress
Documentation/lei-q.pod | 6 +-
Documentation/lei-security.pod | 8 ++
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiExportKw.pm | 12 +--
lib/PublicInbox/LeiExternal.pm | 5 ++
lib/PublicInbox/LeiForgetMailSync.pm | 9 +--
lib/PublicInbox/LeiImport.pm | 20 +++--
lib/PublicInbox/LeiImportKw.pm | 12 ++-
lib/PublicInbox/LeiInspect.pm | 103 +++++++++++++++++---------
lib/PublicInbox/LeiLcat.pm | 91 +++++++++++++----------
lib/PublicInbox/LeiLsMailSource.pm | 11 +--
lib/PublicInbox/LeiMailSync.pm | 102 +++++++++++++++++--------
lib/PublicInbox/LeiNoteEvent.pm | 5 +-
lib/PublicInbox/LeiQuery.pm | 9 ++-
lib/PublicInbox/LeiRefreshMailSync.pm | 18 +++--
lib/PublicInbox/LeiSavedSearch.pm | 22 +++---
lib/PublicInbox/LeiTag.pm | 14 ++--
lib/PublicInbox/LeiToMail.pm | 23 +++++-
lib/PublicInbox/LeiUp.pm | 25 +++----
lib/PublicInbox/LeiXSearch.pm | 36 ++++++---
lib/PublicInbox/Search.pm | 1 -
lib/PublicInbox/SharedKV.pm | 19 +++--
lib/PublicInbox/TestCommon.pm | 11 ++-
t/lei-import-nntp.t | 44 +++++++++++
t/lei-q-save.t | 6 +-
t/lei-watch.t | 2 +-
26 files changed, 414 insertions(+), 202 deletions(-)
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-09-21 7:41 7% [PATCH 00/12] lei: fix various annoyances Eric Wong
2021-09-21 7:41 3% ` [PATCH 02/12] lei inspect: support NNTP URLs Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).