diff options
Diffstat (limited to 'lib/PublicInbox/LeiInspect.pm')
| -rw-r--r-- | lib/PublicInbox/LeiInspect.pm | 112 |
1 files changed, 64 insertions, 48 deletions
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm index 2158b996..576ab2c7 100644 --- a/lib/PublicInbox/LeiInspect.pm +++ b/lib/PublicInbox/LeiInspect.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # "lei inspect" general purpose inspector for stuff in SQLite and @@ -12,6 +12,16 @@ use parent qw(PublicInbox::IPC); use PublicInbox::Config; use PublicInbox::MID qw(mids); use PublicInbox::NetReader qw(imap_uri nntp_uri); +use PublicInbox::LeiOverview; +*iso8601 = \&PublicInbox::LeiOverview::iso8601; + +sub _json_prep ($) { + my ($smsg) = @_; + $smsg->{$_} += 0 for qw(bytes lines); # integerize + $smsg->{dt} = iso8601($smsg->{ds}) if defined($smsg->{ds}); + $smsg->{rt} = iso8601($smsg->{ts}) if defined($smsg->{ts}); + +{ %$smsg } # unbless and scalarize +} sub inspect_blob ($$) { my ($lei, $oidhex) = @_; @@ -72,22 +82,9 @@ sub inspect_sync_folder ($$) { $ent } -sub inspect_docid ($$;$) { - my ($lei, $docid, $ent) = @_; - require PublicInbox::Search; - $ent //= {}; - my $xdb; - if ($xdb = delete $ent->{xdb}) { # from inspect_num - } elsif (defined(my $dir = $lei->{opt}->{dir})) { - no warnings 'once'; - $xdb = $PublicInbox::Search::X{Database}->new($dir); - } else { - $xdb = $lei->{lse}->xdb; - } - $xdb or return $lei->fail('no Xapian DB'); - my $doc = $xdb->get_document($docid); # raises +sub _inspect_doc ($$) { + my ($ent, $doc) = @_; my $data = $doc->get_data; - $ent->{docid} = $docid; $ent->{data_length} = length($data); $ent->{description} = $doc->get_description; $ent->{$_} = $doc->$_ for (qw(termlist_count values_count)); @@ -99,7 +96,6 @@ sub inspect_docid ($$;$) { my $term = ($1 // ''); push @{$ent->{terms}->{$term}}, $tn; } - @$_ = sort(@$_) for values %{$ent->{terms} // {}}; $cur = $doc->values_begin; $end = $doc->values_end; for (; $cur != $end; $cur++) { @@ -113,6 +109,24 @@ sub inspect_docid ($$;$) { $ent; } +sub inspect_docid ($$;$) { + my ($lei, $docid, $ent) = @_; + require PublicInbox::Search; + $ent //= {}; + my $xdb; + if ($xdb = delete $ent->{xdb}) { # from inspect_num + } elsif (defined(my $dir = $lei->{opt}->{dir})) { + no warnings 'once'; + $xdb = $PublicInbox::Search::X{Database}->new($dir); + } elsif ($lei->{lse}) { + $xdb = $lei->{lse}->xdb; + } + $xdb or return $lei->fail('no Xapian DB'); + my $doc = $xdb->get_document($docid); # raises + $ent->{docid} = $docid; + _inspect_doc($ent, $doc); +} + sub dir2ibx ($$) { my ($lei, $dir) = @_; if (-f "$dir/ei.lock") { @@ -132,39 +146,43 @@ sub inspect_num ($$) { my $ent = { num => $num }; if (defined(my $dir = $lei->{opt}->{dir})) { $ibx = dir2ibx($lei, $dir) or return; - if ($ent->{xdb} = $ibx->xdb) { - my $num2docid = $lei->{lse}->can('num2docid'); - $docid = $num2docid->($ibx, $num); + if (my $srch = $ibx->search) { + $ent->{xdb} = $srch->xdb and + $docid = $srch->num2docid($num); } - } else { + } elsif ($lei->{lse}) { $ibx = $lei->{lse}; $lei->{lse}->xdb; # set {nshard} for num2docid $docid = $lei->{lse}->num2docid($num); } if ($ibx && $ibx->over) { my $smsg = $ibx->over->get_art($num); - $ent->{smsg} = { %$smsg } if $smsg; + $ent->{smsg} = _json_prep($smsg) if $smsg; } defined($docid) ? inspect_docid($lei, $docid, $ent) : $ent; } sub inspect_mid ($$) { my ($lei, $mid) = @_; - my ($ibx, $over); + my $ibx; my $ent = { mid => $mid }; if (defined(my $dir = $lei->{opt}->{dir})) { - my $num2docid = $lei->{lse}->can('num mid => [ $mid ] 2docid'); - $ibx = dir2ibx($lei, $dir) or return; - # $ent->{xdb} = $ibx->xdb // - # return $lei->fail("no Xapian DB for $dir"); + $ibx = dir2ibx($lei, $dir) } else { $ibx = $lei->{lse}; - $lei->{lse}->xdb; # set {nshard} for num2docid } if ($ibx && $ibx->over) { my ($id, $prev); while (my $smsg = $ibx->over->next_by_mid($mid, \$id, \$prev)) { - push @{$ent->{smsg}}, { %$smsg } + push @{$ent->{smsg}}, _json_prep($smsg); + } + } + if ($ibx && $ibx->search) { + my $mset = $ibx->search->mset(qq{mid:"$mid"}); + for (sort { $a->get_docid <=> $b->get_docid } $mset->items) { + my $tmp = { docid => $_->get_docid }; + _inspect_doc($tmp, $_->get_document); + push @{$ent->{xdoc}}, $tmp; } } $ent; @@ -215,7 +233,8 @@ sub inspect_argv { # via wq_do $lei->{1}->autoflush(0); $lei->out('[') if $multi; while (defined(my $x = shift @$argv)) { - inspect1($lei, $x, scalar(@$argv)) or return; + eval { inspect1($lei, $x, scalar(@$argv)) or return }; + warn "E: $@\n" if $@; } $lei->out(']') if $multi; } @@ -227,24 +246,16 @@ sub inspect_start ($$) { $lei->{wq1} = $self; $lei->wait_wq_events($op_c, $ops); $self->wq_do('inspect_argv'); - $self->wq_close(1); + $self->wq_close; } -sub ins_add { # InputPipe->consume callback - my ($lei) = @_; # $_[1] = $rbuf - if (defined $_[1]) { - $_[1] eq '' and return eval { - my $str = delete $lei->{istr}; - $str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s; - my $eml = PublicInbox::Eml->new(\$str); - inspect_start($lei, [ - 'blob:'.$lei->git_oid($eml)->hexdigest, - map { "mid:$_" } @{mids($eml)} ]); - }; - $lei->{istr} .= $_[1]; - } else { - $lei->fail("error reading stdin: $!"); - } +sub do_inspect { # lei->do_env cb + my ($lei) = @_; + my $str = delete $lei->{stdin_buf}; + PublicInbox::Eml::strip_from($str); + my $eml = PublicInbox::Eml->new(\$str); + inspect_start($lei, [ 'blob:'.$lei->git_oid($eml)->hexdigest, + map { "mid:$_" } @{mids($eml)} ]); } sub lei_inspect { @@ -261,8 +272,7 @@ sub lei_inspect { return $lei->fail(<<'') if @argv; no args allowed on command-line with --stdin - require PublicInbox::InputPipe; - PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei); + $lei->slurp_stdin(\&do_inspect); } else { inspect_start($lei, \@argv); } @@ -274,4 +284,10 @@ sub _complete_inspect { # TODO: message-ids?, blobs? could get expensive... } +sub ipc_atfork_child { + my ($self) = @_; + $self->{lei}->_lei_atfork_child; + $self->SUPER::ipc_atfork_child; +} + 1; |