diff options
Diffstat (limited to 'lib/PublicInbox/LeiOverview.pm')
| -rw-r--r-- | lib/PublicInbox/LeiOverview.pm | 88 |
1 files changed, 45 insertions, 43 deletions
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm index 4db1d8c8..0529bbe4 100644 --- a/lib/PublicInbox/LeiOverview.pm +++ b/lib/PublicInbox/LeiOverview.pm @@ -21,12 +21,12 @@ use PublicInbox::LeiToMail; # cf. https://en.wikipedia.org/wiki/JSON_streaming my $JSONL = 'ldjson|ndjson|jsonl'; # 3 names for the same thing -sub _iso8601 ($) { strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_[0])) } +sub iso8601 ($) { strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_[0])) } # we open this in the parent process before ->wq_io_do handoff sub ovv_out_lk_init ($) { my ($self) = @_; - my $tmp = File::Temp->new("lei-ovv.dst.$$.lock-XXXXXX", + my $tmp = File::Temp->new("lei-ovv.dst.$$.lock-XXXX", TMPDIR => 1, UNLINK => 0); $self->{"lk_id.$self.$$"} = $self->{lock_path} = $tmp->filename; } @@ -37,16 +37,16 @@ sub ovv_out_lk_cancel ($) { unlink($lock_path); } -sub detect_fmt ($$) { - my ($lei, $dst) = @_; +sub detect_fmt ($) { + my ($dst) = @_; if ($dst =~ m!\A([:/]+://)!) { - $lei->fail("$1 support not implemented, yet\n"); - } elsif (!-e $dst || -d _) { - 'maildir'; # the default TODO: MH? + die "$1 support not implemented, yet\n"; + } elsif (!-e $dst || -d _) { # maildir is the default TODO: MH + -e "$dst/inbox.lock" ? 'v2' : 'maildir'; } elsif (-f _ || -p _) { - $lei->fail("unable to determine mbox family of $dst\n"); + die "unable to determine mbox family of $dst\n"; } else { - $lei->fail("unable to determine format of $dst\n"); + die "unable to determine format of $dst\n"; } } @@ -60,22 +60,22 @@ sub new { my $fmt = $opt->{$ofmt_key}; $fmt = lc($fmt) if defined $fmt; if ($dst =~ m!\A([a-z0-9\+]+)://!is) { - defined($fmt) and return $lei->fail(<<""); + defined($fmt) and die <<""; --$ofmt_key=$fmt invalid with URL $dst $fmt = lc $1; } elsif ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/ my $ofmt = lc $1; $fmt //= $ofmt; - return $lei->fail(<<"") if $fmt ne $ofmt; + die <<"" if $fmt ne $ofmt; --$ofmt_key=$fmt and --output=$ofmt conflict } - $fmt //= 'json' if $dst eq '/dev/stdout'; - $fmt //= detect_fmt($lei, $dst) or return; + my $devfd = $lei->path_to_fd($dst) // return; + $fmt //= $devfd >= 0 ? 'json' : detect_fmt($dst); if (index($dst, '://') < 0) { # not a URL, so assume path - $dst = File::Spec->canonpath($dst); + $dst = $lei->canonpath_harder($dst); } # else URL my $self = bless { fmt => $fmt, dst => $dst }, $class; @@ -84,12 +84,12 @@ sub new { if ($fmt =~ /\A($JSONL|(?:concat)?json)\z/) { $json = $self->{json} = ref(PublicInbox::Config->json); } - if ($dst eq '/dev/stdout') { - my $isatty = $lei->{need_pager} = -t $lei->{1}; + if ($devfd >= 0) { + my $isatty = $lei->{need_pager} = -t $lei->{$devfd}; $opt->{pretty} //= $isatty; if (!$isatty && -f _) { - my $fl = fcntl($lei->{1}, F_GETFL, 0) // - return $lei->fail("fcntl(stdout): $!"); + my $fl = fcntl($lei->{$devfd}, F_GETFL, 0) // + die("fcntl(/dev/fd/$devfd): $!\n"); ovv_out_lk_init($self) unless ($fl & O_APPEND); } else { ovv_out_lk_init($self); @@ -100,15 +100,14 @@ sub new { if ($json) { $lei->{dedupe} //= PublicInbox::LeiDedupe->new($lei); } else { - # default to the cheapest sort since MUA usually resorts - $opt->{'sort'} //= 'docid' if $dst ne '/dev/stdout'; - $lei->{l2m} = eval { PublicInbox::LeiToMail->new($lei) }; - return $lei->fail($@) if $@; + $lei->{l2m} = PublicInbox::LeiToMail->new($lei); if ($opt->{mua} && $lei->{l2m}->lock_free) { $lei->{early_mua} = 1; $opt->{alert} //= [ ':WINCH,:bell' ] if -t $lei->{1}; } } + die("--shared is only for v2 inbox output\n") if + $self->{fmt} ne 'v2' && $lei->{opt}->{shared}; $self; } @@ -120,7 +119,7 @@ sub ovv_begin { } # TODO HTML/Atom/... } -# called once by parent (via PublicInbox::EOFpipe) +# called once by parent (via PublicInbox::PktOp '' => query_done) sub ovv_end { my ($self, $lei) = @_; if ($self->{fmt} eq 'json') { @@ -136,23 +135,26 @@ sub ovv_end { sub _unbless_smsg { my ($smsg, $mitem) = @_; - delete @$smsg{qw(lines bytes num tid)}; - $smsg->{rt} = _iso8601(delete $smsg->{ts}); # JMAP receivedAt - $smsg->{dt} = _iso8601(delete $smsg->{ds}); # JMAP UTCDate + # TODO: make configurable + # num/tid are nonsensical with multi-inbox search, + # lines/bytes are not generally useful + delete @$smsg{qw(num tid lines bytes)}; + $smsg->{rt} = iso8601(delete $smsg->{ts}); # JMAP receivedAt + $smsg->{dt} = iso8601(delete $smsg->{ds}); # JMAP UTCDate $smsg->{pct} = get_pct($mitem) if $mitem; if (my $r = delete $smsg->{references}) { - $smsg->{refs} = [ map { "<$_>" } ($r =~ m/$MID_EXTRACT/go) ]; + @{$smsg->{refs}} = ($r =~ m/$MID_EXTRACT/go); } if (my $m = delete($smsg->{mid})) { - $smsg->{'m'} = "<$m>"; + $smsg->{'m'} = $m; } for my $f (qw(from to cc)) { my $v = delete $smsg->{$f} or next; $smsg->{substr($f, 0, 1)} = pairs($v); } $smsg->{'s'} = delete $smsg->{subject}; - # can we be bothered to parse From/To/Cc into arrays? - scalar { %$smsg }; # unbless + my $kw = delete($smsg->{kw}); + scalar { %$smsg, ($kw && scalar(@$kw) ? (kw => $kw) : ()) }; # unbless } sub ovv_atexit_child { @@ -177,7 +179,10 @@ sub _json_pretty { $pair =~ s/(null|"),"/$1, "/g; $pair; } @$v) . ']'; - } else { # references + } elsif ($k eq 'kw') { # keywords are short, one-line + $v = $json->encode($v); + $v =~ s/","/", "/g; + } else { # refs, labels, ... $v = '[' . join($sep, map { substr($json->encode([$_]), 1, -1); } @$v) . ']'; @@ -190,7 +195,7 @@ sub _json_pretty { } sub ovv_each_smsg_cb { # runs in wq worker usually - my ($self, $lei, $ibxish) = @_; + my ($self, $lei) = @_; my ($json, $dedupe); if (my $pkg = $self->{json}) { $json = $pkg->new; @@ -203,25 +208,20 @@ sub ovv_each_smsg_cb { # runs in wq worker usually $dedupe->prepare_dedupe; } $lei->{ovv_buf} = \(my $buf = '') if !$l2m; - if ($l2m && !$ibxish) { # remote https?:// mboxrd - my $g2m = $l2m->can('git_to_mail'); - my $wcb = $l2m->write_cb($lei); - sub { - my ($smsg, undef, $eml) = @_; # no mitem in $_[1] - $wcb->(undef, $smsg, $eml); - }; - } elsif ($l2m && $l2m->{-wq_s1}) { - my $git_dir = $ibxish->git->{git_dir}; + if ($l2m) { sub { - my ($smsg, $mitem) = @_; + my ($smsg, $mitem, $eml) = @_; $smsg->{pct} = get_pct($mitem) if $mitem; - $l2m->wq_io_do('write_mail', [], $git_dir, $smsg); + eval { $l2m->wq_io_do('write_mail', [], $smsg, $eml) }; + $lei->fail($@) if $@ && !$!{ECONNRESET} && !$!{EPIPE}; } } elsif ($self->{fmt} =~ /\A(concat)?json\z/ && $lei->{opt}->{pretty}) { my $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},"; + my $lse = $lei->{lse}; sub { # DIY prettiness :P my ($smsg, $mitem) = @_; return if $dedupe->is_smsg_dup($smsg); + $lse->xsmsg_vmd($smsg, $smsg->{L} ? undef : 1); $smsg = _unbless_smsg($smsg, $mitem); $buf .= "{\n"; $buf .= join(",\n", map { @@ -241,9 +241,11 @@ sub ovv_each_smsg_cb { # runs in wq worker usually } } elsif ($json) { my $ORS = $self->{fmt} eq 'json' ? ",\n" : "\n"; # JSONL + my $lse = $lei->{lse}; sub { my ($smsg, $mitem) = @_; return if $dedupe->is_smsg_dup($smsg); + $lse->xsmsg_vmd($smsg, $smsg->{L} ? undef : 1); $buf .= $json->encode(_unbless_smsg(@_)) . $ORS; return if length($buf) < 65536; my $lk = $self->lock_for_scope; |