From 63283ae1b51203c930332e6887296cb123e5db6c Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 18 Feb 2021 23:22:22 +0300 Subject: lei convert: mail format conversion sub-command This will make testing IMAP support for other commands easier, as it doesn't write to lei/store at all. Like the pager and MUA, "git credential" is always spawned by script/lei (and not lei-daemon) so it has a controlling terminal for password prompts. v2: fix missing requires, correct test ordering v3: ensure config exists for IMAP auth --- lib/PublicInbox/LeiConvert.pm | 160 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 lib/PublicInbox/LeiConvert.pm (limited to 'lib/PublicInbox/LeiConvert.pm') diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm new file mode 100644 index 00000000..78fd5e17 --- /dev/null +++ b/lib/PublicInbox/LeiConvert.pm @@ -0,0 +1,160 @@ +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ + +# front-end for the "lei convert" sub-command +package PublicInbox::LeiConvert; +use strict; +use v5.10.1; +use parent qw(PublicInbox::IPC); +use PublicInbox::Eml; +use PublicInbox::InboxWritable qw(eml_from_path); +use PublicInbox::PktOp; +use PublicInbox::LeiStore; +use PublicInbox::LeiOverview; + +sub mbox_cb { + my ($eml, $self) = @_; + my @kw = PublicInbox::LeiStore::mbox_keywords($eml); + $eml->header_set($_) for qw(Status X-Status); + $self->{wcb}->(undef, { kw => \@kw }, $eml); +} + +sub imap_cb { # ->imap_each + my ($url, $uid, $kw, $eml, $self) = @_; + $self->{wcb}->(undef, { kw => $kw }, $eml); +} + +sub mdir_cb { + my ($kw, $eml, $self) = @_; + $self->{wcb}->(undef, { kw => $kw }, $eml); +} + +sub do_convert { # via wq_do + my ($self) = @_; + my $lei = $self->{lei}; + my $in_fmt = $lei->{opt}->{'in-format'}; + if (my $stdin = delete $self->{0}) { + PublicInbox::MboxReader->$in_fmt($stdin, \&mbox_cb, $self); + } + for my $input (@{$self->{inputs}}) { + my $ifmt = lc($in_fmt // ''); + if ($input =~ m!\A(?:imap|nntp)s?://!) { # TODO: nntp + $lei->{nrd}->imap_each($input, \&imap_cb, $self); + next; + } elsif ($input =~ s!\A([a-z0-9]+):!!i) { + $ifmt = lc $1; + } + if (-f $input) { + open my $fh, '<', $input or + return $lei->fail("open $input: $!"); + PublicInbox::MboxReader->$ifmt($fh, \&mbox_cb, $self); + } elsif (-d _) { + PublicInbox::MdirReader::maildir_each_eml($input, + \&mdir_cb, $self); + } else { + die "BUG: $input unhandled"; # should've failed earlier + } + } + delete $lei->{1}; + delete $self->{wcb}; # commit +} + +sub convert_start { + my ($lei) = @_; + my $ops = { + '!' => [ $lei->can('fail_handler'), $lei ], + '|' => [ $lei->can('sigpipe_handler'), $lei ], + 'x_it' => [ $lei->can('x_it'), $lei ], + 'child_error' => [ $lei->can('child_error'), $lei ], + '' => [ $lei->can('dclose'), $lei ], + }; + ($lei->{pkt_op_c}, $lei->{pkt_op_p}) = PublicInbox::PktOp->pair($ops); + my $self = $lei->{cnv}; + $self->wq_workers_start('lei_convert', 1, $lei->oldset, {lei => $lei}); + my $op = delete $lei->{pkt_op_c}; + delete $lei->{pkt_op_p}; + $self->wq_io_do('do_convert', []); + $self->wq_close(1); + $lei->event_step_init; # wait for shutdowns + if ($lei->{oneshot}) { + while ($op->{sock}) { $op->event_step } + } +} + +sub call { # the main "lei convert" method + my ($cls, $lei, @inputs) = @_; + my $opt = $lei->{opt}; + $opt->{kw} //= 1; + my $self = $lei->{cnv} = bless {}, $cls; + my $in_fmt = $opt->{'in-format'}; + my ($nrd, @f, @d); + $opt->{dedupe} //= 'none'; + my $ovv = PublicInbox::LeiOverview->new($lei, 'out-format'); + $lei->{l2m} or return + $lei->fail("output not specified or is not a mail destination"); + $opt->{augment} = 1 unless $ovv->{dst} eq '/dev/stdout'; + if ($opt->{stdin}) { + @inputs and return $lei->fail("--stdin and @inputs do not mix"); + $lei->check_input_format(undef, 'in-format') or return; + $self->{0} = $lei->{0}; + } + # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX + for my $input (@inputs) { + my $input_path = $input; + if ($input =~ m!\A(?:imap|nntp)s?://!i) { + require PublicInbox::NetReader; + $nrd //= PublicInbox::NetReader->new; + $nrd->add_url($input); + } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { + my $ifmt = lc $1; + if (($in_fmt // $ifmt) ne $ifmt) { + return $lei->fail(<<""); +--in-format=$in_fmt and `$ifmt:' conflict + + } + if (-f $input_path) { + require PublicInbox::MboxReader; + PublicInbox::MboxReader->can($ifmt) or return + $lei->fail("$ifmt not supported"); + } elsif (-d _) { + require PublicInbox::MdirReader; + $ifmt eq 'maildir' or return + $lei->fail("$ifmt not supported"); + } else { + return $lei->fail("Unable to handle $input"); + } + } elsif (-f $input) { push @f, $input } + elsif (-d _) { push @d, $input } + else { return $lei->fail("Unable to handle $input") } + } + if (@f) { $lei->check_input_format(\@f, 'in-format') or return } + if (@d) { # TODO: check for MH vs Maildir, here + require PublicInbox::MdirReader; + } + $self->{inputs} = \@inputs; + return convert_start($lei) if !$nrd; + + if (my $err = $nrd->errors) { + return $lei->fail($err); + } + $nrd->{quiet} = $opt->{quiet}; + $lei->{nrd} = $nrd; + require PublicInbox::LeiAuth; + my $auth = $lei->{auth} = PublicInbox::LeiAuth->new($nrd); + $auth->auth_start($lei, \&convert_start, $lei); +} + +sub ipc_atfork_child { + my ($self) = @_; + my $lei = $self->{lei}; + $lei->lei_atfork_child; + my $l2m = delete $lei->{l2m}; + $l2m->pre_augment($lei); + $l2m->do_augment($lei); + $l2m->post_augment($lei); + $self->{wcb} = $l2m->write_cb($lei); + $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb(); + $self->SUPER::ipc_atfork_child; +} + +1; -- cgit v1.2.3-24-ge0c7