diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | lib/PublicInbox/LEI.pm | 17 | ||||
| -rw-r--r-- | lib/PublicInbox/LeiConvert.pm | 60 | ||||
| -rw-r--r-- | lib/PublicInbox/LeiImport.pm | 57 | ||||
| -rw-r--r-- | lib/PublicInbox/LeiInput.pm | 85 |
5 files changed, 94 insertions, 126 deletions
@@ -187,6 +187,7 @@ lib/PublicInbox/LeiDedupe.pm lib/PublicInbox/LeiExternal.pm lib/PublicInbox/LeiHelp.pm lib/PublicInbox/LeiImport.pm +lib/PublicInbox/LeiInput.pm lib/PublicInbox/LeiMirror.pm lib/PublicInbox/LeiOverview.pm lib/PublicInbox/LeiP2q.pm diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 9e3bb9b7..0bd52a46 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -419,23 +419,6 @@ sub fail ($$;$) { undef; } -sub check_input_format ($;$) { - my ($self, $files) = @_; - my $opt_key = 'in-format'; - my $fmt = $self->{opt}->{$opt_key}; - if (!$fmt) { - my $err = $files ? "regular file(s):\n@$files" : '--stdin'; - return fail($self, "--$opt_key unset for $err"); - } - require PublicInbox::MboxLock if $files; - require PublicInbox::MboxReader; - return 1 if $fmt eq 'eml'; - # XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail - PublicInbox::MboxReader->can($fmt) or - return fail($self, "--$opt_key=$fmt unrecognized"); - 1; -} - sub out ($;@) { my $self = shift; return if print { $self->{1} // return } @_; # likely diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm index 8d3b221a..0aa13229 100644 --- a/lib/PublicInbox/LeiConvert.pm +++ b/lib/PublicInbox/LeiConvert.pm @@ -5,7 +5,7 @@ package PublicInbox::LeiConvert; use strict; use v5.10.1; -use parent qw(PublicInbox::IPC); +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use PublicInbox::Eml; use PublicInbox::LeiStore; use PublicInbox::LeiOverview; @@ -79,64 +79,14 @@ sub do_convert { # via wq_do sub lei_convert { # the main "lei convert" method my ($lei, @inputs) = @_; - my $opt = $lei->{opt}; - $opt->{kw} //= 1; + $lei->{opt}->{kw} //= 1; + $lei->{opt}->{dedupe} //= 'none'; my $self = $lei->{cnv} = bless {}, __PACKAGE__; - my $in_fmt = $opt->{'in-format'}; - my (@f, @d); - $opt->{dedupe} //= 'none'; my $ovv = PublicInbox::LeiOverview->new($lei, 'out-format'); $lei->{l2m} or return $lei->fail("output not specified or is not a mail destination"); - my $net = $lei->{net}; # NetWriter may be created by l2m - $opt->{augment} = 1 unless $ovv->{dst} eq '/dev/stdout'; - if ($opt->{stdin}) { - @inputs and return $lei->fail("--stdin and @inputs do not mix"); - $lei->check_input_format(undef) or return; - $self->{0} = $lei->{0}; - } - # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX - for my $input (@inputs) { - my $input_path = $input; - if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { - require PublicInbox::NetReader; - $net //= PublicInbox::NetReader->new; - $net->add_url($input); - } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { - my $ifmt = lc $1; - if (($in_fmt // $ifmt) ne $ifmt) { - return $lei->fail(<<""); ---in-format=$in_fmt and `$ifmt:' conflict - - } - if (-f $input_path) { - require PublicInbox::MboxLock; - require PublicInbox::MboxReader; - PublicInbox::MboxReader->can($ifmt) or return - $lei->fail("$ifmt not supported"); - } elsif (-d _) { - require PublicInbox::MdirReader; - $ifmt eq 'maildir' or return - $lei->fail("$ifmt not supported"); - } else { - return $lei->fail("Unable to handle $input"); - } - } elsif (-f $input) { push @f, $input } - elsif (-d _) { push @d, $input } - else { return $lei->fail("Unable to handle $input") } - } - if (@f) { $lei->check_input_format(\@f) or return } - if (@d) { # TODO: check for MH vs Maildir, here - require PublicInbox::MdirReader; - } - $self->{inputs} = \@inputs; - if ($net) { - if (my $err = $net->errors) { - return $lei->fail($err); - } - $net->{quiet} = $opt->{quiet}; - $lei->{net} //= $net; - } + $lei->{opt}->{augment} = 1 unless $ovv->{dst} eq '/dev/stdout'; + $self->prepare_inputs($lei, \@inputs) or return; my $op = $lei->workers_start($self, 'lei_convert', 1, { '' => [ $lei->can('dclose'), $lei ] }); diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index 0e2a96e8..e769fba8 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -5,7 +5,7 @@ package PublicInbox::LeiImport; use strict; use v5.10.1; -use parent qw(PublicInbox::IPC); +use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use PublicInbox::Eml; use PublicInbox::PktOp qw(pkt_do); @@ -67,60 +67,9 @@ sub lei_import { # the main "lei import" method my ($lei, @inputs) = @_; my $sto = $lei->_lei_store(1); $sto->write_prepare($lei); - my ($net, @f, @d); $lei->{opt}->{kw} //= 1; - my $self = $lei->{imp} = bless { inputs => \@inputs }, __PACKAGE__; - if ($lei->{opt}->{stdin}) { - @inputs and return $lei->fail("--stdin and @inputs do not mix"); - $lei->check_input_format or return; - $self->{0} = $lei->{0}; - } - - my $fmt = $lei->{opt}->{'in-format'}; - # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX - for my $input (@inputs) { - my $input_path = $input; - if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { - require PublicInbox::NetReader; - $net //= PublicInbox::NetReader->new; - $net->add_url($input); - } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { - my $ifmt = lc $1; - if (($fmt // $ifmt) ne $ifmt) { - return $lei->fail(<<""); ---in-format=$fmt and `$ifmt:' conflict - - } - if (-f $input_path) { - require PublicInbox::MboxLock; - require PublicInbox::MboxReader; - PublicInbox::MboxReader->can($ifmt) or return - $lei->fail("$ifmt not supported"); - } elsif (-d _) { - require PublicInbox::MdirReader; - $ifmt eq 'maildir' or return - $lei->fail("$ifmt not supported"); - } else { - return $lei->fail("Unable to handle $input"); - } - } elsif (-f $input) { push @f, $input - } elsif (-d _) { push @d, $input - } else { return $lei->fail("Unable to handle $input") } - } - if (@f) { $lei->check_input_format(\@f) or return } - if (@d) { # TODO: check for MH vs Maildir, here - require PublicInbox::MdirReader; - } - $self->{inputs} = \@inputs; - if ($net) { - if (my $err = $net->errors) { - return $lei->fail($err); - } - $net->{quiet} = $lei->{opt}->{quiet}; - $lei->{net} = $net; - require PublicInbox::LeiAuth; - $lei->{auth} = PublicInbox::LeiAuth->new; - } + my $self = $lei->{imp} = bless {}, __PACKAGE__; + $self->prepare_inputs($lei, \@inputs) or return; import_start($lei); } diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm new file mode 100644 index 00000000..89585a52 --- /dev/null +++ b/lib/PublicInbox/LeiInput.pm @@ -0,0 +1,85 @@ +# Copyright (C) 2021 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> + +# parent class for LeiImport, LeiConvert +package PublicInbox::LeiInput; +use strict; +use v5.10.1; + +sub check_input_format ($;$) { + my ($lei, $files) = @_; + my $opt_key = 'in-format'; + my $fmt = $lei->{opt}->{$opt_key}; + if (!$fmt) { + my $err = $files ? "regular file(s):\n@$files" : '--stdin'; + return $lei->fail("--$opt_key unset for $err"); + } + require PublicInbox::MboxLock if $files; + require PublicInbox::MboxReader; + return 1 if $fmt eq 'eml'; + # XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail + PublicInbox::MboxReader->can($fmt) or + return $lei->fail("--$opt_key=$fmt unrecognized"); + 1; +} + + +sub prepare_inputs { + my ($self, $lei, $inputs) = @_; + my $in_fmt = $lei->{opt}->{'in-format'}; + if ($lei->{opt}->{stdin}) { + @$inputs and return + $lei->fail("--stdin and @$inputs do not mix"); + check_input_format($lei) or return; + $self->{0} = $lei->{0}; + } + my $net = $lei->{net}; # NetWriter may be created by l2m + my $fmt = $lei->{opt}->{'in-format'}; + my (@f, @d); + # e.g. Maildir:/home/user/Mail/ or imaps://example.com/INBOX + for my $input (@$inputs) { + my $input_path = $input; + if ($input =~ m!\A(?:imaps?|nntps?|s?news)://!i) { + require PublicInbox::NetReader; + $net //= PublicInbox::NetReader->new; + $net->add_url($input); + } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { + my $ifmt = lc $1; + if (($in_fmt // $ifmt) ne $ifmt) { + return $lei->fail(<<""); +--in-format=$in_fmt and `$ifmt:' conflict + + } + if (-f $input_path) { + require PublicInbox::MboxLock; + require PublicInbox::MboxReader; + PublicInbox::MboxReader->can($ifmt) or return + $lei->fail("$ifmt not supported"); + } elsif (-d _) { + require PublicInbox::MdirReader; + $ifmt eq 'maildir' or return + $lei->fail("$ifmt not supported"); + } else { + return $lei->fail("Unable to handle $input"); + } + } elsif (-f $input) { push @f, $input } + elsif (-d _) { push @d, $input } + else { return $lei->fail("Unable to handle $input") } + } + if (@f) { check_input_format($lei, \@f) or return } + if (@d) { # TODO: check for MH vs Maildir, here + require PublicInbox::MdirReader; + } + if ($net) { + if (my $err = $net->errors) { + return $lei->fail($err); + } + $net->{quiet} = $lei->{opt}->{quiet}; + require PublicInbox::LeiAuth; + $lei->{auth} //= PublicInbox::LeiAuth->new; + $lei->{net} //= $net; + } + $self->{inputs} = $inputs; +} + +1; |