diff options
Diffstat (limited to 'lib/PublicInbox/LeiImport.pm')
| -rw-r--r-- | lib/PublicInbox/LeiImport.pm | 77 |
1 files changed, 50 insertions, 27 deletions
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm index 69d63ab6..5521188c 100644 --- a/lib/PublicInbox/LeiImport.pm +++ b/lib/PublicInbox/LeiImport.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # front-end for the "lei import" sub-command @@ -7,6 +7,7 @@ use strict; use v5.10.1; use parent qw(PublicInbox::IPC PublicInbox::LeiInput); use PublicInbox::InboxWritable qw(eml_from_path); +use PublicInbox::Compat qw(uniqstr); # /^input_/ subs are used by (or override) PublicInbox::LeiInput superclass @@ -36,12 +37,11 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn my $kw = PublicInbox::MdirReader::flags2kw($fl); substr($folder, 0, 0) = 'maildir:'; # add prefix my $lse = $self->{lse} //= $self->{lei}->{sto}->search; - my $lms = $self->{-lms_ro} //= $self->{lei}->lms; # may be 0 or undef + my $lms = $self->{-lms_rw} //= $self->{lei}->lms; # may be 0 or undef my @oidbin = $lms ? $lms->name_oidbin($folder, $bn) : (); - @oidbin > 1 and $self->{lei}->err("W: $folder/*/$$bn not unique:\n", + @oidbin > 1 and warn("W: $folder/*/$$bn not unique:\n", map { "\t".unpack('H*', $_)."\n" } @oidbin); - my %seen; - my @docids = sort { $a <=> $b } grep { !$seen{$_}++ } + my @docids = sort { $a <=> $b } uniqstr map { $lse->over->oidbin_exists($_) } @oidbin; my $vmd = $self->{-import_kw} ? { kw => $kw } : undef; if (scalar @docids) { @@ -53,6 +53,29 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn } } +sub input_mh_cb { + my ($mhdir, $n, $kw, $eml, $self) = @_; + substr($mhdir, 0, 0) = 'mh:'; # add prefix + my $lse = $self->{lse} //= $self->{lei}->{sto}->search; + my $lms = $self->{-lms_rw} //= $self->{lei}->lms; # may be 0 or undef + my @oidbin = $lms ? $lms->num_oidbin($mhdir, $n) : (); + @oidbin > 1 and warn("W: $mhdir/$n not unique:\n", + map { "\t".unpack('H*', $_)."\n" } @oidbin); + my @docids = sort { $a <=> $b } uniqstr + map { $lse->over->oidbin_exists($_) } @oidbin; + if (scalar @docids) { + $lse->kw_changed(undef, $kw, \@docids) or return; + } + if (defined $eml) { + my $vmd = $self->{-import_kw} ? { kw => $kw } : undef; + $vmd->{sync_info} = [ $mhdir, $n + 0 ] if $self->{-mail_sync}; + $self->input_eml_cb($eml, $vmd); + } + # TODO: + # elsif (my $ikw = $self->{lei}->{ikw}) { # old message, kw only + # $ikw->wq_io_do('ck_update_kw', [], "mh:$dir", $uid, $kw); +} + sub input_net_cb { # imap_each / nntp_each my ($uri, $uid, $kw, $eml, $self) = @_; if (defined $eml) { @@ -71,9 +94,7 @@ sub do_import_index ($$@) { my $sto = $lei->_lei_store(1); $sto->write_prepare($lei); $self->{-import_kw} = $lei->{opt}->{kw} // 1; - my $vmd_mod = $self->vmd_mod_extract(\@inputs); - return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err}; - $self->{all_vmd} = $vmd_mod if scalar keys %$vmd_mod; + $self->{all_vmd} = $lei->{vmd_mod} if keys %{$lei->{vmd_mod}}; $lei->ale; # initialize for workers to read (before LeiPmdir->new) $self->{-mail_sync} = $lei->{opt}->{'mail-sync'} // 1; $self->prepare_inputs($lei, \@inputs) or return; @@ -87,8 +108,8 @@ sub do_import_index ($$@) { # $j = $net->net_concurrency($j); TODO if ($lei->{opt}->{incremental} // 1) { $net->{incremental} = 1; - $net->{-lms_ro} = $lei->lms // 0; - if ($self->{-import_kw} && $net->{-lms_ro} && + $net->{-lms_rw} = $lei->lms // 0; + if ($self->{-import_kw} && $net->{-lms_rw} && !$lei->{opt}->{'new-only'} && $net->{imap_order}) { require PublicInbox::LeiImportKw; @@ -100,17 +121,11 @@ sub do_import_index ($$@) { my $nproc = $self->detect_nproc; $j = $nproc if $j > $nproc; } - if ($lei->{opt}->{'new-only'} && (!$net || !$net->{imap_order})) { - $lei->err('# --new-only is only for IMAP'); - } - my $ops = {}; - $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; + ($lei->{opt}->{'new-only'} && (!$net || !$net->{imap_order})) and + warn "# --new-only is only for IMAP\n"; $lei->{-eml_noisy} = 1; - (my $op_c, $ops) = $lei->workers_start($self, $j, $ops); - $lei->{wq1} = $self; $lei->{-err_type} = 'non-fatal'; - net_merge_all_done($self) unless $lei->{auth}; - $lei->wait_wq_events($op_c, $ops); + $lei->wq1_start($self, $j); } sub lei_import { # the main "lei import" method @@ -121,16 +136,24 @@ sub lei_import { # the main "lei import" method sub _complete_import { my ($lei, @argv) = @_; - my ($re, $cur, $match_cb) = $lei->complete_url_prepare(\@argv); - my @k = $lei->url_folder_cache->keys($argv[-1] // undef, 1); - my @m = map { $match_cb->($_) } @k; - my %f = map { $_ => 1 } (@m ? @m : @k); + my $has_arg = @argv; + my ($pfx, $cur, $match_cb) = $lei->complete_url_prepare(\@argv); + my @try = $has_arg ? ($pfx.$cur, $argv[-1]) : ($argv[-1]); + push(@try, undef) if defined $try[-1]; + my (@f, @k); + for (@try) { + @k = $lei->url_folder_cache->keys($_, 1) and last; + } + my @L = eval { $lei->_lei_store->search->all_terms('L') }; + push(@k, map { "+L:$_" } @L); if (my $lms = $lei->lms) { - @k = $lms->folders($argv[-1] // undef, 1); - @m = map { $match_cb->($_) } @k; - if (@m) { @f{@m} = @m } else { @f{@k} = @k } + for (@try) { + @f = $lms->folders($_, 1) and last; + } + push @k, @f; } - keys %f; + my @m = map { $match_cb->($_) } @k; + @m ? @m : @k; } no warnings 'once'; |