* [PATCH 6/6] lei up: fix --mua with single output
@ 2021-09-13 20:53 5% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2021-09-13 20:53 UTC (permalink / raw)
To: meta
Oops :x
Fixes: b584a53f053a7629 ("lei up: support --all for IMAP folders")
---
lib/PublicInbox/LeiUp.pm | 25 +++++++++++++++----------
t/lei-up.t | 4 ++++
2 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index a16117c9..53f06dbc 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -54,16 +54,21 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
- my $l = bless { %$lei }, ref($lei);
- $l->{opt} = { %{$l->{opt}} }; # deep copy
- delete $l->{sock}; # do not close
- $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
-
- # make close($l->{1}) happy in lei->dclose
- open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ my $l;
+ if (defined($lei->{opt}->{mua})) { # single output
+ $l = $lei;
+ } else { # multiple outputs
+ $l = bless { %$lei }, ref($lei);
+ $l->{opt} = { %{$l->{opt}} }; # deep copy
+ delete $l->{sock}; # do not close
+ # make close($l->{1}) happy in lei->dclose
+ open my $fh, '>&', $l->{1} or
+ return $l->child_error(0, "dup: $!");
+ $l->{1} = $fh;
+ }
local $PublicInbox::LEI::current_lei = $l;
local %ENV = %{$l->{env}};
- $l->{1} = $fh;
+ $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
eval {
$l->qerr("# updating $out");
up1($l, $out);
@@ -92,7 +97,7 @@ sub lei_up {
$lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
- length($opt->{mua}//'') and return
+ defined($opt->{mua}) and return
$lei->fail('--all and --mua= are incompatible');
@outs = PublicInbox::LeiSavedSearch::list($lei);
if ($all eq 'local') {
@@ -110,7 +115,7 @@ sub lei_up {
$self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
- length($opt->{mua} // '')) and return $lei->fail(<<EOM);
+ defined($opt->{mua})) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
EOM
if ($self->{remote}) { # setup lei->{auth}
diff --git a/t/lei-up.t b/t/lei-up.t
index 6b34774d..8937cfb1 100644
--- a/t/lei-up.t
+++ b/t/lei-up.t
@@ -34,6 +34,10 @@ test_lei(sub {
open $fh, "$ENV{HOME}/b" or xbail "open: $!";
$uc = do { local $/; <$fh> };
is($uc, $exp, 'uncompressed both match');
+
+ lei_ok [ 'up', "$ENV{HOME}/b", "--mua=touch $ENV{HOME}/c" ],
+ undef, { run_mode => 0 };
+ ok(-f "$ENV{HOME}/c", '--mua works with single output');
});
done_testing;
^ permalink raw reply related [relevance 5%]
* [PATCH 0/4] lei up --all support for IMAP
@ 2021-09-07 11:32 7% Eric Wong
2021-09-07 11:32 4% ` [PATCH 3/4] lei up: support --all for IMAP folders Eric Wong
0 siblings, 1 reply; 3+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
It's not efficient since it wastes an IMAP connection for
auth-only (unlike every other lei command which reuses the
connection once authed). However, there may be ways we can
work around it in the future.
Eric Wong (4):
xt/net_writer_imap: test "lei up" on single IMAP output
lei: dump and clear log at exit
lei up: support --all for IMAP folders
doc: lei-*.pod: update to Tor v3 .onion address
Documentation/lei-convert.pod | 2 +-
Documentation/lei-edit-search.pod | 2 +-
Documentation/lei-forget-search.pod | 2 +-
Documentation/lei-lcat.pod | 2 +-
Documentation/lei-ls-mail-sync.pod | 2 +-
Documentation/lei-ls-search.pod | 2 +-
Documentation/lei-rediff.pod | 2 +-
Documentation/lei-up.pod | 17 +++--
lib/PublicInbox/LEI.pm | 20 +++---
lib/PublicInbox/LeiAuth.pm | 15 ++--
lib/PublicInbox/LeiToMail.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 104 +++++++++++++++++-----------
xt/net_writer-imap.t | 4 ++
13 files changed, 114 insertions(+), 63 deletions(-)
^ permalink raw reply [relevance 7%]
* [PATCH 3/4] lei up: support --all for IMAP folders
2021-09-07 11:32 7% [PATCH 0/4] lei up --all support for IMAP Eric Wong
@ 2021-09-07 11:32 4% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2021-09-07 11:32 UTC (permalink / raw)
To: meta
Since "lei up" is expected to be a heavily-used command,
better support for IMAP seems like a reasonable idea.
This is inefficient since we waste an IMAP(S) TCP connection
since it dies when an auth-only LeiUp worker process dies, but
it's better than not working at all, right now.
---
Documentation/lei-up.pod | 15 ++++-
lib/PublicInbox/LEI.pm | 19 ++++---
lib/PublicInbox/LeiAuth.pm | 15 +++--
lib/PublicInbox/LeiToMail.pm | 3 +-
lib/PublicInbox/LeiUp.pm | 104 ++++++++++++++++++++++-------------
xt/net_writer-imap.t | 1 +
6 files changed, 102 insertions(+), 55 deletions(-)
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index cea0f619..ca4cf4fe 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -6,15 +6,24 @@ lei-up - update a saved search
lei up [OPTIONS] OUTPUT
-lei up [OPTIONS] --all=TYPE
+lei up [OPTIONS] --all[=<local|remote>]
=head1 DESCRIPTION
-Update the saved search at C<OUTPUT> or all saved searches of C<TYPE>
-(currently C<local> is the only supported value).
+Update the saved search at C<OUTPUT> or all saved searches.
=head1 OPTIONS
+=over
+
+=item --all[=<local|remote>]
+
+C<--all> updates all saved searches (listed in L<lei-ls-search(1)>).
+C<--all=local> only updates local mailboxes, C<--all=remote> only
+updates remote mailboxes (currently C<imap://> and C<imaps://>).
+
+=back
+
The following options, described in L<lei-q(1)>, are supported.
=over
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bd44cfae..a258722e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -605,16 +605,19 @@ sub incr {
$self->{counters}->{$field} += $nr;
}
+sub pkt_ops {
+ my ($lei, $ops) = @_;
+ $ops->{'!'} = [ \&fail_handler, $lei ];
+ $ops->{'|'} = [ \&sigpipe_handler, $lei ];
+ $ops->{x_it} = [ \&x_it, $lei ];
+ $ops->{child_error} = [ \&child_error, $lei ];
+ $ops->{incr} = [ \&incr, $lei ];
+ $ops;
+}
+
sub workers_start {
my ($lei, $wq, $jobs, $ops, $flds) = @_;
- $ops = {
- '!' => [ \&fail_handler, $lei ],
- '|' => [ \&sigpipe_handler, $lei ],
- 'x_it' => [ \&x_it, $lei ],
- 'child_error' => [ \&child_error, $lei ],
- 'incr' => [ \&incr, $lei ],
- ($ops ? %$ops : ()),
- };
+ $ops = pkt_ops($lei, { ($ops ? %$ops : ()) });
$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm
index 465a2758..9b09cecf 100644
--- a/lib/PublicInbox/LeiAuth.pm
+++ b/lib/PublicInbox/LeiAuth.pm
@@ -30,10 +30,16 @@ sub do_auth_atfork { # used by IPC WQ workers
return if $wq->{-wq_worker_nr} != 0; # only first worker calls this
my $lei = $wq->{lei};
my $net = $lei->{net};
+ if ($net->{-auth_done}) { # from previous worker... (ugly)
+ $lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or
+ $lei->fail("pkt_do net_merge_continue: $!");
+ return;
+ }
eval { # fill auth info (may prompt user or read netrc)
my $mics = $net->imap_common_init($lei);
my $nn = $net->nntp_common_init($lei);
# broadcast successful auth info to lei-daemon:
+ $net->{-auth_done} = 1;
$lei->{pkt_op_p}->pkt_do('net_merge_continue', $net) or
die "pkt_do net_merge_continue: $!";
$net->{mics_cached} = $mics if $mics;
@@ -51,14 +57,15 @@ sub net_merge_all { # called in wq worker via wq_broadcast
# called by top-level lei-daemon when first worker is done with auth
# passes updated net auth info to current workers
sub net_merge_continue {
- my ($wq, $net_new) = @_;
+ my ($wq, $lei, $net_new) = @_;
+ $wq->{-net_new} = $net_new; # for "lei up"
$wq->wq_broadcast('PublicInbox::LeiAuth::net_merge_all', $net_new);
- $wq->net_merge_all_done; # defined per-WQ
+ $wq->net_merge_all_done($lei); # defined per-WQ
}
sub op_merge { # prepares PktOp->pair ops
- my ($self, $ops, $wq) = @_;
- $ops->{net_merge_continue} = [ \&net_merge_continue, $wq ];
+ my ($self, $ops, $wq, $lei) = @_;
+ $ops->{net_merge_continue} = [ \&net_merge_continue, $wq, $lei ];
}
sub new { bless \(my $x), __PACKAGE__ }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 2c7a92de..01f08384 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -396,7 +396,8 @@ sub new {
} elsif ($fmt =~ /\Aimaps?\z/) {
require PublicInbox::NetWriter;
require PublicInbox::URIimap;
- my $net = PublicInbox::NetWriter->new;
+ # {net} may exist from "lei up" for auth
+ my $net = $lei->{net} // PublicInbox::NetWriter->new;
$net->{quiet} = $lei->{opt}->{quiet};
my $uri = PublicInbox::URIimap->new($dst)->canonical;
$net->add_url($$uri);
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index a39d6047..30358e9d 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -5,8 +5,14 @@
package PublicInbox::LeiUp;
use strict;
use v5.10.1;
+# n.b. we use LeiInput to setup IMAP auth
+use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::LeiSavedSearch;
-use parent qw(PublicInbox::IPC);
+use PublicInbox::DS;
+use PublicInbox::PktOp;
+use PublicInbox::LeiFinmsg;
+use PublicInbox::LEI;
+my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
sub up1 ($$) {
my ($lei, $out) = @_;
@@ -48,75 +54,95 @@ sub up1 ($$) {
sub up1_redispatch {
my ($lei, $out, $op_p) = @_;
- require PublicInbox::LeiFinmsg;
- $lei->{fmsg} //= PublicInbox::LeiFinmsg->new($lei->{2});
my $l = bless { %$lei }, ref($lei);
$l->{opt} = { %{$l->{opt}} };
- delete $l->{sock};
- $l->{''} = $op_p; # daemon only
+ delete $l->{sock}; # do not close
+ $l->{''} = $op_p; # daemon only ($l => $lei => script/lei)
# make close($l->{1}) happy in lei->dclose
open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ local $PublicInbox::LEI::current_lei = $l;
$l->{1} = $fh;
eval {
$l->qerr("# updating $out");
up1($l, $out);
- $l->qerr("# $out done");
};
- $l->child_error(0, $@) if $@;
+ $lei->child_error(0, $@) if $@ || $l->{failed}; # lei->fail()
+}
+
+sub redispatch_all ($$) {
+ my ($self, $lei) = @_;
+ # re-dispatch into our event loop w/o creating an extra fork-level
+ $lei->{fmsg} = PublicInbox::LeiFinmsg->new($lei->{2});
+ my ($op_c, $op_p) = PublicInbox::PktOp->pair;
+ for my $o (@{$self->{local} // []}, @{$self->{remote} // []}) {
+ PublicInbox::DS::requeue(sub {
+ up1_redispatch($lei, $o, $op_p);
+ });
+ }
+ $lei->event_step_init;
+ $lei->pkt_ops($op_c->{ops} = { '' => [$lei->can('dclose'), $lei] });
}
sub lei_up {
my ($lei, @outs) = @_;
- $lei->{lse} = $lei->_lei_store(1)->search;
my $opt = $lei->{opt};
- my @local;
- if (defined $opt->{all}) {
+ my $self = bless { -mail_sync => 1 }, __PACKAGE__;
+ $lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
+ if (defined(my $all = $opt->{all})) {
return $lei->fail("--all and @outs incompatible") if @outs;
length($opt->{mua}//'') and return
$lei->fail('--all and --mua= are incompatible');
-
- # supporting IMAP outputs is more involved due to
- # git-credential prompts. TODO: add this in 1.8
- $opt->{all} eq 'local' or return
- $lei->fail('only --all=local works at the moment');
- my @all = PublicInbox::LeiSavedSearch::list($lei);
- @local = grep(!m!\Aimaps?://!i, @all);
+ @outs = PublicInbox::LeiSavedSearch::list($lei);
+ if ($all eq 'local') {
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq 'remote') {
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ } elsif ($all eq '') {
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+ } else {
+ $lei->fail("only --all=$all not understood");
+ }
} else {
- @local = @outs;
+ $self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+ $self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
}
- if (scalar(@outs) > 1) {
- length($opt->{mua}//'') and return $lei->fail(<<EOM);
+ ((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
+ length($opt->{mua} // '')) and return $lei->fail(<<EOM);
multiple outputs and --mua= are incompatible
EOM
- # TODO:
- return $lei->fail(<<EOM) if grep(m!\Aimaps?://!i, @outs);
-multiple destinations only supported for local outputs (FIXME)
-EOM
+ if ($self->{remote}) { # setup lei->{auth}
+ $self->prepare_inputs($lei, $self->{remote}) or return;
}
- if (scalar(@local) > 1) {
- $lei->_lei_store->write_prepare($lei); # share early
- # daemon mode, re-dispatch into our event loop w/o
- # creating an extra fork-level
- require PublicInbox::DS;
- require PublicInbox::PktOp;
- my ($op_c, $op_p) = PublicInbox::PktOp->pair;
- for my $o (@local) {
- PublicInbox::DS::requeue(sub {
- up1_redispatch($lei, $o, $op_p);
- });
- }
- $lei->event_step_init;
- $op_c->{ops} = { '' => [$lei->can('dclose'), $lei] };
+ if ($lei->{auth}) { # start auth worker
+ require PublicInbox::NetWriter;
+ bless $lei->{net}, 'PublicInbox::NetWriter';
+ $lei->{auth}->op_merge(my $ops = {}, $self, $lei);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
+ $lei->{wq1} = $self;
+ $lei->wait_wq_events($op_c, $ops);
+ # net_merge_all_done will fire when auth is done
} else {
- up1($lei, $local[0]);
+ redispatch_all($self, $lei); # see below
}
}
+# called in top-level lei-daemon when LeiAuth is done
+sub net_merge_all_done {
+ my ($self, $lei) = @_;
+ $lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
+ $self->wq_close(1);
+ redispatch_all($self, $lei);
+}
+
sub _complete_up {
my ($lei, @argv) = @_;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } PublicInbox::LeiSavedSearch::list($lei);
}
+no warnings 'once';
+*ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
+
1;
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index aeed3144..0a4cea68 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -242,6 +242,7 @@ EOM
lei_ok qw(q m:testmessage --no-external -o), $folder_url;
lei_ok qw(up), $folder_url;
+ lei_ok qw(up --all=remote);
});
undef $cleanup; # remove temporary folder
^ permalink raw reply related [relevance 4%]
Results 1-3 of 3 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-09-07 11:32 7% [PATCH 0/4] lei up --all support for IMAP Eric Wong
2021-09-07 11:32 4% ` [PATCH 3/4] lei up: support --all for IMAP folders Eric Wong
2021-09-13 20:53 [PATCH 0/6] a batch of misc fixes Eric Wong
2021-09-13 20:53 5% ` [PATCH 6/6] lei up: fix --mua with single output Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).