From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 34FDB1FAEE for ; Mon, 19 Mar 2018 08:15:02 +0000 (UTC) From: "Eric Wong (Contractor, The Linux Foundation)" To: meta@public-inbox.org Subject: [PATCH 11/27] import: implement barrier operation for v1 repos Date: Mon, 19 Mar 2018 08:14:43 +0000 Message-Id: <20180319081459.10645-12-e@80x24.org> In-Reply-To: <20180319081459.10645-1-e@80x24.org> References: <20180319081459.10645-1-e@80x24.org> List-Id: This will allow WatchMaildir to use ->barrier operations instead of reaching inside for nchg. This also ensures dumb HTTP clients can see changes to V2 repos immediately. --- lib/PublicInbox/Import.pm | 58 ++++++++++++++++++++++++++--------------- lib/PublicInbox/V2Writable.pm | 7 +---- lib/PublicInbox/WatchMaildir.pm | 2 +- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm index 6a640e2..12df7d5 100644 --- a/lib/PublicInbox/Import.pm +++ b/lib/PublicInbox/Import.pm @@ -148,6 +148,42 @@ sub progress { undef; } +sub _update_git_info ($$) { + my ($self, $do_gc) = @_; + # for compatibility with existing ssoma installations + # we can probably remove this entirely by 2020 + my $git_dir = $self->{git}->{git_dir}; + my @cmd = ('git', "--git-dir=$git_dir"); + my $index = "$git_dir/ssoma.index"; + if (-e $index && !$ENV{FAST}) { + my $env = { GIT_INDEX_FILE => $index }; + run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env); + } + run_die([@cmd, 'update-server-info'], undef); + ($self->{path_type} eq '2/38') and eval { + require PublicInbox::SearchIdx; + my $inbox = $self->{inbox} || $git_dir; + my $s = PublicInbox::SearchIdx->new($inbox); + $s->index_sync({ ref => $self->{ref} }); + }; + eval { run_die([@cmd, qw(gc --auto)], undef) } if $do_gc; +} + +sub barrier { + my ($self) = @_; + + # For safety, we ensure git checkpoint is complete before because + # the data in git is still more important than what is in Xapian + # in v2. Performance may be gained by delaying the ->progress + # call but we lose safety + if ($self->{nchg}) { + $self->checkpoint; + $self->progress('checkpoint'); + _update_git_info($self, 0); + $self->{nchg} = 0; + } +} + # used for v2 sub get_mark { my ($self, $mark) = @_; @@ -341,28 +377,8 @@ sub done { my $pid = delete $self->{pid} or die 'BUG: missing {pid} when done'; waitpid($pid, 0) == $pid or die 'fast-import did not finish'; $? == 0 or die "fast-import failed: $?"; - my $nchg = delete $self->{nchg}; - # for compatibility with existing ssoma installations - # we can probably remove this entirely by 2020 - my $git_dir = $self->{git}->{git_dir}; - my @cmd = ('git', "--git-dir=$git_dir"); - my $index = "$git_dir/ssoma.index"; - if ($nchg && -e $index && !$ENV{FAST}) { - my $env = { GIT_INDEX_FILE => $index }; - run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env); - } - if ($nchg) { - run_die([@cmd, 'update-server-info'], undef); - ($self->{path_type} eq '2/38') and eval { - require PublicInbox::SearchIdx; - my $inbox = $self->{inbox} || $git_dir; - my $s = PublicInbox::SearchIdx->new($inbox); - $s->index_sync({ ref => $self->{ref} }); - }; - - eval { run_die([@cmd, qw(gc --auto)], undef) }; - } + _update_git_info($self, 1) if delete $self->{nchg}; $self->{ssoma_lock} or return; my $lockfh = delete $self->{lockfh} or die "BUG: not locked: $!"; diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index fd9bf61..fbc71c8 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -263,13 +263,8 @@ sub checkpoint { sub barrier { my ($self) = @_; - # For safety, we ensure git checkpoint is complete before because - # the data in git is still more important than what is in Xapian. - # Performance may be gained by delaying ->progress call but we - # lose safety if (my $im = $self->{im}) { - $im->checkpoint; - $im->progress('checkpoint'); + $im->barrier; } my $skel = $self->{skel}; my $parts = $self->{idx_parts}; diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm index 3da6b27..c72d939 100644 --- a/lib/PublicInbox/WatchMaildir.pm +++ b/lib/PublicInbox/WatchMaildir.pm @@ -91,7 +91,7 @@ sub _done_for_now { my ($self) = @_; my $importers = $self->{importers}; foreach my $im (values %$importers) { - $im->done if $im->{nchg}; + $im->barrier; } my $opendirs = $self->{opendirs}; -- EW