diff options
author | Eric Wong <e@80x24.org> | 2021-10-06 09:44:50 +0000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2021-10-06 11:23:49 +0000 |
commit | 89193578d21f847478f844e9e85495b9cae8842b (patch) | |
tree | 72bb07d5e8b4f48d63bc9f9272d948d848a5b57f /lib/PublicInbox/SearchIdx.pm | |
parent | 5cec164a0c5688a0410964cdc07c7e843e175545 (diff) | |
download | public-inbox-89193578d21f847478f844e9e85495b9cae8842b.tar.gz |
We need to ensure -extindex --gc runs don't prevent other work from happening in the meantime. I actually caused my -extindex to OOM due to the lack of checkpoints :x We'll also hoist out the shard scanning into its own sub in preparation for lei/store usage.
Diffstat (limited to 'lib/PublicInbox/SearchIdx.pm')
-rw-r--r-- | lib/PublicInbox/SearchIdx.pm | 32 |
1 files changed, 9 insertions, 23 deletions
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index e5c872d5..78db329d 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -637,14 +637,21 @@ sub update_vmd { sub xdb_remove { my ($self, @docids) = @_; - $self->begin_txn_lazy; - my $xdb = $self->{xdb} or return; + begin_txn_lazy($self); + my $xdb = $self->{xdb} // die 'BUG: missing {xdb}'; for my $docid (@docids) { eval { $xdb->delete_document($docid) }; warn "E: #$docid not in in Xapian? $@\n" if $@; } } +sub xdb_remove_quiet { + my ($self, $docid) = @_; + begin_txn_lazy($self); + my $xdb = $self->{xdb} // die 'BUG: missing {xdb}'; + eval { $xdb->delete_document($docid) }; +} + sub index_git_blob_id { my ($doc, $pfx, $objid) = @_; @@ -1098,25 +1105,4 @@ sub eidx_shard_new { $self; } -# ensure there's no stale Xapian docs by treating $over as canonical -sub over_check { - my ($self, $over) = @_; - begin_txn_lazy($self); - my $sth = $over->dbh->prepare(<<''); -SELECT COUNT(*) FROM over WHERE num = ? - - my $xdb = $self->{xdb}; - my $cur = $xdb->postlist_begin(''); - my $end = $xdb->postlist_end(''); - my $xdir = $self->xdir; - for (; $cur != $end; $cur++) { - my $docid = $cur->get_docid; - $sth->execute($docid); - my $x = $sth->fetchrow_array; - next if $x > 0; - warn "I: removing $xdir #$docid, not in `over'\n"; - $xdb->delete_document($docid); - } -} - 1; |