From 44afe76a0a351e350e18a94876fc30edc4adc404 Mon Sep 17 00:00:00 2001 From: "Eric Wong (Contractor, The Linux Foundation)" Date: Tue, 27 Feb 2018 20:25:23 +0000 Subject: searchidx: do not modify Xapian DB while iterating Iterating through a list of documents while modifying them does not seem to be supported in Xapian and it can trigger DatabaseCorruptError exceptions. This only worked with past datasets out of dumb luck. With the work-in-progress "v2" public-inbox layout, this problem might become more visible as the "thread skeleton" is partitioned out to a separate, smaller Xapian database. I've reproduced the problem on both Debian 8.x and 9.x with Xapian 1.2.19 (chert backend) and 1.4.3 (glass backend) respectively. --- lib/PublicInbox/SearchIdx.pm | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 32594137..f4238fe8 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -740,15 +740,22 @@ sub create_ghost { sub merge_threads { my ($self, $winner_tid, $loser_tid) = @_; return if $winner_tid == $loser_tid; - my ($head, $tail) = $self->find_doc_ids('G' . $loser_tid); my $db = $self->{xdb}; - for (; $head != $tail; $head->inc) { - my $docid = $head->get_docid; - my $doc = $db->get_document($docid); - $doc->remove_term('G' . $loser_tid); - $doc->add_term('G' . $winner_tid); - $db->replace_document($docid, $doc); + my $batch_size = 1000; # don't let @ids grow too large to avoid OOM + while (1) { + my ($head, $tail) = $self->find_doc_ids('G' . $loser_tid); + return if $head == $tail; + my @ids; + for (; $head != $tail && @ids < $batch_size; $head->inc) { + push @ids, $head->get_docid; + } + foreach my $docid (@ids) { + my $doc = $db->get_document($docid); + $doc->remove_term('G' . $loser_tid); + $doc->add_term('G' . $winner_tid); + $db->replace_document($docid, $doc); + } } } -- cgit v1.2.3-24-ge0c7