From 031dcde21cd8dab5494d9715ba50d6a539e3fb42 Mon Sep 17 00:00:00 2001 From: "Eric Wong (Contractor, The Linux Foundation)" Date: Sat, 3 Mar 2018 20:21:05 +0000 Subject: searchidx: store the primary MID in doc data for NNTP We can't rely on header order for Message-ID after all since we fall back to existing MIDs if they exist and are unseen. This lets us use SearchMsg->mid to get the MID we associated with the NNTP article number to ensure all NNTP article lookups roundtrip correctly. --- lib/PublicInbox/SearchIdx.pm | 6 ++++-- lib/PublicInbox/SearchIdxPart.pm | 8 ++++---- lib/PublicInbox/SearchMsg.pm | 9 +++++---- lib/PublicInbox/V2Writable.pm | 34 ++++++++++++++++++++-------------- 4 files changed, 33 insertions(+), 24 deletions(-) (limited to 'lib/PublicInbox') diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index a70e1ebf..71469a95 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -302,7 +302,8 @@ sub index_body ($$$) { } sub add_message { - my ($self, $mime, $bytes, $num, $blob) = @_; # mime = Email::MIME object + # mime = Email::MIME object + my ($self, $mime, $bytes, $num, $oid, $mid0) = @_; my $doc_id; my $mids = mids($mime->header_obj); my $skel = $self->{skeleton}; @@ -370,7 +371,8 @@ sub add_message { # populates smsg->references for smsg->to_doc_data my $refs = parse_references($smsg); - my $data = $smsg->to_doc_data($blob); + $mid0 = $mids->[0] unless defined $mid0; + my $data = $smsg->to_doc_data($oid, $mid0); foreach my $mid (@$mids) { $tg->index_text($mid, 1, 'XM'); } diff --git a/lib/PublicInbox/SearchIdxPart.pm b/lib/PublicInbox/SearchIdxPart.pm index 2f577ecf..6d8cb2a7 100644 --- a/lib/PublicInbox/SearchIdxPart.pm +++ b/lib/PublicInbox/SearchIdxPart.pm @@ -51,7 +51,7 @@ sub partition_worker_loop ($$$) { $xdb = $txn = undef; } else { chomp $line; - my ($len, $artnum, $object_id) = split(/ /, $line); + my ($len, $artnum, $oid, $mid0) = split(/ /, $line); $xdb ||= $self->_xdb_acquire; if (!$txn) { $xdb->begin_transaction; @@ -61,7 +61,7 @@ sub partition_worker_loop ($$$) { $n == $len or die "short read: $n != $len\n"; my $mime = PublicInbox::MIME->new(\$msg); $artnum = int($artnum); - $self->add_message($mime, $n, $artnum, $object_id); + $self->add_message($mime, $n, $artnum, $oid, $mid0); } } warn "$$ still in transaction\n" if $txn; @@ -70,9 +70,9 @@ sub partition_worker_loop ($$$) { # called by V2Writable sub index_raw { - my ($self, $len, $msgref, $artnum, $object_id) = @_; + my ($self, $len, $msgref, $artnum, $object_id, $mid0) = @_; my $w = $self->{w}; - print $w "$len $artnum $object_id\n", $$msgref or die + print $w "$len $artnum $object_id $mid0\n", $$msgref or die "failed to write partition $!\n"; $w->flush or die "failed to flush: $!\n"; } diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index 93e6fd8b..a62a6490 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -31,13 +31,14 @@ sub get_val ($$) { sub load_from_data ($$) { my ($self) = $_[0]; # data = $_[1] - my ($subj, $from, $refs, $to, $cc, $blob) = split(/\n/, $_[1]); + my ($subj, $from, $refs, $to, $cc, $blob, $mid0) = split(/\n/, $_[1]); $self->{subject} = $subj; $self->{from} = $from; $self->{references} = $refs; $self->{to} = $to; $self->{cc} = $cc; $self->{blob} = $blob; + $self->{mid} = $mid0; } sub load_expand { @@ -120,11 +121,11 @@ sub ts { } sub to_doc_data { - my ($self, $blob) = @_; + my ($self, $oid, $mid0) = @_; my @rows = ($self->subject, $self->from, $self->references, $self->to, $self->cc); - push @rows, $blob if defined $blob; - join("\n", @rows); + $oid = '' unless defined $oid; + join("\n", @rows, $oid, $mid0); } sub references { diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index c73d859b..caabc8e4 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -62,8 +62,10 @@ sub add { # leaking FDs to it... $self->idx_init; - my $num = num_for($self, $mime); + my $mid0; + my $num = num_for($self, $mime, \$mid0); defined $num or return; # duplicate + defined $mid0 or die "BUG: $mid0 undefined\n"; my $im = $self->importer; my $cmt = $im->add($mime); $cmt = $im->get_mark($cmt); @@ -73,7 +75,7 @@ sub add { my $nparts = $self->{partitions}; my $part = $num % $nparts; my $idx = $self->idx_part($part); - $idx->index_raw($len, $msgref, $num, $oid); + $idx->index_raw($len, $msgref, $num, $oid, $mid0); my $n = $self->{transact_bytes} += $len; if ($n > (PublicInbox::SearchIdx::BATCH_BYTES * $nparts)) { $self->checkpoint; @@ -83,12 +85,15 @@ sub add { } sub num_for { - my ($self, $mime) = @_; + my ($self, $mime, $mid0) = @_; my $mids = mids($mime->header_obj); if (@$mids) { my $mid = $mids->[0]; my $num = $self->{skel}->{mm}->mid_insert($mid); - return $num if defined($num); # common case + if (defined $num) { # common case + $$mid0 = $mid; + return $num; + }; # crap, Message-ID is already known, hope somebody just resent: $self->done; # write barrier, clears $self->{skel} @@ -111,38 +116,39 @@ sub num_for { $num = $self->{skel}->{mm}->mid_insert($m); if (defined $num) { warn "alternative <$m> for <$mid> found\n"; + $$mid0 = $m; return $num; } } } # none of the existing Message-IDs are good, generate a new one: - num_for_harder($self, $mime); + num_for_harder($self, $mime, $mid0); } sub num_for_harder { - my ($self, $mime) = @_; + my ($self, $mime, $mid0) = @_; my $hdr = $mime->header_obj; my $dig = content_digest($mime); - my $mid = $dig->clone->hexdigest . '@localhost'; - my $num = $self->{skel}->{mm}->mid_insert($mid); + $$mid0 = $dig->clone->hexdigest . '@localhost'; + my $num = $self->{skel}->{mm}->mid_insert($$mid0); unless (defined $num) { # it's hard to spoof the last Received: header my @recvd = $hdr->header_raw('Received'); $dig->add("Received: $_") foreach (@recvd); - $mid = $dig->clone->hexdigest . '@localhost'; - $num = $self->{skel}->{mm}->mid_insert($mid); + $$mid0 = $dig->clone->hexdigest . '@localhost'; + $num = $self->{skel}->{mm}->mid_insert($$mid0); # fall back to a random Message-ID and give up determinism: until (defined($num)) { $dig->add(rand); - $mid = $dig->clone->hexdigest . '@localhost'; - warn "using random Message-ID <$mid> as fallback\n"; - $num = $self->{skel}->{mm}->mid_insert($mid); + $$mid0 = $dig->clone->hexdigest . '@localhost'; + warn "using random Message-ID <$$mid0> as fallback\n"; + $num = $self->{skel}->{mm}->mid_insert($$mid0); } } my @cur = $hdr->header_raw('Message-Id'); - $hdr->header_set('Message-Id', "<$mid>", @cur); + $hdr->header_set('Message-Id', "<$$mid0>", @cur); $num; } -- cgit v1.2.3-24-ge0c7