* [PATCH 2/5] search: avoid creating ghosts for circular References
@ 2015-08-18 1:21 7% ` Eric Wong
2015-08-18 1:21 6% ` [PATCH 5/5] search: bump SCHEMA_VERSION to 4 Eric Wong
1 sibling, 0 replies; 2+ results
From: Eric Wong @ 2015-08-18 1:21 UTC (permalink / raw)
To: meta
Some mail software incorrectly creates circular references
and causes us to create ghosts before the actual mail doc
is created.
---
lib/PublicInbox/Search.pm | 43 ++++++++++++++++++++++++++++++-------------
t/search.t | 19 +++++++++++++++++++
2 files changed, 49 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 617c267..db86301 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -79,8 +79,8 @@ sub add_message {
my $db = $self->{xdb};
my $doc_id;
- my $mid = mid_clean($mime->header_obj->header_raw('Message-ID'));
- $mid = mid_compressed($mid);
+ my $mid_orig = mid_clean($mime->header_obj->header_raw('Message-ID'));
+ my $mid = mid_compressed($mid_orig);
my $was_ghost = 0;
my $ct_msg = $mime->header('Content-Type') || 'text/plain';
my $enc_msg = PublicInbox::View::enc_for($ct_msg);
@@ -176,7 +176,7 @@ sub add_message {
};
if ($@) {
- warn "failed to index message <$mid>: $@\n";
+ warn "failed to index message <$mid_orig>: $@\n";
return undef;
}
$doc_id;
@@ -184,11 +184,11 @@ sub add_message {
# returns deleted doc_id on success, undef on missing
sub remove_message {
- my ($self, $mid) = @_;
+ my ($self, $mid_orig) = @_;
my $db = $self->{xdb};
my $doc_id;
- $mid = mid_clean($mid);
- $mid = mid_compressed($mid);
+ $mid_orig = mid_clean($mid_orig);
+ my $mid = mid_compressed($mid_orig);
eval {
$doc_id = $self->find_unique_doc_id('mid', $mid);
@@ -196,7 +196,7 @@ sub remove_message {
};
if ($@) {
- warn "failed to remove message <$mid>: $@\n";
+ warn "failed to remove message <$mid_orig>: $@\n";
return undef;
}
$doc_id;
@@ -347,16 +347,33 @@ sub link_message_to_parents {
if ($irt =~ /<([^>]+)>/) {
$irt = $1;
}
- push @refs, $irt;
+
+ # maybe some crazies will try to make a circular reference:
+ if ($irt eq $mid) {
+ $irt = undef;
+ } else {
+ push @refs, $irt;
+ }
}
my $tid;
if (@refs) {
- @refs = map { mid_compressed($_) } @refs;
- my %uniq;
- @refs = grep { !$uniq{$_}++ } @refs; # uniq
-
- $doc->add_term(xpfx('inreplyto') . $refs[-1]);
+ my @crefs = map { mid_compressed($_) } @refs;
+ my %uniq = ($mid => 1);
+
+ # prevent circular references via References: here:
+ @refs = ();
+ foreach my $ref (@crefs) {
+ next if $uniq{$ref};
+ $uniq{$ref} = 1;
+ push @refs, $ref;
+ }
+ $irt = undef if (defined $irt && !$uniq{$irt});
+ }
+ if (@refs) {
+ if (defined $irt) {
+ $doc->add_term(xpfx('inreplyto') . $irt);
+ }
my $ref_pfx = xpfx('references');
diff --git a/t/search.t b/t/search.t
index 0ad0886..55abe9e 100644
--- a/t/search.t
+++ b/t/search.t
@@ -243,6 +243,25 @@ sub filter_mids {
"quoted result returned if nothing else");
}
+# circular references
+{
+ my $doc_id = $rw->add_message(Email::MIME->create(
+ header_str => [
+ Date => 'Sat, 02 Oct 2010 00:00:01 +0000',
+ Subject => 'Circle',
+ 'Message-ID' => '<circle@a>',
+ 'References' => '<circle@a>',
+ 'In-Reply-To' => '<circle@a>',
+ From => 'Circle <circle@example.com>',
+ To => 'list@example.com',
+ ],
+ body => "LOOP!\n"));
+ ok($doc_id > 0, "doc_id defined with circular reference");
+ my $smsg = $rw->lookup_message('circle@a');
+ $smsg->ensure_metadata;
+ is($smsg->{references}, undef, "no references created");
+}
+
done_testing();
1;
--
EW
^ permalink raw reply related [relevance 7%]
* [PATCH 5/5] search: bump SCHEMA_VERSION to 4
2015-08-18 1:21 7% ` [PATCH 2/5] search: avoid creating ghosts for circular References Eric Wong
@ 2015-08-18 1:21 6% ` Eric Wong
1 sibling, 0 replies; 2+ results
From: Eric Wong @ 2015-08-18 1:21 UTC (permalink / raw)
To: meta
The following two commits affect indexing behavior, so
change the schema version to avoid compatibility problems
or missing messages:
search: common Subject: normalization for Re: prefixes
search: avoid creating ghosts for circular References
---
lib/PublicInbox/Search.pm | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index d767941..b9f283f 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -22,7 +22,8 @@ use constant {
# 1 - subject_path is lower-cased
# 2 - subject_path is mid_compressed in the index, only
# 3 - message-ID is compressed if it includes '%' (hack!)
- SCHEMA_VERSION => 3,
+ # 4 - change "Re: " normalization, avoid circular Reference ghosts
+ SCHEMA_VERSION => 4,
QP_FLAGS => FLAG_PHRASE|FLAG_BOOLEAN|FLAG_LOVEHATE|FLAG_WILDCARD,
};
--
EW
^ permalink raw reply related [relevance 6%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2015-08-18 1:21 [PATCH 1/5] view: cleaner Message-ID filtering for References Eric Wong
2015-08-18 1:21 7% ` [PATCH 2/5] search: avoid creating ghosts for circular References Eric Wong
2015-08-18 1:21 6% ` [PATCH 5/5] search: bump SCHEMA_VERSION to 4 Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).