From bbb8c7788ebaa7f3679d387725ee1998f0b2f615 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 8 Jan 2019 11:13:27 +0000 Subject: searchmsg: get rid of termlist scanning for mid It doesn't seem to be used anywhere --- lib/PublicInbox/SearchMsg.pm | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) (limited to 'lib/PublicInbox/SearchMsg.pm') diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index c7787ea1..5a2ca83b 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -157,29 +157,17 @@ sub references { defined $x ? $x : ''; } -sub _get_term_val ($$$) { - my ($self, $pfx, $re) = @_; - my $doc = $self->{doc}; - my $end = $doc->termlist_end; - my $i = $doc->termlist_begin; - $i->skip_to($pfx); - if ($i != $end) { - my $val = $i->get_termname; - $val =~ s/$re// and return $val; - } - undef; -} - sub mid ($;$) { my ($self, $mid) = @_; if (defined $mid) { $self->{mid} = $mid; - } elsif (my $rv = $self->{mid}) { + } elsif (defined(my $rv = $self->{mid})) { $rv; } elsif ($self->{doc}) { - $self->{mid} = _get_term_val($self, 'Q', qr/\AQ/); + die "SHOULD NOT HAPPEN\n"; } else { + die "NO {mime} for mid\n" unless $self->{mime}; $self->_extract_mid; # v1 w/o Xapian } } -- cgit v1.2.3-24-ge0c7 From 588111b86a69c28384f4c907fe7b1b87aa484f32 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 8 Jan 2019 11:13:29 +0000 Subject: searchmsg: remove Xapian::Document field We don't need to be carrying this around with the many SearchMsg objects we have. This saves about 20K from a large SearchView "&x=t" response. --- lib/PublicInbox/SearchMsg.pm | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'lib/PublicInbox/SearchMsg.pm') diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index 5a2ca83b..65e085f4 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -13,20 +13,18 @@ use Time::Local qw(timegm); sub new { my ($class, $mime) = @_; - my $doc = Search::Xapian::Document->new; - bless { doc => $doc, mime => $mime }, $class; + bless { mime => $mime }, $class; } sub wrap { - my ($class, $doc, $mid) = @_; - bless { doc => $doc, mime => undef, mid => $mid }, $class; + my ($class, $mid) = @_; + bless { mid => $mid }, $class; } sub get { my ($class, $head, $db, $mid) = @_; my $doc_id = $head->get_docid; - my $doc = $db->get_document($doc_id); - load_expand(wrap($class, $doc, $mid)) + load_expand(wrap($class, $mid), $db->get_document($doc_id)); } sub get_val ($$) { @@ -70,8 +68,7 @@ sub load_from_data ($$) { } sub load_expand { - my ($self) = @_; - my $doc = $self->{doc}; + my ($self, $doc) = @_; my $data = $doc->get_data or return; $self->{ts} = get_val($doc, PublicInbox::Search::TS()); my $dt = get_val($doc, PublicInbox::Search::DT()); @@ -84,8 +81,8 @@ sub load_expand { sub load_doc { my ($class, $doc) = @_; - my $self = bless { doc => $doc }, $class; - $self->load_expand; + my $self = bless {}, $class; + load_expand($self, $doc); } # :bytes and :lines metadata in RFC 3977 @@ -164,8 +161,6 @@ sub mid ($;$) { $self->{mid} = $mid; } elsif (defined(my $rv = $self->{mid})) { $rv; - } elsif ($self->{doc}) { - die "SHOULD NOT HAPPEN\n"; } else { die "NO {mime} for mid\n" unless $self->{mime}; $self->_extract_mid; # v1 w/o Xapian -- cgit v1.2.3-24-ge0c7 From d7ceeaadbab7034eccb88b0c53b9538c1c92b25a Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 8 Jan 2019 11:13:31 +0000 Subject: searchmsg: remove unused fields for PSGI in Xapian results These fields are only necessary in NNTP and not even stored in Xapian; so keeping them around for the PSGI web UI search results wastes nearly 80K when loading large result sets. --- lib/PublicInbox/SearchMsg.pm | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'lib/PublicInbox/SearchMsg.pm') diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index 65e085f4..292efcee 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -57,11 +57,14 @@ sub load_from_data ($$) { # To: and Cc: are stored to optimize HDR/XHDR in NNTP since # some NNTP clients will use that for message displays. + # NNTP only, and only stored in Over(view), not Xapian $self->{to}, $self->{cc}, $self->{blob}, $self->{mid}, + + # NNTP only $self->{bytes}, $self->{lines} ) = split(/\n/, $_[1]); @@ -79,10 +82,18 @@ sub load_expand { $self; } +# Only called by PSGI interface, not NNTP sub load_doc { my ($class, $doc) = @_; my $self = bless {}, $class; - load_expand($self, $doc); + my $smsg = load_expand($self, $doc); + + from_name($smsg); # fill in {from_name} so we can delete {from} + + # drop NNTP-only fields which aren't relevant to PSGI results: + # saves ~80K on a 200 item search result: + delete @$smsg{qw(from ts to cc bytes lines)}; + $smsg; } # :bytes and :lines metadata in RFC 3977 -- cgit v1.2.3-24-ge0c7 From 7766a71df4cb715a240fd97a03dc0c31ab153c21 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 8 Jan 2019 11:13:33 +0000 Subject: over: cull unneeded fields for get_thread On a certain ugly /$INBOX/$MESSAGE_ID/T/ endpoint with 1000 messages in the thread, this cuts memory usage from 2.5M to 1.9M (which still isn't great, but it's a start). --- lib/PublicInbox/SearchMsg.pm | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'lib/PublicInbox/SearchMsg.pm') diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index 292efcee..722a1b94 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -82,18 +82,21 @@ sub load_expand { $self; } +sub psgi_cull ($) { + my ($self) = @_; + from_name($self); # fill in {from_name} so we can delete {from} + + # drop NNTP-only fields which aren't relevant to PSGI results: + # saves ~80K on a 200 item search result: + delete @$self{qw(from ts to cc bytes lines)}; + $self; +} + # Only called by PSGI interface, not NNTP sub load_doc { my ($class, $doc) = @_; my $self = bless {}, $class; - my $smsg = load_expand($self, $doc); - - from_name($smsg); # fill in {from_name} so we can delete {from} - - # drop NNTP-only fields which aren't relevant to PSGI results: - # saves ~80K on a 200 item search result: - delete @$smsg{qw(from ts to cc bytes lines)}; - $smsg; + psgi_cull(load_expand($self, $doc)); } # :bytes and :lines metadata in RFC 3977 -- cgit v1.2.3-24-ge0c7