From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B82C22141E for ; Thu, 10 Jan 2019 21:35:50 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 6/7] over: cull unneeded fields for get_thread Date: Thu, 10 Jan 2019 21:35:48 +0000 Message-Id: <20190110213549.19778-7-e@80x24.org> In-Reply-To: <20190110213549.19778-1-e@80x24.org> References: <20190110213549.19778-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: On a certain ugly /$INBOX/$MESSAGE_ID/T/ endpoint with 1000 messages in the thread, this cuts memory usage from 2.5M to 1.9M (which still isn't great, but it's a start). --- lib/PublicInbox/Over.pm | 19 ++++++++++++------- lib/PublicInbox/SearchMsg.pm | 19 +++++++++++-------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm index dda1e6d..598c9fc 100644 --- a/lib/PublicInbox/Over.pm +++ b/lib/PublicInbox/Over.pm @@ -40,13 +40,16 @@ sub disconnect { $_[0]->{dbh} = undef } sub connect { $_[0]->{dbh} ||= $_[0]->dbh_new } -sub load_from_row { - my ($smsg) = @_; +sub load_from_row ($;$) { + my ($smsg, $cull) = @_; bless $smsg, 'PublicInbox::SearchMsg'; if (defined(my $data = delete $smsg->{ddd})) { $data = uncompress($data); utf8::decode($data); - $smsg->load_from_data($data); + PublicInbox::SearchMsg::load_from_data($smsg, $data); + + # saves over 600K for 1000+ message threads + PublicInbox::SearchMsg::psgi_cull($smsg) if $cull; } $smsg } @@ -57,7 +60,8 @@ sub do_get { my $lim = (($opts->{limit} || 0) + 0) || DEFAULT_LIMIT; $sql .= "LIMIT $lim"; my $msgs = $dbh->selectall_arrayref($sql, { Slice => {} }, @args); - load_from_row($_) for @$msgs; + my $cull = $opts->{cull}; + load_from_row($_, $cull) for @$msgs; $msgs } @@ -82,6 +86,7 @@ sub nothing () { wantarray ? (0, []) : [] }; sub get_thread { my ($self, $mid, $prev) = @_; my $dbh = $self->connect; + my $opts = { cull => 1 }; my $id = $dbh->selectrow_array(<<'', undef, $mid); SELECT id FROM msgid WHERE mid = ? LIMIT 1 @@ -109,7 +114,7 @@ SELECT tid,sid FROM over WHERE num = ? LIMIT 1 my $cols = 'num,ts,ds,ddd'; unless (wantarray) { - return do_get($self, <<"", {}, $tid, $sid, $num); + return do_get($self, <<"", $opts, $tid, $sid, $num); SELECT $cols FROM over WHERE $cond_all ORDER BY $sort_col ASC @@ -123,14 +128,14 @@ SELECT COUNT(num) FROM over WHERE $cond_all # giant thread, prioritize strict (tid) matches and throw # in the loose (sid) matches at the end - my $msgs = do_get($self, <<"", {}, $tid, $num); + my $msgs = do_get($self, <<"", $opts, $tid, $num); SELECT $cols FROM over WHERE tid = ? AND num > ? ORDER BY $sort_col ASC # do we have room for loose matches? get the most recent ones, first: my $lim = DEFAULT_LIMIT - scalar(@$msgs); if ($lim > 0) { - my $opts = { limit => $lim }; + $opts->{limit} = $lim; my $loose = do_get($self, <<"", $opts, $tid, $sid, $num); SELECT $cols FROM over WHERE tid != ? AND sid = ? AND num > ? ORDER BY $sort_col DESC diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index 292efce..722a1b9 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -82,18 +82,21 @@ sub load_expand { $self; } +sub psgi_cull ($) { + my ($self) = @_; + from_name($self); # fill in {from_name} so we can delete {from} + + # drop NNTP-only fields which aren't relevant to PSGI results: + # saves ~80K on a 200 item search result: + delete @$self{qw(from ts to cc bytes lines)}; + $self; +} + # Only called by PSGI interface, not NNTP sub load_doc { my ($class, $doc) = @_; my $self = bless {}, $class; - my $smsg = load_expand($self, $doc); - - from_name($smsg); # fill in {from_name} so we can delete {from} - - # drop NNTP-only fields which aren't relevant to PSGI results: - # saves ~80K on a 200 item search result: - delete @$smsg{qw(from ts to cc bytes lines)}; - $smsg; + psgi_cull(load_expand($self, $doc)); } # :bytes and :lines metadata in RFC 3977 -- EW