From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF, T_SCC_BODY_TEXT_LINE shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B50871F601 for ; Sat, 10 Sep 2022 08:18:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1662797931; bh=PlS3Wo1jT3DgEb81Ql2zIooC9Hm4RRksMeQ3CGD0HPA=; h=From:To:Subject:Date:In-Reply-To:References:From; b=T5f4EV4YH3DQ+vImu6G6Cns7k7vb9zvULEXEK4Zs0rbkA69RSDwbPh9sBwU4KvYD0 ViVvMvblWYElPISZzIGrLy9r4k576e9iTd3udL4byJN07jQHk/IebkW6YRRuRcnFKP YxtofvJ9fYalw5ttEVbLThI11LbL6v5lPr9CvIOQ= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 01/38] xt: fold perf-obfuscate into perf-msgview, future-proof Date: Sat, 10 Sep 2022 08:16:52 +0000 Message-Id: <20220910081729.2011934-2-e@80x24.org> In-Reply-To: <20220910081729.2011934-1-e@80x24.org> References: <20220910081729.2011934-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: perf-obfuscate was close enough to perf-msgview that it only required setting the `obfuscate' field of the inbox. Then update perf-msgview to account for upcoming internal changes. The current use of {obuf} and concat ops results in excessive scratchpad space and I may be able to even get speedups by avoiding concat ops. --- MANIFEST | 1 - xt/perf-msgview.t | 10 ++++--- xt/perf-obfuscate.t | 66 --------------------------------------------- 3 files changed, 7 insertions(+), 70 deletions(-) delete mode 100644 xt/perf-obfuscate.t diff --git a/MANIFEST b/MANIFEST index ac21ddcc..8be912d0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -602,7 +602,6 @@ xt/nntpd-validate.t xt/over-fsck.perl xt/perf-msgview.t xt/perf-nntpd.t -xt/perf-obfuscate.t xt/perf-threading.t xt/pop3d-mpop.t xt/solver.t diff --git a/xt/perf-msgview.t b/xt/perf-msgview.t index 7f92ce85..ef261359 100644 --- a/xt/perf-msgview.t +++ b/xt/perf-msgview.t @@ -11,6 +11,8 @@ use PublicInbox::WwwStream; my $inboxdir = $ENV{GIANT_INBOX_DIR} // $ENV{GIANT_PI_DIR}; my $blob = $ENV{TEST_BLOB}; +my $obfuscate = $ENV{PI_OBFUSCATE} ? 1 : 0; +diag "PI_OBFUSCATE=$obfuscate"; plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir; my @cat = qw(cat-file --buffer --batch-check --batch-all-objects); @@ -21,7 +23,8 @@ if (require_git(2.19, 1)) { "git <2.19, cat-file lacks --unordered, locality suffers\n"; } require_mods qw(Plack::Util); -my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'name' }); +my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'name', + obfuscate => $obfuscate}); my $git = $ibx->git; my $fh = $blob ? undef : $git->popen(@cat); if ($fh) { @@ -46,10 +49,11 @@ $ctx->{mhref} = '../'; my $cb = sub { $eml = PublicInbox::Eml->new(shift); $eml->each_part(\&PublicInbox::View::add_text_body, $ctx, 1); - $ctx->zflush; + $ctx->zflush(grep defined, delete @$ctx{'obuf'}); # compat ++$m; delete $ctx->{zbuf}; - ${$ctx->{obuf}} = ''; + ${$ctx->{obuf}} = ''; # compat + $ctx->{gz} = PublicInbox::GzipFilter::gzip_or_die(); }; my $t = timeit(1, sub { diff --git a/xt/perf-obfuscate.t b/xt/perf-obfuscate.t deleted file mode 100644 index 4da36124..00000000 --- a/xt/perf-obfuscate.t +++ /dev/null @@ -1,66 +0,0 @@ -#!perl -w -# Copyright (C) all contributors -# License: AGPL-3.0+ -use strict; -use v5.10.1; -use PublicInbox::TestCommon; -use Benchmark qw(:all); -use PublicInbox::Inbox; -use PublicInbox::View; -use PublicInbox::WwwStream; - -my $inboxdir = $ENV{GIANT_INBOX_DIR}; -plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir; - -my $obfuscate = $ENV{PI_OBFUSCATE} ? 1 : 0; -diag "obfuscate=$obfuscate\n"; - -my @cat = qw(cat-file --buffer --batch-check --batch-all-objects); -if (require_git(2.19, 1)) { - push @cat, '--unordered'; -} else { - warn -"git <2.19, cat-file lacks --unordered, locality suffers\n"; -} -require_mods qw(Plack::Util); -my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'name' , - obfuscate => $obfuscate}); -my $git = $ibx->git; -my $fh = $git->popen(@cat); -my $vec = ''; -vec($vec, fileno($fh), 1) = 1; -select($vec, undef, undef, 60) or die "timed out waiting for --batch-check"; - -my $ctx = bless { - env => { HTTP_HOST => 'example.com', 'psgi.url_scheme' => 'https' }, - ibx => $ibx, - www => Plack::Util::inline_object(style => sub {''}), - gz => PublicInbox::GzipFilter::gzip_or_die(), -}, 'PublicInbox::WwwStream'; -my ($eml, $res, $oid, $type); -my $n = 0; -my $m = 0; -${$ctx->{obuf}} = ''; -$ctx->{mhref} = '../'; - -my $cb = sub { - $eml = PublicInbox::Eml->new(shift); - $eml->each_part(\&PublicInbox::View::add_text_body, $ctx, 1); - $ctx->zflush; - ++$m; - delete $ctx->{zbuf}; - ${$ctx->{obuf}} = ''; -}; - -my $t = timeit(1, sub { - while (<$fh>) { - ($oid, $type) = split / /; - next if $type ne 'blob'; - ++$n; - $git->cat_async($oid, $cb); - } - $git->async_wait_all; -}); -diag 'add_text_body took '.timestr($t)." for $n <=> $m messages"; -is($m, $n, 'rendered all messages'); -done_testing();