From: Eric Wong <e@yhbt.net>
To: meta@public-inbox.org
Subject: [PATCH 1/5] xt/eml_check_limits: check limits against an inbox
Date: Sun, 10 May 2020 22:37:11 +0000 [thread overview]
Message-ID: <20200510223715.19254-2-e@yhbt.net> (raw)
In-Reply-To: <20200510223715.19254-1-e@yhbt.net>
This allows maintainers to easily check limits against the
contents of existing inboxes. This script covers most of
the new limits enforced by PublicInbox::Eml.
Usage is similar to most xt/*.t scripts:
GIANT_INBOX_DIR=/path/to/inbox prove -bvw xt/eml_check_limits.t
Setting `TEST_CLASS=PublicInbox::MIME' allows us to check
performance and memory use against the old subclass of
Email::MIME.
---
MANIFEST | 1 +
xt/eml_check_limits.t | 76 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 77 insertions(+)
create mode 100644 xt/eml_check_limits.t
diff --git a/MANIFEST b/MANIFEST
index 9c804a0780e..b1512c7a919 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -333,6 +333,7 @@ t/x-unknown-alpine.eml
t/xcpdb-reshard.t
xt/cmp-msgstr.t
xt/cmp-msgview.t
+xt/eml_check_limits.t
xt/git-http-backend.t
xt/git_async_cmp.t
xt/mem-msgview.t
diff --git a/xt/eml_check_limits.t b/xt/eml_check_limits.t
new file mode 100644
index 00000000000..39de047645b
--- /dev/null
+++ b/xt/eml_check_limits.t
@@ -0,0 +1,76 @@
+#!perl -w
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use Test::More;
+use PublicInbox::TestCommon;
+use PublicInbox::Eml;
+use PublicInbox::Inbox;
+use List::Util qw(max);
+use Benchmark qw(:all :hireswallclock);
+use PublicInbox::Spawn qw(popen_rd);
+use Carp ();
+require_git(2.19); # for --unordered
+require_mods(qw(BSD::Resource));
+BSD::Resource->import(qw(getrusage));
+my $cls = $ENV{TEST_CLASS};
+require_mods($cls) if $cls;
+$cls //= 'PublicInbox::Eml';
+my $inboxdir = $ENV{GIANT_INBOX_DIR};
+plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir;
+local $PublicInbox::Eml::mime_nesting_limit = 0x7fffffff;
+local $PublicInbox::Eml::mime_parts_limit = 0x7fffffff;
+local $PublicInbox::Eml::header_size_limit = 0x7fffffff;
+my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'x' });
+my $git = $ibx->git;
+my @cat = qw(cat-file --buffer --batch-check --batch-all-objects --unordered);
+my $fh = $git->popen(@cat);
+my ($m, $n);
+my $max_nest = [ 0, '' ]; # [ bytes, blob oid ]
+my $max_idx = [ 0, '' ];
+my $max_parts = [ 0, '' ];
+my $max_size = [ 0, '' ];
+my $max_hdr = [ 0, '' ];
+my $info = [ 0, '' ];
+my $each_part_cb = sub {
+ my ($p) = @_;
+ my ($part, $depth, $idx) = @$p;
+ $max_nest = [ $depth, $info->[1] ] if $depth > $max_nest->[0];
+ my $max = max(split(/\./, $idx));
+ $max_idx = [ $max, $info->[1] ] if $max > $max_idx->[0];
+ ++$info->[0];
+};
+
+my ($bref, $oid, $size);
+local $SIG{__WARN__} = sub { diag "$inboxdir $oid ", @_ };
+my $cat_cb = sub {
+ ($bref, $oid, undef, $size) = @_;
+ ++$m;
+ $info = [ 0, $oid ];
+ my $eml = $cls->new($bref);
+ my $hdr_len = length($eml->header_obj->as_string);
+ $max_hdr = [ $hdr_len, $oid ] if $hdr_len > $max_hdr->[0];
+ $eml->each_part($each_part_cb, $info, 1);
+ $max_parts = $info if $info->[0] > $max_parts->[0];
+ $max_size = [ $size, $oid ] if $size > $max_size->[0];
+};
+
+my $t = timeit(1, sub {
+ $git->cat_async_begin;
+ my ($blob, $type);
+ while (<$fh>) {
+ ($blob, $type) = split / /;
+ next if $type ne 'blob';
+ ++$n;
+ $git->cat_async($blob, $cat_cb);
+ }
+ $git->cat_async_wait;
+});
+is($m, $n, 'scanned all messages');
+diag "$$ $inboxdir took ".timestr($t)." for $n <=> $m messages";
+diag "$$ max_nest $max_nest->[0] @ $max_nest->[1]";
+diag "$$ max_idx $max_idx->[0] @ $max_idx->[1]";
+diag "$$ max_parts $max_parts->[0] @ $max_parts->[1]";
+diag "$$ max_size $max_size->[0] @ $max_size->[1]";
+diag "$$ max_hdr $max_hdr->[0] @ $max_hdr->[1]";
+diag "$$ RSS ".getrusage()->maxrss. ' k';
+done_testing;
next prev parent reply other threads:[~2020-05-10 22:37 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-05-10 22:37 [PATCH 0/5] scattered dev/CLI-oriented changes Eric Wong
2020-05-10 22:37 ` Eric Wong [this message]
2020-05-10 22:37 ` [PATCH 2/5] rename "ContentId" to "ContentHash" Eric Wong
2020-05-10 22:37 ` [PATCH 3/5] overidx: document the SQLite PRAGMA we use Eric Wong
2020-05-10 22:37 ` [PATCH 4/5] msgmap: use TRUNCATE for journal_mode, for now Eric Wong
2020-05-10 22:37 ` [PATCH 5/5] spawn: use ~/.cache/public-inbox/inline-c if writable Eric Wong
2020-05-11 0:29 ` Eric Wong
2020-05-11 4:27 ` [PATCH v2] " Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200510223715.19254-2-e@yhbt.net \
--to=e@yhbt.net \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).