diff options
Diffstat (limited to 'lib/PublicInbox/MdirReader.pm')
| -rw-r--r-- | lib/PublicInbox/MdirReader.pm | 25 |
1 files changed, 21 insertions, 4 deletions
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm index 1685e4d8..b49c8ceb 100644 --- a/lib/PublicInbox/MdirReader.pm +++ b/lib/PublicInbox/MdirReader.pm @@ -8,6 +8,7 @@ package PublicInbox::MdirReader; use strict; use v5.10.1; use PublicInbox::InboxWritable qw(eml_from_path); +use Digest::SHA qw(sha256_hex); # returns Maildir flags from a basename ('' for no flags, undef for invalid) sub maildir_basename_flags { @@ -24,14 +25,25 @@ sub maildir_path_flags { $i >= 0 ? maildir_basename_flags(substr($f, $i + 1)) : undef; } -sub maildir_each_file ($$;@) { - my ($dir, $cb, @arg) = @_; +sub shard_ok ($$$) { + my ($bn, $mod, $shard) = @_; + # can't get dirent.d_ino w/ pure Perl readdir, so we extract + # the OID if it looks like one instead of doing stat(2) + my $hex = $bn =~ m!\A([a-f0-9]{40,})! ? $1 : sha256_hex($bn); + my $recno = hex(substr($hex, 0, 8)); + ($recno % $mod) == $shard; +} + +sub maildir_each_file { + my ($self, $dir, $cb, @arg) = @_; $dir .= '/' unless substr($dir, -1) eq '/'; + my ($mod, $shard) = @{$self->{shard_info} // []}; for my $d (qw(new/ cur/)) { my $pfx = $dir.$d; opendir my $dh, $pfx or next; while (defined(my $bn = readdir($dh))) { maildir_basename_flags($bn) // next; + next if defined($mod) && !shard_ok($bn, $mod, $shard); $cb->($pfx.$bn, @arg); } } @@ -40,15 +52,17 @@ sub maildir_each_file ($$;@) { my %c2kw = ('D' => 'draft', F => 'flagged', P => 'forwarded', R => 'answered', S => 'seen'); -sub maildir_each_eml ($$;@) { - my ($dir, $cb, @arg) = @_; +sub maildir_each_eml { + my ($self, $dir, $cb, @arg) = @_; $dir .= '/' unless substr($dir, -1) eq '/'; + my ($mod, $shard) = @{$self->{shard_info} // []}; my $pfx = $dir . 'new/'; if (opendir(my $dh, $pfx)) { while (defined(my $bn = readdir($dh))) { next if substr($bn, 0, 1) eq '.'; my @f = split(/:/, $bn, -1); next if scalar(@f) != 1; + next if defined($mod) && !shard_ok($bn, $mod, $shard); my $f = $pfx.$bn; my $eml = eml_from_path($f) or next; $cb->($f, [], $eml, @arg); @@ -59,6 +73,7 @@ sub maildir_each_eml ($$;@) { while (defined(my $bn = readdir($dh))) { my $fl = maildir_basename_flags($bn) // next; next if index($fl, 'T') >= 0; + next if defined($mod) && !shard_ok($bn, $mod, $shard); my $f = $pfx.$bn; my $eml = eml_from_path($f) or next; my @kw = sort(map { $c2kw{$_} // () } split(//, $fl)); @@ -66,4 +81,6 @@ sub maildir_each_eml ($$;@) { } } +sub new { bless {}, __PACKAGE__ } + 1; |