1 files changed, 15 insertions, 8 deletions
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 316111bf..30d3fe92 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -20,13 +20,14 @@ use Carp qw(croak);
  use POSIX qw(strftime);
  require PublicInbox::Git;
  
-use constant MAX_MID_SIZE => 244; # max term size - 1 in Xapian
  use constant {
+        MAX_MID_SIZE => 244, # max term size - 1 in Xapian
          PERM_UMASK => 0,
          OLD_PERM_GROUP => 1,
          OLD_PERM_EVERYBODY => 2,
          PERM_GROUP => 0660,
          PERM_EVERYBODY => 0664,
+        BATCH_BYTES => 1_000_000,
  };
  
  sub new {
@@ -71,7 +72,6 @@ sub _xdb_acquire {
                  require File::Path;
                  _lock_acquire($self);
                  File::Path::mkpath($dir);
-                $self->{batch_size} = 100;
                  $flag = Search::Xapian::DB_CREATE_OR_OPEN;
          }
          $self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag);
@@ -395,6 +395,15 @@ sub index_sync {
          with_umask($self, sub { $self->_index_sync($opts) });
  }
  
+sub batch_adjust ($$$$) {
+        my ($max, $bytes, $batch_cb, $latest) = @_;
+        $$max -= $bytes;
+        if ($$max <= 0) {
+                $$max = BATCH_BYTES;
+                $batch_cb->($latest, 1);
+        }
+}
+
  sub rlog {
          my ($self, $log, $add_cb, $del_cb, $batch_cb) = @_;
          my $hex = '[a-f0-9]';
@@ -404,23 +413,21 @@ sub rlog {
          my $git = $self->{git};
          my $latest;
          my $bytes;
-        my $max = $self->{batch_size}; # may be undef
+        my $max = BATCH_BYTES;
          local $/ = "\n";
          my $line;
          while (defined($line = <$log>)) {
                  if ($line =~ /$addmsg/o) {
                          my $blob = $1;
                          my $mime = do_cat_mail($git, $blob, \$bytes) or next;
+                        batch_adjust(\$max, $bytes, $batch_cb, $latest);
                          $add_cb->($self, $mime, $bytes, $blob);
                  } elsif ($line =~ /$delmsg/o) {
                          my $blob = $1;
-                        my $mime = do_cat_mail($git, $blob) or next;
+                        my $mime = do_cat_mail($git, $blob, \$bytes) or next;
+                        batch_adjust(\$max, $bytes, $batch_cb, $latest);
                          $del_cb->($self, $mime);
                  } elsif ($line =~ /^commit ($h40)/o) {
-                        if (defined $max && --$max <= 0) {
-                                $max = $self->{batch_size};
-                                $batch_cb->($latest, 1);
-                        }
                          $latest = $1;
                  }
          }