diff options
author | Eric Wong <e@80x24.org> | 2021-06-03 01:05:20 +0000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2021-06-03 01:09:43 +0000 |
commit | bdecd7ed8e0dcf0b45491b947cd737ba8cfe38a3 (patch) | |
tree | 33616d6248bf6b8d2a78d2a609f5ef8389b36b47 /lib/PublicInbox/LeiMailSync.pm | |
parent | 6ff03ba2be9247f1ead26c2524fadc789de558f1 (diff) | |
download | public-inbox-bdecd7ed8e0dcf0b45491b947cd737ba8cfe38a3.tar.gz |
On a 4-core CPU, this speeds up "lei import" on a largish IMAP inbox with 75K messages from ~21 minutes down to 40s. Parallelizing with the new LeiImportKw WQ worker class gives a near-linear speedup and brought the runtime down to ~5:40. The new idx_fid_uid index on the "fid" and "uid" columns of blob2num in mail_sync.sqlite3 brought us the final speedup. An additional index on over.sqlite3#xref3(oidbin) did not help, since idx_nntp already exists and speeds up the new ->oidbin_exists internal API. I initially experimented with a separate "lei import-kw" command but decided against it since it's useless outside of IMAP+JMAP and would require extra cognitive overhead for both users and hackers. So LeiImportKw is just a WQ worker used by "lei import" and not its own user-visible command. v2: fix ikw_done_wait arg handling (ugh, confusing API :x)
Diffstat (limited to 'lib/PublicInbox/LeiMailSync.pm')
-rw-r--r-- | lib/PublicInbox/LeiMailSync.pm | 17 |
1 files changed, 10 insertions, 7 deletions
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm index 22ee1109..75603d89 100644 --- a/lib/PublicInbox/LeiMailSync.pm +++ b/lib/PublicInbox/LeiMailSync.pm @@ -54,6 +54,10 @@ CREATE TABLE IF NOT EXISTS blob2num ( UNIQUE (oidbin, fid, uid) ) + # speeds up LeiImport->ck_update_kw (for "lei import") by 5-6x: + $dbh->do(<<''); +CREATE INDEX IF NOT EXISTS idx_fid_uid ON blob2num(fid,uid) + $dbh->do(<<''); CREATE TABLE IF NOT EXISTS blob2name ( oidbin VARBINARY NOT NULL, @@ -361,15 +365,14 @@ sub forget_folder { $dbh->do('DELETE FROM folders WHERE fid = ?', undef, $fid); } -sub imap_oid2 ($$$) { - my ($self, $uri, $uid) = @_; # $uri MUST have UIDVALIDITY - my $fid = $self->{fmap}->{"$uri"} //= fid_for($self, "$uri") // return; +sub imap_oidbin ($$$) { + my ($self, $url, $uid) = @_; # $url MUST have UIDVALIDITY + my $fid = $self->{fmap}->{$url} //= fid_for($self, $url) // return; my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1); SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ? EOM $sth->execute($fid, $uid); - my ($oidbin) = $sth->fetchrow_array; - $oidbin ? unpack('H*', $oidbin) : undef; + $sth->fetchrow_array; } sub imap_oid { @@ -384,10 +387,10 @@ sub imap_oid { } $lei->qerr(@{$err->{qerr}}) if $err->{qerr}; } - imap_oid2($self, $folders->[0], $uid_uri->uid); + my $oidbin = imap_oidbin($self, $folders->[0], $uid_uri->uid); + $oidbin ? unpack('H*', $oidbin) : undef; } - # FIXME: something with "lei <up|q>" is causing uncommitted transaction # warnings, not sure what... sub DESTROY { |