From 761736a312a103ba522abac52a604564f9e788ce Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Fri, 18 Sep 2015 01:37:53 +0000 Subject: read-only NNTP server Implementing NEWNEWS, XHDR, XOVER efficiently will require additional caching on top of msgmap. This seems to work with lynx and slrnpull, haven't tried clients. DO NOT run in production, yet, denial-of-service vulnerabilities await! --- lib/PublicInbox/Msgmap.pm | 74 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 16 deletions(-) (limited to 'lib/PublicInbox/Msgmap.pm') diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index a1748af9..c0fc636f 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -4,7 +4,7 @@ package PublicInbox::Msgmap; use strict; use warnings; -use fields qw(dbh mid_insert mid_for num_for); +use fields qw(dbh mid_insert mid_for num_for num_minmax); use DBI; use DBD::SQLite; @@ -23,43 +23,55 @@ sub new { sqlite_use_immediate_transaction => 1, }); $dbh->do('PRAGMA case_sensitive_like = ON'); - - $writable and create_tables($dbh); my $self = fields::new($class); $self->{dbh} = $dbh; + + if ($writable) { + create_tables($dbh); + $self->created_at(time) unless $self->created_at; + } $self; } -# accessor -sub last_commit { - my ($self, $commit) = @_; - my $dbh = $self->{dbh}; - my $prev; +sub meta_accessor { + my ($self, $key, $value) = @_; use constant { - key => 'last_commit', meta_select => 'SELECT val FROM meta WHERE key = ? LIMIT 1', meta_update => 'UPDATE meta SET val = ? WHERE key = ? LIMIT 1', meta_insert => 'INSERT INTO meta (key,val) VALUES (?,?)', }; - defined $commit or - return $dbh->selectrow_array(meta_select, undef, key); + my $dbh = $self->{dbh}; + my $prev; + defined $value or + return $dbh->selectrow_array(meta_select, undef, $key); $dbh->begin_work; eval { - $prev = $dbh->selectrow_array(meta_select, undef, key); + $prev = $dbh->selectrow_array(meta_select, undef, $key); if (defined $prev) { - $dbh->do(meta_update, undef, $commit, key); + $dbh->do(meta_update, undef, $value, $key); } else { - $dbh->do(meta_insert, undef, key, $commit); + $dbh->do(meta_insert, undef, $key, $value); } $dbh->commit; }; - return $prev unless $@; + my $err = $@; + return $prev unless $err; $dbh->rollback; - die $@; + die $err; +} + +sub last_commit { + my ($self, $commit) = @_; + $self->meta_accessor('last_commit', $commit); +} + +sub created_at { + my ($self, $second) = @_; + $self->meta_accessor('created_at', $second); } sub mid_insert { @@ -92,6 +104,15 @@ sub num_for { $sth->fetchrow_array; } +sub minmax { + my ($self) = @_; + my $dbh = $self->{dbh}; + use constant NUM_MINMAX => 'SELECT MIN(num),MAX(num) FROM msgmap'; + my $sth = $self->{num_minmax} ||= $dbh->prepare(NUM_MINMAX); + $sth->execute; + $sth->fetchrow_array; +} + sub mid_prefixes { my ($self, $pfx, $limit) = @_; @@ -134,4 +155,25 @@ sub create_tables { 'val VARCHAR(255) NOT NULL)'); } +sub each_id_batch { + my ($self, $cb) = @_; + my $dbh = $self->{dbh}; + my $n = 0; + my $total = 0; + my $nr; + my $sth = $dbh->prepare('SELECT num FROM msgmap WHERE num > ? '. + 'ORDER BY num ASC LIMIT 1000'); + while (1) { + $sth->execute($n); + my $ary = $sth->fetchall_arrayref; + @$ary = map { $_->[0] } @$ary; + $nr = scalar @$ary; + last if $nr == 0; + $total += $nr; + $n = $ary->[-1]; + $cb->($ary); + } + $total; +} + 1; -- cgit v1.2.3-24-ge0c7