user/dev discussion of public-inbox itself
 help / color / Atom feed
75380f0f5f229214a3579d81a011c78b0a3c76dc blob 5278 bytes (raw)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# Copyright (C) 2018 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use warnings;
use Test::More;
use PublicInbox::MIME;
use PublicInbox::ContentId qw(content_digest);
use File::Temp qw/tempdir/;
use File::Path qw(remove_tree);

foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
	eval "require $mod";
	plan skip_all => "$mod missing for v1reindex.t" if $@;
}
use_ok 'PublicInbox::SearchIdx';
use_ok 'PublicInbox::Import';
my $mainrepo = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
is(system(qw(git init -q --bare), $mainrepo), 0);
my $ibx_config = {
	mainrepo => $mainrepo,
	name => 'test-v1reindex',
	-primary_address => 'test@example.com',
	indexlevel => 'full',
};
my $ibx = PublicInbox::Inbox->new($ibx_config);
my $mime = PublicInbox::MIME->create(
	header => [
		From => 'a@example.com',
		To => 'test@example.com',
		Subject => 'this is a subject',
		Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
	],
	body => "hello world\n",
);
my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
foreach my $i (1..10) {
	$mime->header_set('Message-Id', "<$i\@example.com>");
	ok($im->add($mime), "message $i added");
	if ($i == 4) {
		$im->remove($mime);
	}
}

if ('test remove later') {
	$mime->header_set('Message-Id', "<5\@example.com>");
	$im->remove($mime);
}

$im->done;
my $rw = PublicInbox::SearchIdx->new($ibx, 1);
eval { $rw->index_sync() };
is($@, '', 'no error from indexing');

my $minmax = [ $ibx->mm->minmax ];
ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
is_deeply($minmax, [ 1, 10 ], 'minmax as expected');

$rw = PublicInbox::SearchIdx->new($ibx, 1);
eval { $rw->index_sync({reindex => 1}) };
is($@, '', 'no error from reindexing');
$im->done;

my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
remove_tree($xap);
ok(!-d $xap, 'Xapian directories removed');
$rw = PublicInbox::SearchIdx->new($ibx, 1);

eval { $rw->index_sync({reindex => 1}) };
is($@, '', 'no error from reindexing');
$im->done;
ok(-d $xap, 'Xapian directories recreated');

delete $ibx->{mm};
is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');

ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
remove_tree($xap);
ok(!-d $xap, 'Xapian directories removed again');

$rw = PublicInbox::SearchIdx->new($ibx, 1);
{
	my @warn;
	local $SIG{__WARN__} = sub { push @warn, @_ };
	eval { $rw->index_sync({reindex => 1}) };
	is($@, '', 'no error from reindexing without msgmap');
	is(scalar(@warn), 0, 'no warnings from reindexing');
	$im->done;
	ok(-d $xap, 'Xapian directories recreated');
	delete $ibx->{mm};
	is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
}

ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
remove_tree($xap);
ok(!-d $xap, 'Xapian directories removed again');

$rw = PublicInbox::SearchIdx->new($ibx, 1);
{
	my @warn;
	local $SIG{__WARN__} = sub { push @warn, @_ };
	eval { $rw->index_sync({reindex => 1}) };
	is($@, '', 'no error from reindexing without msgmap');
	is_deeply(\@warn, [], 'no warnings');
	$im->done;
	ok(-d $xap, 'Xapian directories recreated');
	delete $ibx->{mm};
	is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
}

ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
remove_tree($xap);
ok(!-d $xap, 'Xapian directories removed again');

$ibx_config->{indexlevel} = 'medium';
$ibx = PublicInbox::Inbox->new($ibx_config);
$rw = PublicInbox::SearchIdx->new($ibx, 1);
{
	my @warn;
	local $SIG{__WARN__} = sub { push @warn, @_ };
	eval { $rw->index_sync({reindex => 1}) };
	is($@, '', 'no error from reindexing without msgmap');
	is_deeply(\@warn, [], 'no warnings');
	$im->done;
	ok(-d $xap, 'Xapian directories recreated');
	delete $ibx->{mm};
	is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
	my $mset = $ibx->search->query('hello world', {mset=>1});
	isnt(0, $mset->size, 'got Xapian search results');
}

ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
remove_tree($xap);
ok(!-d $xap, 'Xapian directories removed again');

$ibx_config->{indexlevel} = 'basic';
$ibx = PublicInbox::Inbox->new($ibx_config);
$rw = PublicInbox::SearchIdx->new($ibx, 1);
{
	my @warn;
	local $SIG{__WARN__} = sub { push @warn, @_ };
	eval { $rw->index_sync({reindex => 1}) };
	is($@, '', 'no error from reindexing without msgmap');
	is_deeply(\@warn, [], 'no warnings');
	$im->done;
	ok(-d $xap, 'Xapian directories recreated');
	delete $ibx->{mm};
	is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
	my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
	is(0, $mset->size, "no Xapian search results");
}

# upgrade existing basic to medium
# note: changing indexlevels is not yet supported in v2,
# and may not be without more effort
$ibx_config->{indexlevel} = 'medium';
$ibx = PublicInbox::Inbox->new($ibx_config);
$rw = PublicInbox::SearchIdx->new($ibx, 1);
# no removals
{
	my @warn;
	local $SIG{__WARN__} = sub { push @warn, @_ };
	eval { $rw->index_sync };
	is($@, '', 'no error from indexing');
	is_deeply(\@warn, [], 'no warnings');
	my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
	isnt(0, $mset->size, 'search OK after basic -> medium');
}

done_testing();
debug log:

solving 75380f0f5f22 ...
found 75380f0f5f22 in https://80x24.org/public-inbox.git

user/dev discussion of public-inbox itself

Archives are clonable:
	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

Example config snippet for mirrors

Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.org/gmane.mail.public-inbox.general

 note: .onion URLs require Tor: https://www.torproject.org/

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git