user/dev discussion of public-inbox itself
 help / color / mirror / Atom feed
27533546e4c4a320bb9cf3bad3deeb79e3b7228d blob 6212 bytes (raw)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# Copyright (C) 2019-2020 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use warnings;
use Test::More;
use PublicInbox::Eml;
use PublicInbox::Inbox;
use PublicInbox::InboxWritable;
use PublicInbox::Spawn qw(which);
require PublicInbox::Admin;
use PublicInbox::TestCommon;
my $PI_TEST_VERSION = $ENV{PI_TEST_VERSION} || 2;
require_git('2.6') if $PI_TEST_VERSION == 2;
require_mods(qw(DBD::SQLite));
my $have_xapian_compact = which($ENV{XAPIAN_COMPACT} || 'xapian-compact');

my $mime = PublicInbox::Eml->new(<<'EOF');
From: a@example.com
To: test@example.com
Subject: this is a subject
Date: Fri, 02 Oct 1993 00:00:00 +0000

hello world
EOF

my $import_index_incremental = sub {
	my ($v, $level, $mime) = @_;
	my $err = '';
	my $this = "pi-$v-$level-indexlevels";
	my ($tmpdir, $for_destroy) = tmpdir();
	local $ENV{PI_CONFIG} = "$tmpdir/config";
	my $ibx = PublicInbox::Inbox->new({
		inboxdir => "$tmpdir/testbox",
		name => $this,
		version => $v,
		-primary_address => 'test@example.com',
		indexlevel => $level,
	});
	my $im = PublicInbox::InboxWritable->new($ibx, {nproc=>1})->importer(0);
	$mime->header_set('Message-ID', '<m@1>');
	ok($im->add($mime), 'first message added');
	$im->done;

	# index master (required for v1)
	my @cmd = (qw(-index -j0), $ibx->{inboxdir}, "-L$level");
	push @cmd, '-c' if $have_xapian_compact;
	ok(run_script(\@cmd, undef, { 2 => \$err }), 'index master');
	my $ro_master = PublicInbox::Inbox->new({
		inboxdir => $ibx->{inboxdir},
		indexlevel => $level
	});
	my $msgs = $ro_master->recent;
	is(scalar(@$msgs), 1, 'only one message in master, so far');
	is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');

	# clone
	@cmd = (qw(git clone --mirror -q));
	my $mirror = "$tmpdir/mirror-$v";
	if ($v == 1) {
		push @cmd, $ibx->{inboxdir}, $mirror;
	} else {
		push @cmd, "$ibx->{inboxdir}/git/0.git", "$mirror/git/0.git";
	}
	my $fetch_dir = $cmd[-1];
	is(xsys(@cmd), 0, "v$v clone OK");

	# inbox init
	local $ENV{PI_CONFIG} = "$tmpdir/.picfg";
	@cmd = ('-init', '-L', $level,
		'mirror', $mirror, '//example.com/test', 'test@example.com');
	push @cmd, '-V2' if $v == 2;
	ok(run_script(\@cmd), "v$v init OK");

	# index mirror
	ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror OK");

	# read-only access
	my $ro_mirror = PublicInbox::Inbox->new({
		inboxdir => $mirror,
		indexlevel => $level,
	});
	$msgs = $ro_mirror->recent;
	is(scalar(@$msgs), 1, 'only one message, so far');
	is($msgs->[0]->{mid}, 'm@1', 'read first message');

	# update master
	$mime->header_set('Message-ID', '<m@2>');
	ok($im->add($mime), '2nd message added');
	$im->done;

	# mirror updates
	is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
	ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror again OK");
	$msgs = $ro_mirror->recent;
	is(scalar(@$msgs), 2, '2nd message seen in mirror');
	is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
		['m@1','m@2'], 'got both messages in mirror');

	# incremental index master (required for v1)
	ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]),
		'index master OK');
	$msgs = $ro_master->recent;
	is(scalar(@$msgs), 2, '2nd message seen in master');
	is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
		['m@1','m@2'], 'got both messages in master');

	my @rw_nums = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};
	is_deeply(\@rw_nums, [1, 2], 'master has expected NNTP articles');

	my @ro_nums = map { $_->{num} } @{$ro_mirror->over->query_ts(0, 0)};
	is_deeply(\@ro_nums, [1, 2], 'mirror has expected NNTP articles');

	# remove message from master
	ok($im->remove($mime), '2nd message removed');
	$im->done;
	@rw_nums = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};
	is_deeply(\@rw_nums, [1], 'unindex NNTP article'.$v.$level);

	if ($level ne 'basic') {
		ok(run_script(['-xcpdb', '-q', $mirror]), "v$v xcpdb OK");
		is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
		   'indexlevel detectable by Admin after xcpdb v' .$v.$level);
		delete $ro_mirror->{$_} for (qw(over search));
		$msgs = $ro_mirror->search->query('m:m@2');
		is(scalar(@$msgs), 1, "v$v found m\@2 via Xapian on $level");
	}

	# sync the mirror
	is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
	ok(run_script([qw(-index -j0), $mirror]), "v$v index mirror again OK");
	$msgs = $ro_mirror->recent;
	is(scalar(@$msgs), 1, '2nd message gone from mirror');
	is_deeply([map { $_->{mid} } @$msgs], ['m@1'],
		'message unavailable in mirror');

	if ($v == 2 && $level eq 'basic') {
		is_deeply([glob("$ibx->{inboxdir}/xap*/?/")], [],
			 'no Xapian shard directories for v2 basic');
	}
	if ($level ne 'basic') {
		$msgs = $ro_mirror->search->reopen->query('m:m@2');
		is(scalar(@$msgs), 0,
			"v$v m\@2 gone from Xapian in mirror on $level");
	}

	# add another message to master and have the mirror
	# sync and reindex it
	my @expect = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};
	foreach my $i (3..5) {
		$mime->header_set('Message-ID', "<m\@$i>");
		ok($im->add($mime), "#$i message added");
		push @expect, $i;
	}
	$im->done;
	is(xsys('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
	ok(run_script([qw(-index -j0 --reindex), $mirror]),
		"v$v index --reindex mirror OK");
	@ro_nums = map { $_->{num} } @{$ro_mirror->over->query_ts(0, 0)};
	@rw_nums = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};
	is_deeply(\@rw_nums, \@expect, "v$v master has expected NNTP articles");
	is_deeply(\@ro_nums, \@expect, "v$v mirror matches master articles");

	is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
	   'indexlevel detectable by Admin '.$v.$level);

	SKIP: {
		skip 'xapian-compact missing', 1 if !$have_xapian_compact;
		my $cmd = [ qw(-compact), $mirror ];
		ok(run_script($cmd, undef, { 2 => \$err}), "compact $level");
	}
};

# we can probably cull some other tests
$import_index_incremental->($PI_TEST_VERSION, 'basic', $mime);

SKIP: {
	require PublicInbox::Search;
	PublicInbox::Search::load_xapian() or
		skip('Xapian perl binding missing', 2);
	foreach my $l (qw(medium full)) {
		$import_index_incremental->($PI_TEST_VERSION, $l, $mime);
	}
}

done_testing();
debug log:

solving 27533546 ...
found 27533546 in https://80x24.org/public-inbox.git

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for the project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git