user/dev discussion of public-inbox itself
 help / color / Atom feed
61053b669b75efda2d045f51e87c522e78eed25c blob 4653 bytes (raw)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# Copyright (C) 2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
use warnings;
use Test::More;
use PublicInbox::MIME;
use PublicInbox::Inbox;
use PublicInbox::InboxWritable;
use File::Temp qw/tempdir/;
require './t/common.perl';
require_git(2.6);
my $this = (split('/', __FILE__))[-1];

foreach my $mod (qw(DBD::SQLite)) {
	eval "require $mod";
	plan skip_all => "$mod missing for $this" if $@;
}

my $path = 'blib/script';
my $index = "$path/public-inbox-index";
my $xcpdb = "$path/public-inbox-xcpdb";

my $mime = PublicInbox::MIME->create(
	header => [
		From => 'a@example.com',
		To => 'test@example.com',
		Subject => 'this is a subject',
		Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
	],
	body => "hello world\n",
);

sub import_index_incremental {
	my ($v, $level) = @_;
	my $tmpdir = tempdir("pi-$this-tmp-XXXXXX", TMPDIR => 1, CLEANUP => 1);
	my $ibx = PublicInbox::Inbox->new({
		mainrepo => "$tmpdir/testbox",
		name => "$this-$v",
		version => $v,
		-primary_address => 'test@example.com',
		indexlevel => $level,
	});
	my $im = PublicInbox::InboxWritable->new($ibx, {nproc=>1})->importer;
	$mime->header_set('Message-ID', '<m@1>');
	ok($im->add($mime), 'first message added');
	$im->done;

	# index master (required for v1)
	is(system($index, $ibx->{mainrepo}, "-L$level"), 0, 'index master OK');
	my $ro_master = PublicInbox::Inbox->new({
		mainrepo => $ibx->{mainrepo},
		indexlevel => $level
	});
	my ($nr, $msgs) = $ro_master->recent;
	is($nr, 1, 'only one message in master, so far');
	is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');

	# clone
	my @cmd = (qw(git clone --mirror -q));
	my $mirror = "$tmpdir/mirror-$v";
	if ($v == 1) {
		push @cmd, $ibx->{mainrepo}, $mirror;
	} else {
		push @cmd, "$ibx->{mainrepo}/git/0.git", "$mirror/git/0.git";
	}
	my $fetch_dir = $cmd[-1];
	is(system(@cmd), 0, "v$v clone OK");

	# inbox init
	local $ENV{PI_CONFIG} = "$tmpdir/.picfg";
	@cmd = ("$path/public-inbox-init", '-L', $level,
		'mirror', $mirror, '//example.com/test', 'test@example.com');
	push @cmd, '-V2' if $v == 2;
	is(system(@cmd), 0, "v$v init OK");

	# index mirror
	is(system($index, $mirror), 0, "v$v index mirror OK");

	# read-only access
	my $ro_mirror = PublicInbox::Inbox->new({
		mainrepo => $mirror,
		indexlevel => 'basic'
	});
	($nr, $msgs) = $ro_mirror->recent;
	is($nr, 1, 'only one message, so far');
	is($msgs->[0]->{mid}, 'm@1', 'read first message');

	# update master
	$mime->header_set('Message-ID', '<m@2>');
	ok($im->add($mime), '2nd message added');
	$im->done;

	# mirror updates
	is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
	is(system($index, $mirror), 0, "v$v index mirror again OK");
	($nr, $msgs) = $ro_mirror->recent;
	is($nr, 2, '2nd message seen in mirror');
	is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
		['m@1','m@2'], 'got both messages in mirror');

	# incremental index master (required for v1)
	is(system($index, $ibx->{mainrepo}, "-L$level"), 0, 'index master OK');
	($nr, $msgs) = $ro_master->recent;
	is($nr, 2, '2nd message seen in master');
	is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
		['m@1','m@2'], 'got both messages in master');

	# remove message from master
	ok($im->remove($mime), '2nd message removed');
	$im->done;

	if ($level ne 'basic') {
		is(system($xcpdb, $mirror), 0, "v$v xcpdb OK");
		delete $ro_mirror->{$_} for (qw(over search));
		($nr, $msgs) = $ro_mirror->search->query('m:m@2');
		is($nr, 1, "v$v found m\@2 via Xapian on $level");
	}

	# sync the mirror
	is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
	is(system($index, $mirror), 0, "v$v index mirror again OK");
	($nr, $msgs) = $ro_mirror->recent;
	is($nr, 1, '2nd message gone from mirror');
	is_deeply([map { $_->{mid} } @$msgs], ['m@1'],
		'message unavailable in mirror');

	if ($v == 2 && $level eq 'basic') {
		is_deeply([glob("$ibx->{mainrepo}/xap*/?/")], [],
			 'no Xapian partition directories for v2 basic');
	}
	if ($level ne 'basic') {
		($nr, $msgs) = $ro_mirror->search->reopen->query('m:m@2');
		is($nr, 0, "v$v m\@2 gone from Xapian in mirror on $level");
	}
}

# we can probably cull some other tests and put full/medium tests, here
for my $level (qw(basic)) {
	for my $v (1..2) {
		subtest("v$v indexlevel=$level" => sub {
			import_index_incremental($v, $level);
		})
	}
}

SKIP: {
	require PublicInbox::Search;
	PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2;
	for my $v (1..2) {
		subtest("v$v indexlevel=medium" => sub {
			import_index_incremental($v, 'medium');
		})
	}
}

done_testing();
debug log:

solving 61053b6 ...
found 61053b6 in https://80x24.org/public-inbox.git

user/dev discussion of public-inbox itself

Archives are clonable:
	git clone --mirror http://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

Example config snippet for mirrors

Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.org/gmane.mail.public-inbox.general

 note: .onion URLs require Tor: https://www.torproject.org/

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git