user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
a63bfdfd7cea49040f63791a79d3da2d082d01fd blob 3517 bytes (raw)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# Copyright (C) 2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>

# front-end for the "lei import" sub-command
package PublicInbox::LeiImport;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC);
use PublicInbox::MboxReader;
use PublicInbox::Eml;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::PktOp;

sub _import_eml { # MboxReader callback
	my ($eml, $sto, $set_kw) = @_;
	$sto->ipc_do('set_eml', $eml, $set_kw ? $sto->mbox_keywords($eml) : ());
}

sub import_done_wait { # dwaitpid callback
	my ($arg, $pid) = @_;
	my ($imp, $lei) = @$arg;
	$lei->child_error($?, 'non-fatal errors during import') if $?;
	my $ign = $lei->{sto}->ipc_do('done'); # PublicInbox::LeiStore::done
	$lei->dclose;
}

sub import_done { # EOF callback for main daemon
	my ($lei) = @_;
	my $imp = delete $lei->{imp} or return;
	$imp->wq_wait_old(\&import_done_wait, $lei);
}

sub call { # the main "lei import" method
	my ($cls, $lei, @argv) = @_;
	my $sto = $lei->_lei_store(1);
	$sto->write_prepare($lei);
	$lei->{opt}->{kw} //= 1;
	my $fmt = $lei->{opt}->{'format'};
	my $self = $lei->{imp} = bless {}, $cls;
	if (my @f = grep { -f } @argv && !$fmt) {
		return $lei->fail("--format unset for regular files:\n@f");
	}
	$self->{0} = $lei->{0} if $lei->{opt}->{stdin};
	my $ops = {
		'!' => [ $lei->can('fail_handler'), $lei ],
		'x_it' => [ $lei->can('x_it'), $lei ],
		'child_error' => [ $lei->can('child_error'), $lei ],
		'' => [ \&import_done, $lei ],
	};
	($lei->{pkt_op_c}, $lei->{pkt_op_p}) = PublicInbox::PktOp->pair($ops);
	my $j = $lei->{opt}->{jobs} // scalar(@argv) || 1;
	my $nproc = $self->detect_nproc;
	$j = $nproc if $j > $nproc;
	$self->wq_workers_start('lei_import', $j, $lei->oldset, {lei => $lei});
	my $op = delete $lei->{pkt_op_c};
	delete $lei->{pkt_op_p};
	$self->wq_io_do('import_stdin', []) if $self->{0};
	for my $x (@argv) {
		$self->wq_io_do('import_path_url', [], $x);
	}
	$self->wq_close(1);
	$lei->event_step_init; # wait for shutdowns
	if ($lei->{oneshot}) {
		while ($op->{sock}) { $op->event_step }
	}
}

sub ipc_atfork_child {
	my ($self) = @_;
	$self->{lei}->lei_atfork_child;
	$self->SUPER::ipc_atfork_child;
}

sub _import_fh {
	my ($lei, $fh, $x) = @_;
	my $set_kw = $lei->{opt}->{kw};
	my $fmt = $lei->{opt}->{'format'};
	eval {
		if ($fmt eq 'eml') {
			my $buf = do { local $/; <$fh> } //
				return $lei->child_error(1 >> 8, <<"");
error reading $x: $!

			my $eml = PublicInbox::Eml->new(\$buf);
			_import_eml($eml, $lei->{sto}, $set_kw);
		} else { # some mbox
			my $cb = PublicInbox::MboxReader->can($fmt);
			$cb or return $lei->child_error(1 >> 8, <<"");
--format $fmt unsupported for $x

			$cb->(undef, $fh, \&_import_eml, $lei->{sto}, $set_kw);
		}
	};
	$lei->child_error(1 >> 8, "<stdin>: $@") if $@;
}

sub _import_maildir { # maildir_each_file cb
	my ($f, $sto, $set_kw) = @_;
	$sto->ipc_do('set_eml_from_maildir', $f, $set_kw);
}

sub import_path_url {
	my ($self, $x) = @_;
	my $lei = $self->{lei};
	# TODO auto-detect?
	if (-f $x) {
		open my $fh, '<', $x or return $lei->child_error(1 >> 8, <<"");
unable to open $x: $!

		_import_fh($lei, $fh, $x);
	} elsif (-d _ && (-d "$x/cur" || -d "$x/new")) {
		require PublicInbox::LeiToMail;
		PublicInbox::LeiToMail::maildir_each_file($x,
					\&_import_maildir,
					$lei->{sto}, $lei->{opt}->{kw});
	} else {
		$lei->fail("$x unsupported (TODO)");
	}
}

sub import_stdin {
	my ($self) = @_;
	_import_fh($self->{lei}, $self->{0}, '<stdin>');
}

1;
debug log:

solving a63bfdfd ...
found a63bfdfd in public-inbox.git

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git