diff options
-rw-r--r-- | MANIFEST | 4 | ||||
-rw-r--r-- | lib/PublicInbox/Git.pm | 16 | ||||
-rw-r--r-- | lib/PublicInbox/SolverGit.pm | 400 | ||||
-rw-r--r-- | t/solve/0001-simple-mod.patch | 20 | ||||
-rw-r--r-- | t/solve/0002-rename-with-modifications.patch | 37 | ||||
-rw-r--r-- | t/solver_git.t | 91 |
6 files changed, 568 insertions, 0 deletions
@@ -101,6 +101,7 @@ lib/PublicInbox/SearchIdxPart.pm lib/PublicInbox/SearchMsg.pm lib/PublicInbox/SearchThread.pm lib/PublicInbox/SearchView.pm +lib/PublicInbox/SolverGit.pm lib/PublicInbox/Spamcheck.pm lib/PublicInbox/Spamcheck/Spamc.pm lib/PublicInbox/Spawn.pm @@ -201,6 +202,9 @@ t/qspawn.t t/reply.t t/search-thr-index.t t/search.t +t/solve/0001-simple-mod.patch +t/solve/0002-rename-with-modifications.patch +t/solver_git.t t/spamcheck_spamc.t t/spawn.t t/thread-cycle.t diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm index 90b9214a..9676086f 100644 --- a/lib/PublicInbox/Git.pm +++ b/lib/PublicInbox/Git.pm @@ -40,6 +40,7 @@ sub new { my ($class, $git_dir) = @_; my @st; $st[7] = $st[10] = 0; + # may contain {-wt} field (working-tree (File::Temp::Dir)) bless { git_dir => $git_dir, st => \@st }, $class } @@ -201,6 +202,21 @@ sub packed_bytes { sub DESTROY { cleanup(@_) } +# show the blob URL for cgit/gitweb/whatever +sub src_blob_url { + my ($self, $oid) = @_; + # blob_fmt = "https://example.com/foo.git/blob/%s" + if (my $bfu = $self->{blob_fmt_url}) { + return sprintf($bfu, $oid); + } + + # don't show full FS path, basename should be OK: + if ($self->{git_dir} =~ m!/([^/]+)\z!) { + return "/path/to/$1"; + } + '???'; +} + 1; __END__ =pod diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm new file mode 100644 index 00000000..f28768a9 --- /dev/null +++ b/lib/PublicInbox/SolverGit.pm @@ -0,0 +1,400 @@ +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> + +# "Solve" blobs which don't exist in git code repositories by +# searching inboxes for post-image blobs. + +# this emits a lot of debugging/tracing information which may be +# publically viewed over HTTP(S). Be careful not to expose +# local filesystem layouts in the process. +package PublicInbox::SolverGit; +use strict; +use warnings; +use File::Temp qw(); +use Fcntl qw(SEEK_SET); +use File::Path qw(make_path); +use PublicInbox::Git qw(git_unquote); +use PublicInbox::Spawn qw(spawn popen_rd); +use PublicInbox::MsgIter qw(msg_iter msg_part_text); +use URI::Escape qw(uri_escape_utf8); + +# don't bother if somebody sends us a patch with these path components, +# it's junk at best, an attack attempt at worse: +my %bad_component = map { $_ => 1 } ('', '.', '..'); + +sub new { + my ($class, $gits, $inboxes) = @_; + bless { + gits => $gits, + inboxes => $inboxes, + }, $class; +} + +# look for existing blobs already in git repos +sub solve_existing ($$) { + my ($self, $want) = @_; + foreach my $git (@{$self->{gits}}) { + my ($oid_full, $type, $size) = $git->check($want->{oid_b}); + if (defined($type) && $type eq 'blob') { + return [ $git, $oid_full, $type, int($size) ]; + } + } + undef; +} + +# returns a hashref with information about a diff: +# { +# oid_a => abbreviated pre-image oid, +# oid_b => abbreviated post-image oid, +# tmp => anonymous file handle with the diff, +# hdr_lines => arrayref of various header lines for mode information +# mode_a => original mode of oid_a (string, not integer), +# ibx => PublicInbox::Inbox object containing the diff +# smsg => PublicInbox::SearchMsg object containing diff +# path_a => pre-image path +# path_b => post-image path +# } +sub extract_diff ($$$$) { + my ($p, $re, $ibx, $smsg) = @_; + my ($part) = @$p; # ignore $depth and @idx; + my $hdr_lines; # diff --git a/... b/... + my $tmp; + my $ct = $part->content_type || 'text/plain'; + my ($s, undef) = msg_part_text($part, $ct); + defined $s or return; + my $di = {}; + foreach my $l (split(/^/m, $s)) { + if ($l =~ /$re/) { + $di->{oid_a} = $1; + $di->{oid_b} = $2; + my $mode_a = $3; + if ($mode_a =~ /\A(?:100644|120000|100755)\z/) { + $di->{mode_a} = $mode_a; + } + + # start writing the diff out to a tempfile + open($tmp, '+>', undef) or die "open(tmp): $!"; + $di->{tmp} = $tmp; + $di->{hdr_lines} = $hdr_lines; + + print $tmp @$hdr_lines, $l or die "print(tmp): $!"; + + # for debugging/diagnostics: + $di->{ibx} = $ibx; + $di->{smsg} = $smsg; + } elsif ($l =~ m!\Adiff --git ("?a/.+) ("?b/.+)$!) { + return $di if $tmp; # got our blob, done! + + my ($path_a, $path_b) = ($1, $2); + + # don't care for leading 'a/' and 'b/' + my (undef, @a) = split(m{/}, git_unquote($path_a)); + my (undef, @b) = split(m{/}, git_unquote($path_b)); + + # get rid of path-traversal attempts and junk patches: + foreach (@a, @b) { + return if $bad_component{$_}; + } + + $di->{path_a} = join('/', @a); + $di->{path_b} = join('/', @b); + $hdr_lines = [ $l ]; + } elsif ($tmp) { + print $tmp $l or die "print(tmp): $!"; + } elsif ($hdr_lines) { + push @$hdr_lines, $l; + } + } + $tmp ? $di : undef; +} + +sub path_searchable ($) { defined($_[0]) && $_[0] =~ m!\A[\w/\. \-]+\z! } + +sub find_extract_diff ($$$) { + my ($self, $ibx, $want) = @_; + my $srch = $ibx->search or return; + + my $post = $want->{oid_b} or die 'BUG: no {oid_b}'; + $post =~ /\A[a-f0-9]+\z/ or die "BUG: oid_b not hex: $post"; + + my $q = "dfpost:$post"; + my $pre = $want->{oid_a}; + if (defined $pre && $pre =~ /\A[a-f0-9]+\z/) { + $q .= " dfpre:$pre"; + } else { + $pre = '[a-f0-9]{7}'; # for $re below + } + + my $path_b = $want->{path_b}; + if (path_searchable($path_b)) { + $q .= qq{ dfn:"$path_b"}; + + my $path_a = $want->{path_a}; + if (path_searchable($path_a) && $path_a ne $path_b) { + $q .= qq{ dfn:"$path_a"}; + } + } + + my $msgs = $srch->query($q, { relevance => 1 }); + my $re = qr/\Aindex ($pre[a-f0-9]*)\.\.($post[a-f0-9]*)(?: (\d+))?/; + + my $di; + foreach my $smsg (@$msgs) { + $ibx->smsg_mime($smsg) or next; + msg_iter(delete($smsg->{mime}), sub { + $di ||= extract_diff($_[0], $re, $ibx, $smsg); + }); + return $di if $di; + } +} + +# pure Perl "git init" +sub do_git_init_wt ($) { + my ($self) = @_; + my $wt = File::Temp->newdir('solver.wt-XXXXXXXX', TMPDIR => 1); + my $dir = $wt->dirname; + + foreach (qw(objects/info refs/heads)) { + make_path("$dir/.git/$_") or die "make_path $_: $!"; + } + open my $fh, '>', "$dir/.git/config" or die "open .git/config: $!"; + print $fh <<'EOF' or die "print .git/config $!"; +[core] + repositoryFormatVersion = 0 + filemode = true + bare = false + fsyncObjectfiles = false + logAllRefUpdates = false +EOF + close $fh or die "close .git/config: $!"; + + open $fh, '>', "$dir/.git/HEAD" or die "open .git/HEAD: $!"; + print $fh "ref: refs/heads/master\n" or die "print .git/HEAD: $!"; + close $fh or die "close .git/HEAD: $!"; + + my $f = '.git/objects/info/alternates'; + open $fh, '>', "$dir/$f" or die "open: $f: $!"; + foreach my $git (@{$self->{gits}}) { + print $fh "$git->{git_dir}/objects\n" or die "print $f: $!"; + } + close $fh or die "close: $f: $!"; + $wt; +} + +sub extract_old_mode ($) { + my ($di) = @_; + if (grep(/\Aold mode (100644|100755|120000)$/, @{$di->{hdr_lines}})) { + return $1; + } + '100644'; +} + +sub reap ($$) { + my ($pid, $msg) = @_; + waitpid($pid, 0) == $pid or die "waitpid($msg): $!"; + $? == 0 or die "$msg failed: $?"; +} + +sub prepare_wt ($$$) { + my ($wt_dir, $existing, $di) = @_; + my $oid_full = $existing->[1]; + my ($r, $w); + my $path_a = $di->{path_a} or die "BUG: path_a missing for $oid_full"; + my $mode_a = $di->{mode_a} || extract_old_mode($di); + my @git = (qw(git -C), $wt_dir); + + pipe($r, $w) or die "pipe: $!"; + my $rdr = { 0 => fileno($r) }; + my $pid = spawn([@git, qw(update-index -z --index-info)], {}, $rdr); + close $r or die "close pipe(r): $!"; + print $w "$mode_a $oid_full\t$path_a\0" or die "print update-index: $!"; + close $w or die "close update-index: $!"; + reap($pid, 'update-index -z --index-info'); + + $pid = spawn([@git, qw(checkout-index -a -f -u)]); + reap($pid, 'checkout-index -a -f -u'); +} + +sub do_apply ($$$$) { + my ($out, $wt_git, $wt_dir, $di) = @_; + + my $tmp = delete $di->{tmp} or die "BUG: no tmp ", di_info($di); + $tmp->flush or die "tmp->flush failed: $!"; + $out->flush or die "err->flush failed: $!"; + sysseek($tmp, 0, SEEK_SET) or die "sysseek(tmp) failed: $!"; + + defined(my $err_fd = fileno($out)) or die "fileno(out): $!"; + my $rdr = { 0 => fileno($tmp), 1 => $err_fd, 2 => $err_fd }; + my $cmd = [ qw(git -C), $wt_dir, + qw(apply --whitespace=warn -3 --verbose) ]; + reap(spawn($cmd, undef, $rdr), 'apply'); + + local $/ = "\0"; + my $rd = popen_rd([qw(git -C), $wt_dir, qw(ls-files -s -z)]); + + defined(my $line = <$rd>) or die "failed to read ls-files: $!"; + chomp $line or die "no trailing \\0 in [$line] from ls-files"; + + my ($info, $file) = split(/\t/, $line, 2); + my ($mode_b, $oid_b_full, $stage) = split(/ /, $info); + + defined($line = <$rd>) and die "extra files in index: $line"; + close $rd or die "close ls-files: $?"; + + $file eq $di->{path_b} or + die "index mismatch: file=$file != path_b=$di->{path_b}"; + my $abs_path = "$wt_dir/$file"; + -r $abs_path or die "WT_DIR/$file not readable"; + my $size = -s _; + + print $out "OK $mode_b $oid_b_full $stage\t$file\n"; + [ $wt_git, $oid_b_full, 'blob', $size, $di ]; +} + +sub di_url ($) { + my ($di) = @_; + # note: we don't pass the PSGI env here, different inboxes + # can have different HTTP_HOST on the same instance. + my $url = $di->{ibx}->base_url; + my $mid = $di->{smsg}->{mid}; + defined($url) ? "<$url/$mid/>" : "<$mid>"; +} + +sub apply_patches ($$$$$) { + my ($self, $out, $wt, $found, $patches) = @_; + my $wt_dir = $wt->dirname; + my $wt_git = PublicInbox::Git->new("$wt_dir/.git"); + $wt_git->{-wt} = $wt; + + my $cur = 0; + my $tot = scalar @$patches; + + foreach my $di (@$patches) { + my $i = ++$cur; + my $oid_a = $di->{oid_a}; + my $existing = $found->{$oid_a}; + my $empty_oid = $oid_a =~ /\A0+\z/; + + if ($empty_oid && $i != 0) { + die "empty oid at [$i/$tot] ", di_url($di); + } + if (!$existing && !$empty_oid) { + die "missing $oid_a at [$i/$tot] ", di_url($di); + } + + # prepare the worktree for patch application: + if ($i == 1 && $existing) { + prepare_wt($wt_dir, $existing, $di); + } + unless (-f "$wt_dir/$di->{path_a}") { + die "missing $di->{path_a} at [$i/$tot] ", di_url($di); + } + + print $out "applying [$i/$tot] ", di_url($di), "\n", + join('', @{$di->{hdr_lines}}), "\n" + or die "print \$out failed: $!"; + + # apply the patch! + $found->{$di->{oid_b}} = do_apply($out, $wt_git, $wt_dir, $di); + } +} + +sub dump_found ($$) { + my ($out, $found) = @_; + foreach my $oid (sort keys %$found) { + my ($git, $oid, $di) = @{$found->{$oid}}; + my $loc = $di ? di_info($di) : $git->src_blob_url($oid); + print $out "$oid from $loc\n"; + } +} + +sub dump_patches ($$) { + my ($out, $patches) = @_; + my $tot = scalar(@$patches); + my $i = 0; + foreach my $di (@$patches) { + ++$i; + print $out "[$i/$tot] ", di_url($di), "\n"; + } +} + +# recreate $oid_b +# Returns a 2-element array ref: [ PublicInbox::Git object, oid_full ] +# or undef if nothing was found. +sub solve ($$$$) { + my ($self, $out, $oid_b, $hints) = @_; + + # should we even get here? Probably not, but somebody + # could be manually typing URLs: + return if $oid_b =~ /\A0+\z/; + + my $req = { %$hints, oid_b => $oid_b }; + my @todo = ($req); + my $found = {}; # { oid_abbrev => [ PublicInbox::Git, oid_full, $di ] } + my $patches = []; # [ array of $di hashes ] + + my $max = $self->{max_steps} || 200; + my $steps = 0; + + while (defined(my $want = pop @todo)) { + # see if we can find the blob in an existing git repo: + if (my $existing = solve_existing($self, $want)) { + my $want_oid = $want->{oid_b}; + return $existing if $want_oid eq $oid_b; # DONE! + + $found->{$want_oid} = $existing; + next; # ok, one blob resolved, more to go? + } + + # scan through inboxes to look for emails which results in + # the oid we want: + foreach my $ibx (@{$self->{inboxes}}) { + my $di = find_extract_diff($self, $ibx, $want) or next; + + unshift @$patches, $di; + + # good, we can find a path to the oid we $want, now + # lets see if we need to apply more patches: + my $src = $di->{oid_a}; + if ($src !~ /\A0+\z/) { + if (++$steps > $max) { + print $out +"Aborting, too many steps to $oid_b\n"; + + return; + } + + # we have to solve it using another oid, fine: + my $job = { + oid_b => $src, + path_b => $di->{path_a}, + }; + push @todo, $job; + } + last; # onto the next @todo item + } + } + + unless (scalar(@$patches)) { + print $out "no patch(es) for $oid_b\n"; + dump_found($out, $found); + return; + } + + # reconstruct the oid_b blob using patches we found: + eval { + my $wt = do_git_init_wt($self); + apply_patches($self, $out, $wt, $found, $patches); + }; + if ($@) { + print $out "E: $@\nfound: "; + dump_found($out, $found); + print $out "patches: "; + dump_patches($out, $patches); + return; + } + + $found->{$oid_b}; +} + +1; diff --git a/t/solve/0001-simple-mod.patch b/t/solve/0001-simple-mod.patch new file mode 100644 index 00000000..c6bb1575 --- /dev/null +++ b/t/solve/0001-simple-mod.patch @@ -0,0 +1,20 @@ +From: WEB DESIGN EXPERT <BOFH@YHBT.net> +To: meta@public-inbox.org +Subject: [PATCH] TODO: take expert web design advice +Date: Mon, 1 Apr 2019 08:15:20 +0000 +Message-Id: <20190401081523.16213-1-BOFH@YHBT.net> + +--- + TODO | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/TODO b/TODO +index 605013e..69df7d5 100644 +--- a/TODO ++++ b/TODO +@@ -109,3 +109,5 @@ all need to be considered for everything we introduce) + + * Optional history squashing to reduce commit and intermediate + tree objects ++ ++ * Make use of <blink> and <marquee> tags diff --git a/t/solve/0002-rename-with-modifications.patch b/t/solve/0002-rename-with-modifications.patch new file mode 100644 index 00000000..aa415e01 --- /dev/null +++ b/t/solve/0002-rename-with-modifications.patch @@ -0,0 +1,37 @@ +From: POLITICAL CORRECTNESS EXPERT <BOFH@YHBT.net> +To: meta@public-inbox.org +Subject: [PATCH] POLITICALLY CORRECT FILE NAMING +Date: Mon, 1 Apr 2019 08:15:20 +0000 +Message-Id: <20190401081523.16213-2-BOFH@YHBT.net> + +HACKING MIGHT GET US REPORTED TO EFF-BEE-EYE +AND USE MARKDOWN CUZ MOAR FLAVORS == BETTER +--- + HACKING => CONTRIBUTING.md | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + rename HACKING => CONTRIBUTING.md (94%) + +diff --git a/HACKING b/CONTRIBUTING.md +similarity index 94% +rename from HACKING +rename to CONTRIBUTING.md +index 3435775..0a92431 100644 +--- a/HACKING ++++ b/CONTRIBUTING.md +@@ -1,5 +1,5 @@ +-hacking public-inbox +--------------------- ++contributing to public-inbox ++---------------------------- + + Send all patches and "git request-pull"-formatted emails to our + self-hosting inbox at meta@public-inbox.org +@@ -15,7 +15,7 @@ Please consider our goals in mind: + Decentralization, Accessibility, Compatibility, Performance + + These goals apply to everyone: users viewing over the web or NNTP, +-sysadmins running public-inbox, and other hackers working public-inbox. ++sysadmins running public-inbox, and other contributors working public-inbox. + + We will reject any feature which advocates or contributes to any + particular instance of a public-inbox becoming a single point of failure. diff --git a/t/solver_git.t b/t/solver_git.t new file mode 100644 index 00000000..fe322eab --- /dev/null +++ b/t/solver_git.t @@ -0,0 +1,91 @@ +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use File::Temp qw(tempdir); +use Cwd qw(abs_path); +require './t/common.perl'; +require_git(2.6); + +my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test + URI::Escape Plack::Builder); +foreach my $mod (@mods) { + eval "require $mod"; + plan skip_all => "$mod missing for $0" if $@; +} +chomp(my $git_dir = `git rev-parse --git-dir 2>/dev/null`); +plan skip_all => "$0 must be run from a git working tree" if $?; +$git_dir = abs_path($git_dir); + +use_ok "PublicInbox::$_" for (qw(Inbox V2Writable MIME Git SolverGit)); + +my $mainrepo = tempdir('pi-solver-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $opts = { + mainrepo => $mainrepo, + name => 'test-v2writable', + version => 2, + -primary_address => 'test@example.com', +}; +my $ibx = PublicInbox::Inbox->new($opts); +my $im = PublicInbox::V2Writable->new($ibx, 1); +$im->{parallel} = 0; + +sub deliver_patch ($) { + open my $fh, '<', $_[0] or die "open: $!"; + my $mime = PublicInbox::MIME->new(do { local $/; <$fh> }); + $im->add($mime); + $im->done; +} + +deliver_patch('t/solve/0001-simple-mod.patch'); + +my $gits = [ PublicInbox::Git->new($git_dir) ]; +my $solver = PublicInbox::SolverGit->new($gits, [ $ibx ]); +open my $log, '+>>', "$mainrepo/solve.log" or die "open: $!"; +my $res = $solver->solve($log, '69df7d5', {}); +ok($res, 'solved a blob!'); +my $wt_git = $res->[0]; +is(ref($wt_git), 'PublicInbox::Git', 'got a git object for the blob'); +my $expect = '69df7d565d49fbaaeb0a067910f03dc22cd52bd0'; +is($res->[1], $expect, 'resolved blob to unabbreviated identifier'); +is($res->[2], 'blob', 'type specified'); +is($res->[3], 4405, 'size returned'); + +is(ref($wt_git->cat_file($res->[1])), 'SCALAR', 'wt cat-file works'); +is_deeply([$expect, 'blob', 4405], + [$wt_git->check($res->[1])], 'wt check works'); + +if (0) { # TODO: check this? + seek($log, 0, 0); + my $z = do { local $/; <$log> }; + diag $z; +} + +$res = undef; +my $wt_git_dir = $wt_git->{git_dir}; +$wt_git = undef; +ok(!-d $wt_git_dir, 'no references to WT held'); + +$res = $solver->solve($log, '0'x40, {}); +is($res, undef, 'no error on z40'); + +my $git_v2_20_1_tag = '7a95a1cd084cb665c5c2586a415e42df0213af74'; +$res = $solver->solve($log, $git_v2_20_1_tag, {}); +is($res, undef, 'no error on a tag not in our repo'); + +deliver_patch('t/solve/0002-rename-with-modifications.patch'); +$res = $solver->solve($log, '0a92431', {}); +ok($res, 'resolved without hints'); + +my $hints = { + oid_a => '3435775', + path_a => 'HACKING', + path_b => 'CONTRIBUTING' +}; +my $hinted = $solver->solve($log, '0a92431', $hints); +# don't compare ::Git objects: +shift @$res; shift @$hinted; +is_deeply($res, $hinted, 'hints work (or did not hurt :P'); + +done_testing(); |