Re: 2.18.0 Regression: packing performance and effectiveness

[Jeff King <peff@peff.net>]

On Thu, Jul 19, 2018 at 09:42:00AM -0700, Elijah Newren wrote:

> Thanks for the quick turnaround.  Unfortunately, I have some bad news.
> With this patch, I get the following:
> 
> $ /usr/bin/time -f 'MaxRSS:%M Time:%e' git gc --aggressive
> Enumerating objects: 4460703, done.
> Counting objects: 100% (4460703/4460703), done.
> Delta compression using up to 40 threads.
> Compressing objects: 100% (3807140/3807140), done.
> Writing objects: 100% (4460703/4460703), done.
> Total 4460703 (delta 2831383), reused 1587071 (delta 0)
> error: failed to unpack compressed delta at offset 183854150 from
> .git/objects/pack/pack-30d4f0b0e5a03dc91a658a0586f4e74cdf4a94d6.pack
> fatal: packed object 20ce811e53dabbb8ef9368c108cbbdfa65639c03 (stored
> in .git/objects/pack/pack-30d4f0b0e5a03dc91a658a0586f4e74cdf4a94d6.pack)
> is corrupt
> error: failed to run prune
> MaxRSS:40025196 Time:2531.52

Looking at that output, my _guess_ is that we somehow end up with a
bogus delta_size value and write out a truncated entry. But I couldn't
reproduce the issue with smaller test cases.

Maybe instrumenting Git with the patch below and running:

  GIT_TRACE_DELTA=$PWD/delta.out git gc --aggressive
  perl -lne '
	/(get|put) ([0-9a-f]{40}) (\d+)/ or next;
	if ($1 eq "put") {
		$h{$2} and print "double put: $2 = ($h{$2}, $3)";
		$h{$2} = $3;
	} else {
		$h{$2} == $3 or print "mismatched get: $2 = ($h{$2}, $3)"
	}
  ' <delta.out

would show some anomalies in the get/set sizes?

---
diff --git a/pack-objects.h b/pack-objects.h
index 978500e474..77a6aae62b 100644
--- a/pack-objects.h
+++ b/pack-objects.h
@@ -327,7 +327,9 @@ static inline void oe_set_size(struct packing_data *pack,
 	}
 }
 
-static inline unsigned long oe_delta_size(struct packing_data *pack,
+static struct trace_key trace_delta = TRACE_KEY_INIT(DELTA);
+
+static inline unsigned long oe_delta_size_1(struct packing_data *pack,
 					  const struct object_entry *e)
 {
 	if (!pack->delta_size)
@@ -335,11 +337,23 @@ static inline unsigned long oe_delta_size(struct packing_data *pack,
 	return pack->delta_size[e - pack->objects];
 }
 
+static inline unsigned long oe_delta_size(struct packing_data *pack,
+					  const struct object_entry *e)
+{
+	unsigned long ret = oe_delta_size_1(pack, e);
+	trace_printf_key(&trace_delta, "get %s %lu",
+			 oid_to_hex(&e->idx.oid), ret);
+	return ret;
+}
+
 void oe_prepare_delta_size_array(struct packing_data *pack);
 static inline void oe_set_delta_size(struct packing_data *pack,
 				     struct object_entry *e,
 				     unsigned long size)
 {
+	trace_printf_key(&trace_delta, "put %s %lu",
+			 oid_to_hex(&e->idx.oid), size);
+
 	e->delta_size_ = size;
 	if (!pack->delta_size && e->delta_size_ == size)
 		return;