[PATCH v3 14/14] commit-graph: build graph from starting commits

[Derrick Stolee <stolee@gmail.com>]

Teach git-commit-graph to read commits from stdin when the
--stdin-commits flag is specified. Commits reachable from these
commits are added to the graph. This is a much faster way to construct
the graph than inspecting all packed objects, but is restricted to
known tips.

For the Linux repository, 700,000+ commits were added to the graph
file starting from 'master' in 7-9 seconds, depending on the number
of packfiles in the repo (1, 24, or 120).

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
---
 Documentation/git-commit-graph.txt | 15 ++++++++++++++-
 builtin/commit-graph.c             | 26 +++++++++++++++++++++-----
 commit-graph.c                     | 26 ++++++++++++++++++++++++--
 commit-graph.h                     |  4 +++-
 t/t5318-commit-graph.sh            | 19 +++++++++++++++++++
 5 files changed, 81 insertions(+), 9 deletions(-)

diff --git a/Documentation/git-commit-graph.txt b/Documentation/git-commit-graph.txt
index 727d5d70bb..bd1c54025a 100644
--- a/Documentation/git-commit-graph.txt
+++ b/Documentation/git-commit-graph.txt
@@ -45,7 +45,12 @@ graph-head file if it is updated by the `--update-head` option.
 +
 With the `--stdin-packs` option, generate the new commit graph by
 walking objects only in the specified packfiles and any commits in
-the existing graph-head.
+the existing graph-head. (Cannot be combined with --stdin-commits.)
++
+With the `--stdin-commits` option, generate the new commit graph by
+walking commits starting at the commits specified in stdin as a list
+of OIDs in hex, one OID per line. (Cannot be combined with
+--stdin-packs.)
 
 'read'::
 
@@ -83,6 +88,14 @@ $ git commit-graph write --update-head --delete-expired
 $ echo <pack-index> | git commit-graph write --update-head --delete-expired --stdin-packs
 ------------------------------------------------
 
+* Write a graph file, extending the current graph file using all
+* commits reachable from refs/heads/*, update graph-head, and delete
+* stale graph files.
++
+------------------------------------------------
+$ git show-ref --heads -s | git commit-graph write --update-head --delete-expired --stdin-commits
+------------------------------------------------
+
 * Read basic information from a graph file.
 +
 ------------------------------------------------
diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c
index fe5f00551c..28d043b5a8 100644
--- a/builtin/commit-graph.c
+++ b/builtin/commit-graph.c
@@ -9,7 +9,7 @@ static char const * const builtin_commit_graph_usage[] = {
 	N_("git commit-graph [--pack-dir <packdir>]"),
 	N_("git commit-graph clear [--pack-dir <packdir>]"),
 	N_("git commit-graph read [--graph-hash=<hash>]"),
-	N_("git commit-graph write [--pack-dir <packdir>] [--update-head] [--delete-expired] [--stdin-packs]"),
+	N_("git commit-graph write [--pack-dir <packdir>] [--update-head] [--delete-expired] [--stdin-packs|--stdin-commits]"),
 	NULL
 };
 
@@ -24,7 +24,7 @@ static const char * const builtin_commit_graph_read_usage[] = {
 };
 
 static const char * const builtin_commit_graph_write_usage[] = {
-	N_("git commit-graph write [--pack-dir <packdir>] [--update-head] [--delete-expired] [--stdin-packs]"),
+	N_("git commit-graph write [--pack-dir <packdir>] [--update-head] [--delete-expired] [--stdin-packs|--stdin-commits]"),
 	NULL
 };
 
@@ -34,6 +34,7 @@ static struct opts_commit_graph {
 	int update_head;
 	int delete_expired;
 	int stdin_packs;
+	int stdin_commits;
 } opts;
 
 static int graph_clear(int argc, const char **argv)
@@ -219,6 +220,8 @@ static int graph_write(int argc, const char **argv)
 	int has_existing;
 	const char **pack_indexes = NULL;
 	int nr_packs = 0;
+	const char **commit_hex = NULL;
+	int nr_commits = 0;
 	const char **lines = NULL;
 	int nr_lines = 0;
 	int alloc_lines = 0;
@@ -233,6 +236,8 @@ static int graph_write(int argc, const char **argv)
 			N_("delete expired head graph file")),
 		OPT_BOOL('s', "stdin-packs", &opts.stdin_packs,
 			N_("only scan packfiles listed by stdin")),
+		OPT_BOOL('C', "stdin-commits", &opts.stdin_commits,
+			N_("start walk at commits listed by stdin")),
 		OPT_END(),
 	};
 
@@ -240,6 +245,9 @@ static int graph_write(int argc, const char **argv)
 			     builtin_commit_graph_write_options,
 			     builtin_commit_graph_write_usage, 0);
 
+	if (opts.stdin_packs && opts.stdin_commits)
+		die(_("cannot use both --stdin-commits and --stdin-packs"));
+
 	if (!opts.pack_dir) {
 		struct strbuf path = STRBUF_INIT;
 		strbuf_addstr(&path, get_object_directory());
@@ -261,13 +269,21 @@ static int graph_write(int argc, const char **argv)
 			strbuf_detach(&buf, NULL);
 		}
 
-		pack_indexes = lines;
-		nr_packs = nr_lines;
+		if (opts.stdin_packs) {
+			pack_indexes = lines;
+			nr_packs = nr_lines;
+		}
+		if (opts.stdin_commits) {
+			commit_hex = lines;
+			nr_commits = nr_lines;
+		}
 	}
 
 	graph_hash = write_commit_graph(opts.pack_dir,
 					pack_indexes,
-					nr_packs);
+					nr_packs,
+					commit_hex,
+					nr_commits);
 
 	if (opts.update_head)
 		update_head_file(opts.pack_dir, graph_hash);
diff --git a/commit-graph.c b/commit-graph.c
index 27a34f5eda..3ff3ab03ca 100644
--- a/commit-graph.c
+++ b/commit-graph.c
@@ -657,7 +657,9 @@ static void close_reachable(struct packed_oid_list *oids)
 
 struct object_id *write_commit_graph(const char *pack_dir,
 				     const char **pack_indexes,
-				     int nr_packs)
+				     int nr_packs,
+				     const char **commit_hex,
+				     int nr_commits)
 {
 	struct packed_oid_list oids;
 	struct packed_commit_list commits;
@@ -710,7 +712,27 @@ struct object_id *write_commit_graph(const char *pack_dir,
 			close_pack(p);
 		}
 	}
-	else
+
+	if (commit_hex) {
+		for (i = 0; i < nr_commits; i++) {
+			const char *end;
+			struct object_id oid;
+			struct commit *result;
+
+			if (commit_hex[i] && parse_oid_hex(commit_hex[i], &oid, &end))
+				continue;
+
+			result = lookup_commit_reference_gently(&oid, 1);
+
+			if (result) {
+				ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc);
+				oids.list[oids.nr] = &(result->object.oid);
+				oids.nr++;
+			}
+		}
+	}
+
+	if (!pack_indexes && !commit_hex)
 		for_each_packed_object(if_packed_commit_add_to_list, &oids, 0);
 
 	close_reachable(&oids);
diff --git a/commit-graph.h b/commit-graph.h
index 918b34dd2b..c412f76707 100644
--- a/commit-graph.h
+++ b/commit-graph.h
@@ -52,7 +52,9 @@ extern struct object_id *get_nth_commit_oid(struct commit_graph *g,
 
 extern struct object_id *write_commit_graph(const char *pack_dir,
 					    const char **pack_indexes,
-					    int nr_packs);
+					    int nr_packs,
+					    const char **commit_hex,
+					    int nr_commits);
 
 #endif
 
diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh
index e3546e6844..d803c12afd 100755
--- a/t/t5318-commit-graph.sh
+++ b/t/t5318-commit-graph.sh
@@ -182,6 +182,25 @@ test_expect_success 'build graph from latest pack with closure' '
 graph_git_behavior 'graph from pack, commit 8 vs merge 1' commits/8 merge/1
 graph_git_behavior 'graph from pack, commit 8 vs merge 2' commits/8 merge/2
 
+test_expect_success 'build graph from commits with closure' '
+	git tag -a -m "merge" tag/merge merge/3 &&
+	git rev-parse tag/merge >commits-in &&
+	git rev-parse commits/8 >>commits-in &&
+	git rev-parse merge/1 >>commits-in &&
+	git rev-parse merge/2 >>commits-in &&
+	graph6=$(cat commits-in | git commit-graph write --update-head --delete-expired --stdin-commits) &&
+	test_path_is_file $packdir/graph-$graph6.graph &&
+	test_path_is_file $packdir/graph-$graph5.graph &&
+	test_path_is_file $packdir/graph-head &&
+	printf $graph6 >expect &&
+	test_cmp expect $packdir/graph-head &&
+	git commit-graph read --graph-hash=$graph6 >output &&
+	graph_read_expect "11" "$packdir" &&
+	test_cmp expect output'
+
+graph_git_behavior 'graph from commits, commit 8 vs merge 1' commits/8 merge/1
+graph_git_behavior 'graph from commits, commit 8 vs merge 2' commits/8 merge/2
+
 test_expect_success 'setup bare repo' '
 	cd .. &&
 	git clone --bare --no-local full bare &&
-- 
2.15.1.45.g9b7079f