git@vger.kernel.org mailing list mirror (one of many)
 help / Atom feed
Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
						download: 
* [PATCH 5/9] submodule: move global changed_submodule_names into fetch submodule struct
      [irrelevant] ` <20180917213559.126404-1-sbeller@google.com>
@ 2018-09-17 21:35   ` Stefan Beller
  2018-09-17 21:35   ` [PATCH 6/9] submodule.c: do not copy around submodule list Stefan Beller
                     ` (3 subsequent siblings)
  4 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-17 21:35 UTC (permalink / raw)
  To: sbeller; +Cc: git

The `changed_submodule_names` are only used for fetching, so let's make it
part of the struct that is passed around for fetching submodules.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/submodule.c b/submodule.c
index c6eff7699f3..3520dd76bdf 100644
--- a/submodule.c
+++ b/submodule.c
@@ -24,7 +24,7 @@
 #include "object-store.h"
 
 static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
-static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
+
 static int initialized_fetch_ref_tips;
 static struct oid_array ref_tips_before_fetch;
 static struct oid_array ref_tips_after_fetch;
@@ -1110,7 +1110,22 @@ void check_for_new_submodule_commits(struct object_id *oid)
 	oid_array_append(&ref_tips_after_fetch, oid);
 }
 
-static void calculate_changed_submodule_paths(void)
+struct submodule_parallel_fetch {
+	int count;
+	struct argv_array args;
+	struct repository *r;
+	const char *prefix;
+	int command_line_option;
+	int default_option;
+	int quiet;
+	int result;
+
+	struct string_list changed_submodule_names;
+};
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+
+static void calculate_changed_submodule_paths(
+	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
 	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
@@ -1148,7 +1163,8 @@ static void calculate_changed_submodule_paths(void)
 			continue;
 
 		if (!submodule_has_commits(path, commits))
-			string_list_append(&changed_submodule_names, name->string);
+			string_list_append(&spf->changed_submodule_names,
+					   name->string);
 	}
 
 	free_submodules_oids(&changed_submodules);
@@ -1185,18 +1201,6 @@ int submodule_touches_in_range(struct object_id *excl_oid,
 	return ret;
 }
 
-struct submodule_parallel_fetch {
-	int count;
-	struct argv_array args;
-	struct repository *r;
-	const char *prefix;
-	int command_line_option;
-	int default_option;
-	int quiet;
-	int result;
-};
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
-
 static int get_fetch_recurse_config(const struct submodule *submodule,
 				    struct submodule_parallel_fetch *spf)
 {
@@ -1257,7 +1261,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
 			    !string_list_lookup(
-					&changed_submodule_names,
+					&spf->changed_submodule_names,
 					submodule->name))
 				continue;
 			default_argv = "on-demand";
@@ -1349,8 +1353,8 @@ int fetch_populated_submodules(struct repository *r,
 	argv_array_push(&spf.args, "--recurse-submodules-default");
 	/* default value, "--submodule-prefix" and its value are added later */
 
-	calculate_changed_submodule_paths();
-	string_list_sort(&changed_submodule_names);
+	calculate_changed_submodule_paths(&spf);
+	string_list_sort(&spf.changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
@@ -1359,7 +1363,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&changed_submodule_names, 1);
+	string_list_clear(&spf.changed_submodule_names, 1);
 	return spf.result;
 }
 
-- 
2.19.0.397.gdd90340f6a-goog


^ permalink raw reply	[relevance 17%]

* [PATCH 6/9] submodule.c: do not copy around submodule list
      [irrelevant] ` <20180917213559.126404-1-sbeller@google.com>
  2018-09-17 21:35   ` [PATCH 5/9] submodule: move global changed_submodule_names into fetch submodule struct Stefan Beller
@ 2018-09-17 21:35   ` Stefan Beller
  2018-09-17 21:35   ` [PATCH 7/9] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
                     ` (2 subsequent siblings)
  4 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-17 21:35 UTC (permalink / raw)
  To: sbeller; +Cc: git

'calculate_changed_submodule_paths' uses a local list to compute the
changed submodules, and then produces the result by copying appropriate
items into the result list.

Instead use the result list directly and prune items afterwards
using string_list_remove_empty_items.

By doing so we'll have access to the util pointer for longer that
contains the commits that we need to fetch, which will be
useful in a later patch.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/submodule.c b/submodule.c
index 3520dd76bdf..00a9a3c6b12 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1128,8 +1128,7 @@ static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
-	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
-	const struct string_list_item *name;
+	struct string_list_item *name;
 
 	/* No need to check if there are no submodules configured */
 	if (!submodule_from_path(the_repository, NULL, NULL))
@@ -1146,9 +1145,9 @@ static void calculate_changed_submodule_paths(
 	 * Collect all submodules (whether checked out or not) for which new
 	 * commits have been recorded upstream in "changed_submodule_names".
 	 */
-	collect_changed_submodules(&changed_submodules, &argv);
+	collect_changed_submodules(&spf->changed_submodule_names, &argv);
 
-	for_each_string_list_item(name, &changed_submodules) {
+	for_each_string_list_item(name, &spf->changed_submodule_names) {
 		struct oid_array *commits = name->util;
 		const struct submodule *submodule;
 		const char *path = NULL;
@@ -1162,12 +1161,14 @@ static void calculate_changed_submodule_paths(
 		if (!path)
 			continue;
 
-		if (!submodule_has_commits(path, commits))
-			string_list_append(&spf->changed_submodule_names,
-					   name->string);
+		if (submodule_has_commits(path, commits)) {
+			oid_array_clear(commits);
+			*name->string = '\0';
+		}
 	}
 
-	free_submodules_oids(&changed_submodules);
+	string_list_remove_empty_items(&spf->changed_submodule_names, 1);
+
 	argv_array_clear(&argv);
 	oid_array_clear(&ref_tips_before_fetch);
 	oid_array_clear(&ref_tips_after_fetch);
@@ -1363,7 +1364,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&spf.changed_submodule_names, 1);
+	free_submodules_oids(&spf.changed_submodule_names);
 	return spf.result;
 }
 
-- 
2.19.0.397.gdd90340f6a-goog


^ permalink raw reply	[relevance 17%]

* [PATCH 7/9] submodule: fetch in submodules git directory instead of in worktree
      [irrelevant] ` <20180917213559.126404-1-sbeller@google.com>
  2018-09-17 21:35   ` [PATCH 5/9] submodule: move global changed_submodule_names into fetch submodule struct Stefan Beller
  2018-09-17 21:35   ` [PATCH 6/9] submodule.c: do not copy around submodule list Stefan Beller
@ 2018-09-17 21:35   ` Stefan Beller
  2018-09-17 21:35   ` [PATCH 8/9] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
  2018-09-17 21:35   ` [PATCH 9/9] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
  4 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-17 21:35 UTC (permalink / raw)
  To: sbeller; +Cc: git

This patch started as a refactoring to make 'get_next_submodule' more
readable, but upon doing so, I realized that "git fetch" of the submodule
actually doesn't need to be run in the submodules worktree. So let's run
it in its git dir instead.

That should pave the way towards fetching submodules that are currently
not checked out.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c                 | 45 +++++++++++++++++++++++++++----------
 t/t5526-fetch-submodules.sh |  7 +++++-
 2 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/submodule.c b/submodule.c
index 00a9a3c6b12..88bce534268 100644
--- a/submodule.c
+++ b/submodule.c
@@ -481,6 +481,12 @@ void prepare_submodule_repo_env(struct argv_array *out)
 			 DEFAULT_GIT_DIR_ENVIRONMENT);
 }
 
+static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out)
+{
+	prepare_submodule_repo_env_no_git_dir(out);
+	argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
+}
+
 /* Helper function to display the submodule header line prior to the full
  * summary output. If it can locate the submodule objects directory it will
  * attempt to lookup both the left and right commits and put them into the
@@ -1227,6 +1233,27 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+static char *get_submodule_git_dir(struct repository *r, const char *path)
+{
+	struct repository subrepo;
+	const char *ret;
+
+	if (repo_submodule_init(&subrepo, r, path)) {
+		/* no entry in .gitmodules? */
+		struct strbuf gitdir = STRBUF_INIT;
+		strbuf_repo_worktree_path(&gitdir, r, "%s/.git", path);
+		if (repo_init(&subrepo, gitdir.buf, NULL)) {
+			strbuf_release(&gitdir);
+			return NULL;
+		}
+	}
+
+	ret = xstrdup(subrepo.gitdir);
+	repo_clear(&subrepo);
+
+	return ret;
+}
+
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
@@ -1234,8 +1261,6 @@ static int get_next_submodule(struct child_process *cp,
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_path = STRBUF_INIT;
-		struct strbuf submodule_git_dir = STRBUF_INIT;
 		struct strbuf submodule_prefix = STRBUF_INIT;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
 		const char *git_dir, *default_argv;
@@ -1274,16 +1299,12 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_repo_worktree_path(&submodule_path, spf->r, "%s", ce->name);
-		strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
 		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		git_dir = read_gitfile(submodule_git_dir.buf);
-		if (!git_dir)
-			git_dir = submodule_git_dir.buf;
-		if (is_directory(git_dir)) {
+		git_dir = get_submodule_git_dir(spf->r, ce->name);
+		if (git_dir) {
 			child_process_init(cp);
-			cp->dir = strbuf_detach(&submodule_path, NULL);
-			prepare_submodule_repo_env(&cp->env_array);
+			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+			cp->dir = git_dir;
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1293,10 +1314,10 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
 			argv_array_push(&cp->args, submodule_prefix.buf);
+
+			free(git_dir);
 			ret = 1;
 		}
-		strbuf_release(&submodule_path);
-		strbuf_release(&submodule_git_dir);
 		strbuf_release(&submodule_prefix);
 		if (ret) {
 			spf->count++;
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 6c2f9b2ba26..42692219a1a 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -566,7 +566,12 @@ test_expect_success 'fetching submodule into a broken repository' '
 
 	test_must_fail git -C dst status &&
 	test_must_fail git -C dst diff &&
-	test_must_fail git -C dst fetch --recurse-submodules
+
+	# git-fetch cannot find the git directory of the submodule,
+	# so it will do nothing, successfully, as it cannot distinguish between
+	# this broken submodule and a submodule that was just set active but
+	# not cloned yet
+	git -C dst fetch --recurse-submodules
 '
 
 test_expect_success "fetch new commits when submodule got renamed" '
-- 
2.19.0.397.gdd90340f6a-goog


^ permalink raw reply	[relevance 28%]

* [PATCH 8/9] fetch: retry fetching submodules if needed objects were not fetched
      [irrelevant] ` <20180917213559.126404-1-sbeller@google.com>
                     ` (2 preceding siblings ...)
  2018-09-17 21:35   ` [PATCH 7/9] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
@ 2018-09-17 21:35   ` Stefan Beller
  2018-09-17 21:35   ` [PATCH 9/9] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
  4 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-17 21:35 UTC (permalink / raw)
  To: sbeller; +Cc: git

Currently when git-fetch is asked to recurse into submodules, it dispatches
a plain "git-fetch -C <submodule-dir>" (with some submodule related options
such as prefix and recusing strategy, but) without any information of the
remote or the tip that should be fetched.

This works surprisingly well in some workflows (such as using submodules
as a third party library), while not so well in other scenarios, such
as in a Gerrit topic-based workflow, that can tie together changes
(potentially across repositories) on the server side. One of the parts
of such a Gerrit workflow is to download a change when wanting to examine
it, and you'd want to have its submodule changes that are in the same
topic downloaded as well. However these submodule changes reside in their
own repository in their own ref (refs/changes/<int>).

Retry fetching a submodule by object name if the object id that the
superproject points to, cannot be found.

This retrying does not happen when the "git fetch" done at the
superproject is not storing the fetched results in remote
tracking branches (i.e. instead just recording them to
FETCH_HEAD) in this step. A later patch will fix this.

builtin/fetch used to only inspect submodules when they were fetched
"on-demand", as in either on/off case it was clear whether the submodule
needs to be fetched. However to know whether we need to try fetching the
object ids, we need to identify the object names, which is done in this
function check_for_new_submodule_commits(), so we'll also run that code
in case the submodule recursion is set to "on".

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c             |  9 ++--
 submodule.c                 | 87 ++++++++++++++++++++++++++++++++++++-
 t/t5526-fetch-submodules.sh | 16 +++++++
 3 files changed, 104 insertions(+), 8 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 61bec5d213d..95c44bf6ffa 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -700,8 +700,7 @@ static int update_local_ref(struct ref *ref,
 			what = _("[new ref]");
 		}
 
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref(msg, ref, 0);
 		format_display(display, r ? '!' : '*', what,
@@ -716,8 +715,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "..");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("fast-forward", ref, 1);
 		format_display(display, r ? '!' : ' ', quickref.buf,
@@ -731,8 +729,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "...");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("forced-update", ref, 1);
 		format_display(display, r ? '!' : '+', quickref.buf,
diff --git a/submodule.c b/submodule.c
index 88bce534268..7d59e56171f 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1127,8 +1127,11 @@ struct submodule_parallel_fetch {
 	int result;
 
 	struct string_list changed_submodule_names;
+	struct string_list retry;
 };
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \
+		  STRING_LIST_INIT_DUP, \
+		  STRING_LIST_INIT_NODUP}
 
 static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
@@ -1259,8 +1262,10 @@ static int get_next_submodule(struct child_process *cp,
 {
 	int ret = 0;
 	struct submodule_parallel_fetch *spf = data;
+	struct string_list_item *it;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
+		int recurse_config;
 		struct strbuf submodule_prefix = STRBUF_INIT;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
 		const char *git_dir, *default_argv;
@@ -1280,7 +1285,9 @@ static int get_next_submodule(struct child_process *cp,
 			}
 		}
 
-		switch (get_fetch_recurse_config(submodule, spf))
+		recurse_config = get_fetch_recurse_config(submodule, spf);
+
+		switch (recurse_config)
 		{
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
@@ -1321,9 +1328,50 @@ static int get_next_submodule(struct child_process *cp,
 		strbuf_release(&submodule_prefix);
 		if (ret) {
 			spf->count++;
+			if (submodule != &default_submodule)
+				/* discard const-ness: */
+				*task_cb = (void*)submodule;
 			return 1;
 		}
 	}
+
+retry_next:
+
+	if (spf->retry.nr) {
+		struct strbuf submodule_prefix = STRBUF_INIT;
+		const struct submodule *sub;
+
+		it = string_list_last(&spf->retry);
+		sub = submodule_from_name(spf->r, &null_oid,
+					  it->string);
+
+		child_process_init(cp);
+		cp->dir = get_submodule_git_dir(spf->r, sub->path);
+		if (!cp->dir) {
+			string_list_pop(&spf->retry, 0);
+			goto retry_next;
+		}
+		prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+		cp->git_cmd = 1;
+
+		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, sub->path);
+		argv_array_init(&cp->args);
+		argv_array_pushv(&cp->args, spf->args.argv);
+		argv_array_push(&cp->args, "on-demand");
+		argv_array_push(&cp->args, "--submodule-prefix");
+		argv_array_push(&cp->args, submodule_prefix.buf);
+
+		/* NEEDSWORK: have get_default_remote from s--h */
+		argv_array_push(&cp->args, "origin");
+		oid_array_for_each_unique(it->util,
+					  append_oid_to_argv, &cp->args);
+
+		*task_cb = NULL; /* make sure we do not recurse forever */
+		strbuf_release(&submodule_prefix);
+		string_list_pop(&spf->retry, 0);
+		return 1;
+	}
+
 	return 0;
 }
 
@@ -1337,14 +1385,49 @@ static int fetch_start_failure(struct strbuf *err,
 	return 0;
 }
 
+static int commit_exists_in_sub(const struct object_id *oid, void *data)
+{
+	struct repository *subrepo = data;
+
+	enum object_type type = oid_object_info(subrepo, oid, NULL);
+
+	return type != OBJ_COMMIT;
+}
+
 static int fetch_finish(int retvalue, struct strbuf *err,
 			void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct submodule *sub = task_cb;
+	struct repository subrepo;
 
 	if (retvalue)
 		spf->result = 1;
 
+	if (!sub)
+		return 0;
+
+	if (repo_submodule_init(&subrepo, spf->r, sub->path) < 0)
+		warning(_("Could not get submodule repository for submodule '%s' in repository '%s'"),
+			  sub->path, spf->r->worktree);
+	else {
+		struct string_list_item *it;
+		struct oid_array *commits;
+
+		it = string_list_lookup(&spf->changed_submodule_names, sub->name);
+		if (!it)
+			return 0;
+
+		commits = it->util;
+		oid_array_filter(commits,
+				 commit_exists_in_sub,
+				 &subrepo);
+
+		if (commits->nr)
+			string_list_append(&spf->retry, sub->name)
+				->util = commits;
+	}
+
 	return 0;
 }
 
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 42692219a1a..af12c50e7dd 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -605,4 +605,20 @@ test_expect_success "fetch new commits when submodule got renamed" '
 	test_cmp expect actual
 '
 
+test_expect_success "fetch new commits on-demand when they are not reachable" '
+	git checkout --detach &&
+	C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
+	git -C submodule update-ref refs/changes/1 $C &&
+	git update-index --cacheinfo 160000 $C submodule &&
+	git commit -m "updated submodule outside of refs/heads" &&
+	D=$(git rev-parse HEAD) &&
+	git update-ref refs/changes/2 $D &&
+	(
+		cd downstream &&
+		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git -C submodule cat-file -t $C &&
+		git checkout --recurse-submodules FETCH_HEAD
+	)
+'
+
 test_done
-- 
2.19.0.397.gdd90340f6a-goog


^ permalink raw reply	[relevance 19%]

* [PATCH 9/9] builtin/fetch: check for submodule updates for non branch fetches
      [irrelevant] ` <20180917213559.126404-1-sbeller@google.com>
                     ` (3 preceding siblings ...)
  2018-09-17 21:35   ` [PATCH 8/9] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
@ 2018-09-17 21:35   ` Stefan Beller
  4 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-17 21:35 UTC (permalink / raw)
  To: sbeller; +Cc: git

Gerrit, the code review tool, has a different workflow than our mailing
list based approach. Usually users upload changes to a Gerrit server and
continuous integration and testing happens by bots. Sometimes however a
user wants to checkout a change locally and look at it locally. For this
use case, Gerrit offers a command line snippet to copy and paste to your
terminal, which looks like

  git fetch https://<host>/gerrit refs/changes/<id> &&
  git checkout FETCH_HEAD

For Gerrit changes that contain changing submodule gitlinks, it would be
easy to extend both the fetch and checkout with the '--recurse-submodules'
flag, such that this command line snippet would produce the state of a
change locally.

However the functionality added in the previous patch, which would
ensure that we fetch the objects in the submodule that the gitlink pointed
at, only works for remote tracking branches so far, not for FETCH_HEAD.

Make sure that fetching a superproject to its FETCH_HEAD, also respects
the existence checks for objects in the submodule recursion.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c             | 5 ++++-
 t/t5526-fetch-submodules.sh | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 95c44bf6ffa..ea6ecd123e7 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -887,11 +887,14 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
 				rc |= update_local_ref(ref, what, rm, &note,
 						       summary_width);
 				free(ref);
-			} else
+			} else {
+				check_for_new_submodule_commits(&rm->old_oid);
 				format_display(&note, '*',
 					       *kind ? kind : "branch", NULL,
 					       *what ? what : "HEAD",
 					       "FETCH_HEAD", summary_width);
+			}
+
 			if (note.len) {
 				if (verbosity >= 0 && !shown_url) {
 					fprintf(stderr, _("From %.*s\n"),
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index af12c50e7dd..a509eabb044 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -615,7 +615,7 @@ test_expect_success "fetch new commits on-demand when they are not reachable" '
 	git update-ref refs/changes/2 $D &&
 	(
 		cd downstream &&
-		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git fetch --recurse-submodules origin refs/changes/2 &&
 		git -C submodule cat-file -t $C &&
 		git checkout --recurse-submodules FETCH_HEAD
 	)
-- 
2.19.0.397.gdd90340f6a-goog


^ permalink raw reply	[relevance 22%]

* Re: [PATCH] fetch: Ensure that fetch.recurseSubmodules overrides submodule.recurse.
      [irrelevant] <20180921185149.8670-1-marcnarc@xiplink.com>
@ 2018-09-21 19:22 ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-21 19:22 UTC (permalink / raw)
  To: Marc Branchaud; +Cc: git

On Fri, Sep 21, 2018 at 12:00 PM Marc Branchaud <marcnarc@xiplink.com> wrote:
>
> Also document this fact.
>
> Signed-off-by: Marc Branchaud <marcnarc@xiplink.com>
> ---
>
> I ran into this bug when I had both fetch.recurseSubmodules=on-demand and
> submodule.recurse=true, and submodule.recurse was set *after*
> fetch.recurseSubmodules in my config.
>
> The fix ensures that fetch.recurseSubmodules always overrides
> submodule.recurse.  If neither is set then fetch still behaves as if
> fetch.recurseSubmodules=on-demand (the documented default).

At least the second paragraph is valuable information in the commit
message, so maybe add it there? I am not sure if the first paragraph is
a good part for the commit message, but maybe helps for writing a test?

> +       reference.  This option overrides the more general submodule.recurse
> +       option, for the `fetch` command.
>
>  fetch.fsckObjects::
>         If it is set to true, git-fetch-pack will check all fetched
> @@ -3465,7 +3466,8 @@ submodule.active::
>  submodule.recurse::
>         Specifies if commands recurse into submodules by default. This
>         applies to all commands that have a `--recurse-submodules` option,
> -       except `clone`.
> +       except `clone`.  Also, the `fetch` command's behaviour can be specified
> +       independently with the fetch.recurseSubmodules option.

There is also push.recurseSubmodules, which should behave similarly?

The series that introduced submodule.recurse ends with 58f4203e7db
(builtin/fetch.c: respect 'submodule.recurse' option, 2017-05-31)
(sb/submodule-blanket-recursive)
seems to have overlooked this only for fetch/push, as the other
commands (checkout, read-tree, reset, grep) do not have their
own specific setting to recurse.


> @@ -88,6 +90,7 @@ static int git_fetch_config(const char *k, const char *v, void *cb)
>                 max_children = parse_submodule_fetchjobs(k, v);
>                 return 0;
>         } else if (!strcmp(k, "fetch.recursesubmodules")) {
> +               recurse_submodules_set_explicitly = 1;

the command line option also overried explicitely, but that
is ensured via the program flow (parse_config happens after
git_config to overlay options, which itself was pre-seeded
with fetch_config_from_gitmodules).

I briefly wondered if this overlaying approach would be better
(i.e. first do git_config with more generic option, and then
again with the more detailed option) as it would save one
global variable, but the downsides are terrible (way more
work to do, more code and such), so I think having a global
makes sense and gets the job done.

Ideally instead of a global we'd have this flag stored in
the repository struct, as eventually in the long run,
fetch_populated_submodules could happen in-process
instead of spawning fetch processes for each submodule
(and their nested submodules which may be configured
differently). But for now the global will do.

Thanks!
Stefan

^ permalink raw reply	[relevance 7%]

* [PATCHv3 0/8] fetch: make sure submodule oids are fetched
@ 2018-09-21 22:35 Stefan Beller
  2018-09-21 22:35 ` [PATCH 2/8] submodule.c: fix indentation Stefan Beller
                   ` (7 more replies)
  0 siblings, 8 replies; 200+ results
From: Stefan Beller @ 2018-09-21 22:35 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

v3:
* I discovered some issues with v2 after sending,
  which is why I rewrote the later patches completely
  and now we pass around a "task" struct that contains everything to know
  about the things to work on and what needs free()ing afterwards.
* as it is no longer string list based, this drops adding string_list_{pop, last}

v2:
* extended commit messages,
* plugged a memory leak
* rewrote the patch "sha1-array: provide oid_array_filter" to be much more like 
  object_array_fiter
* fixed a typo pointed out by Ramsay.

The range diff is below.
  
Thanks,
Stefan

v1:
Currently when git-fetch is asked to recurse into submodules, it dispatches
a plain "git-fetch -C <submodule-dir>" (and some submodule related options
such as prefix and recusing strategy, but) without any information of the
remote or the tip that should be fetched.

This works surprisingly well in some workflows, not so well in others,
which this series aims to fix.

The first patches provide new basic functionality and do some refactoring;
the interesting part is in the two last patches.

This was discussed in
https://public-inbox.org/git/20180808221752.195419-1-sbeller@google.com/
and I think I addressed all feedback so far.

Stefan Beller (8):
  sha1-array: provide oid_array_filter
  submodule.c: fix indentation
  submodule.c: sort changed_submodule_names before searching it
  submodule: move global changed_submodule_names into fetch submodule
    struct
  submodule.c: do not copy around submodule list
  submodule: fetch in submodules git directory instead of in worktree
  fetch: retry fetching submodules if needed objects were not fetched
  builtin/fetch: check for submodule updates for non branch fetches

 builtin/fetch.c             |  14 +-
 sha1-array.c                |  17 +++
 sha1-array.h                |   9 ++
 submodule.c                 | 268 ++++++++++++++++++++++++++++--------
 t/t5526-fetch-submodules.sh |  23 +++-
 5 files changed, 268 insertions(+), 63 deletions(-)

-- 
2.19.0.444.g18242da7ef-goog


^ permalink raw reply	[relevance 10%]

* [PATCH 2/8] submodule.c: fix indentation
  2018-09-21 22:35 [PATCHv3 0/8] fetch: make sure submodule oids are fetched Stefan Beller
@ 2018-09-21 22:35 ` Stefan Beller
  2018-09-21 22:35 ` [PATCH 3/8] submodule.c: sort changed_submodule_names before searching it Stefan Beller
                   ` (6 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-21 22:35 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

The submodule subsystem is really bad at staying within 80 characters.
Fix it while we are here.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/submodule.c b/submodule.c
index ed05339b588..67469a8f513 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1245,7 +1245,8 @@ static int get_next_submodule(struct child_process *cp,
 		if (!submodule) {
 			const char *name = default_name_or_path(ce->name);
 			if (name) {
-				default_submodule.path = default_submodule.name = name;
+				default_submodule.path = name;
+				default_submodule.name = name;
 				submodule = &default_submodule;
 			}
 		}
@@ -1255,8 +1256,10 @@ static int get_next_submodule(struct child_process *cp,
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names,
-							 submodule->name))
+			if (!submodule ||
+			    !unsorted_string_list_lookup(
+					&changed_submodule_names,
+					submodule->name))
 				continue;
 			default_argv = "on-demand";
 			break;
-- 
2.19.0.444.g18242da7ef-goog


^ permalink raw reply	[relevance 26%]

* [PATCH 3/8] submodule.c: sort changed_submodule_names before searching it
  2018-09-21 22:35 [PATCHv3 0/8] fetch: make sure submodule oids are fetched Stefan Beller
  2018-09-21 22:35 ` [PATCH 2/8] submodule.c: fix indentation Stefan Beller
@ 2018-09-21 22:35 ` Stefan Beller
  2018-09-21 22:35 ` [PATCH 4/8] submodule: move global changed_submodule_names into fetch submodule struct Stefan Beller
                   ` (5 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-21 22:35 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

We can string_list_insert() to maintain sorted-ness of the
list as we find new items, or we can string_list_append() to
build an unsorted list and sort it at the end just once.

To pick which one is more appropriate, we notice the fact
that we discover new items more or less in the already
sorted order.  That makes "append then sort" more
appropriate.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/submodule.c b/submodule.c
index 67469a8f513..2f5c19d0aac 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1257,7 +1257,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
-			    !unsorted_string_list_lookup(
+			    !string_list_lookup(
 					&changed_submodule_names,
 					submodule->name))
 				continue;
@@ -1351,6 +1351,7 @@ int fetch_populated_submodules(struct repository *r,
 	/* default value, "--submodule-prefix" and its value are added later */
 
 	calculate_changed_submodule_paths();
+	string_list_sort(&changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
-- 
2.19.0.444.g18242da7ef-goog


^ permalink raw reply	[relevance 17%]

* [PATCH 4/8] submodule: move global changed_submodule_names into fetch submodule struct
  2018-09-21 22:35 [PATCHv3 0/8] fetch: make sure submodule oids are fetched Stefan Beller
  2018-09-21 22:35 ` [PATCH 2/8] submodule.c: fix indentation Stefan Beller
  2018-09-21 22:35 ` [PATCH 3/8] submodule.c: sort changed_submodule_names before searching it Stefan Beller
@ 2018-09-21 22:35 ` Stefan Beller
  2018-09-21 22:35 ` [PATCH 5/8] submodule.c: do not copy around submodule list Stefan Beller
                   ` (4 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-21 22:35 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

The `changed_submodule_names` are only used for fetching, so let's make it
part of the struct that is passed around for fetching submodules.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/submodule.c b/submodule.c
index 2f5c19d0aac..7b4d136849b 100644
--- a/submodule.c
+++ b/submodule.c
@@ -25,7 +25,7 @@
 #include "commit-reach.h"
 
 static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
-static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
+
 static int initialized_fetch_ref_tips;
 static struct oid_array ref_tips_before_fetch;
 static struct oid_array ref_tips_after_fetch;
@@ -1111,7 +1111,22 @@ void check_for_new_submodule_commits(struct object_id *oid)
 	oid_array_append(&ref_tips_after_fetch, oid);
 }
 
-static void calculate_changed_submodule_paths(void)
+struct submodule_parallel_fetch {
+	int count;
+	struct argv_array args;
+	struct repository *r;
+	const char *prefix;
+	int command_line_option;
+	int default_option;
+	int quiet;
+	int result;
+
+	struct string_list changed_submodule_names;
+};
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+
+static void calculate_changed_submodule_paths(
+	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
 	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
@@ -1149,7 +1164,8 @@ static void calculate_changed_submodule_paths(void)
 			continue;
 
 		if (!submodule_has_commits(path, commits))
-			string_list_append(&changed_submodule_names, name->string);
+			string_list_append(&spf->changed_submodule_names,
+					   name->string);
 	}
 
 	free_submodules_oids(&changed_submodules);
@@ -1186,18 +1202,6 @@ int submodule_touches_in_range(struct object_id *excl_oid,
 	return ret;
 }
 
-struct submodule_parallel_fetch {
-	int count;
-	struct argv_array args;
-	struct repository *r;
-	const char *prefix;
-	int command_line_option;
-	int default_option;
-	int quiet;
-	int result;
-};
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
-
 static int get_fetch_recurse_config(const struct submodule *submodule,
 				    struct submodule_parallel_fetch *spf)
 {
@@ -1258,7 +1262,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
 			    !string_list_lookup(
-					&changed_submodule_names,
+					&spf->changed_submodule_names,
 					submodule->name))
 				continue;
 			default_argv = "on-demand";
@@ -1350,8 +1354,8 @@ int fetch_populated_submodules(struct repository *r,
 	argv_array_push(&spf.args, "--recurse-submodules-default");
 	/* default value, "--submodule-prefix" and its value are added later */
 
-	calculate_changed_submodule_paths();
-	string_list_sort(&changed_submodule_names);
+	calculate_changed_submodule_paths(&spf);
+	string_list_sort(&spf.changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
@@ -1360,7 +1364,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&changed_submodule_names, 1);
+	string_list_clear(&spf.changed_submodule_names, 1);
 	return spf.result;
 }
 
-- 
2.19.0.444.g18242da7ef-goog


^ permalink raw reply	[relevance 17%]

* [PATCH 5/8] submodule.c: do not copy around submodule list
  2018-09-21 22:35 [PATCHv3 0/8] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (2 preceding siblings ...)
  2018-09-21 22:35 ` [PATCH 4/8] submodule: move global changed_submodule_names into fetch submodule struct Stefan Beller
@ 2018-09-21 22:35 ` Stefan Beller
  2018-09-21 22:35 ` [PATCH 6/8] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
                   ` (3 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-21 22:35 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

'calculate_changed_submodule_paths' uses a local list to compute the
changed submodules, and then produces the result by copying appropriate
items into the result list.

Instead use the result list directly and prune items afterwards
using string_list_remove_empty_items.

By doing so we'll have access to the util pointer for longer that
contains the commits that we need to fetch, which will be
useful in a later patch.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/submodule.c b/submodule.c
index 7b4d136849b..209680ec076 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1129,8 +1129,7 @@ static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
-	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
-	const struct string_list_item *name;
+	struct string_list_item *name;
 
 	/* No need to check if there are no submodules configured */
 	if (!submodule_from_path(the_repository, NULL, NULL))
@@ -1147,9 +1146,9 @@ static void calculate_changed_submodule_paths(
 	 * Collect all submodules (whether checked out or not) for which new
 	 * commits have been recorded upstream in "changed_submodule_names".
 	 */
-	collect_changed_submodules(&changed_submodules, &argv);
+	collect_changed_submodules(&spf->changed_submodule_names, &argv);
 
-	for_each_string_list_item(name, &changed_submodules) {
+	for_each_string_list_item(name, &spf->changed_submodule_names) {
 		struct oid_array *commits = name->util;
 		const struct submodule *submodule;
 		const char *path = NULL;
@@ -1163,12 +1162,14 @@ static void calculate_changed_submodule_paths(
 		if (!path)
 			continue;
 
-		if (!submodule_has_commits(path, commits))
-			string_list_append(&spf->changed_submodule_names,
-					   name->string);
+		if (submodule_has_commits(path, commits)) {
+			oid_array_clear(commits);
+			*name->string = '\0';
+		}
 	}
 
-	free_submodules_oids(&changed_submodules);
+	string_list_remove_empty_items(&spf->changed_submodule_names, 1);
+
 	argv_array_clear(&argv);
 	oid_array_clear(&ref_tips_before_fetch);
 	oid_array_clear(&ref_tips_after_fetch);
@@ -1364,7 +1365,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&spf.changed_submodule_names, 1);
+	free_submodules_oids(&spf.changed_submodule_names);
 	return spf.result;
 }
 
-- 
2.19.0.444.g18242da7ef-goog


^ permalink raw reply	[relevance 17%]

* [PATCH 6/8] submodule: fetch in submodules git directory instead of in worktree
  2018-09-21 22:35 [PATCHv3 0/8] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (3 preceding siblings ...)
  2018-09-21 22:35 ` [PATCH 5/8] submodule.c: do not copy around submodule list Stefan Beller
@ 2018-09-21 22:35 ` Stefan Beller
  2018-09-21 22:35 ` [PATCH 7/8] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
                   ` (2 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-21 22:35 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

This patch started as a refactoring to make 'get_next_submodule' more
readable, but upon doing so, I realized that "git fetch" of the submodule
actually doesn't need to be run in the submodules worktree. So let's run
it in its git dir instead.

That should pave the way towards fetching submodules that are currently
not checked out.

This patch leaks the cp->dir in get_next_submodule, as any further
callback in run_processes_parallel doesn't have access to the child
process any more. In an early iteration of this patch, the function
get_submodule_repo_for directly returned the string containing the
git directory, which would be a better design choice for this patch.

However the next patch both fixes the memory leak of cp->dir and also has
a use case for using the full repository handle of the submodule, so
it makes sense to introduce the get_submodule_repo_for here already.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c                 | 47 +++++++++++++++++++++++++++----------
 t/t5526-fetch-submodules.sh |  7 +++++-
 2 files changed, 40 insertions(+), 14 deletions(-)

diff --git a/submodule.c b/submodule.c
index 209680ec076..8829e8d4eff 100644
--- a/submodule.c
+++ b/submodule.c
@@ -482,6 +482,12 @@ void prepare_submodule_repo_env(struct argv_array *out)
 			 DEFAULT_GIT_DIR_ENVIRONMENT);
 }
 
+static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out)
+{
+	prepare_submodule_repo_env_no_git_dir(out);
+	argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
+}
+
 /* Helper function to display the submodule header line prior to the full
  * summary output. If it can locate the submodule objects directory it will
  * attempt to lookup both the left and right commits and put them into the
@@ -1228,6 +1234,25 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+static struct repository *get_submodule_repo_for(struct repository *r,
+						 const char *path)
+{
+	struct repository *ret = xmalloc(sizeof(*ret));
+
+	if (repo_submodule_init(ret, r, path)) {
+		/* no entry in .gitmodules? */
+		struct strbuf gitdir = STRBUF_INIT;
+		strbuf_repo_worktree_path(&gitdir, r, "%s/.git", path);
+		if (repo_init(ret, gitdir.buf, NULL)) {
+			strbuf_release(&gitdir);
+			return NULL;
+		}
+		strbuf_release(&gitdir);
+	}
+
+	return ret;
+}
+
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
@@ -1235,12 +1260,11 @@ static int get_next_submodule(struct child_process *cp,
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_path = STRBUF_INIT;
-		struct strbuf submodule_git_dir = STRBUF_INIT;
 		struct strbuf submodule_prefix = STRBUF_INIT;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
-		const char *git_dir, *default_argv;
+		const char *default_argv;
 		const struct submodule *submodule;
+		struct repository *repo;
 		struct submodule default_submodule = SUBMODULE_INIT;
 
 		if (!S_ISGITLINK(ce->ce_mode))
@@ -1275,16 +1299,12 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_repo_worktree_path(&submodule_path, spf->r, "%s", ce->name);
-		strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
 		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		git_dir = read_gitfile(submodule_git_dir.buf);
-		if (!git_dir)
-			git_dir = submodule_git_dir.buf;
-		if (is_directory(git_dir)) {
+		repo = get_submodule_repo_for(spf->r, ce->name);
+		if (repo) {
 			child_process_init(cp);
-			cp->dir = strbuf_detach(&submodule_path, NULL);
-			prepare_submodule_repo_env(&cp->env_array);
+			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+			cp->dir = xstrdup(repo->gitdir);
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1294,10 +1314,11 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
 			argv_array_push(&cp->args, submodule_prefix.buf);
+
+			repo_clear(repo);
+			free(repo);
 			ret = 1;
 		}
-		strbuf_release(&submodule_path);
-		strbuf_release(&submodule_git_dir);
 		strbuf_release(&submodule_prefix);
 		if (ret) {
 			spf->count++;
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 6c2f9b2ba26..42692219a1a 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -566,7 +566,12 @@ test_expect_success 'fetching submodule into a broken repository' '
 
 	test_must_fail git -C dst status &&
 	test_must_fail git -C dst diff &&
-	test_must_fail git -C dst fetch --recurse-submodules
+
+	# git-fetch cannot find the git directory of the submodule,
+	# so it will do nothing, successfully, as it cannot distinguish between
+	# this broken submodule and a submodule that was just set active but
+	# not cloned yet
+	git -C dst fetch --recurse-submodules
 '
 
 test_expect_success "fetch new commits when submodule got renamed" '
-- 
2.19.0.444.g18242da7ef-goog


^ permalink raw reply	[relevance 27%]

* [PATCH 7/8] fetch: retry fetching submodules if needed objects were not fetched
  2018-09-21 22:35 [PATCHv3 0/8] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (4 preceding siblings ...)
  2018-09-21 22:35 ` [PATCH 6/8] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
@ 2018-09-21 22:35 ` Stefan Beller
  2018-09-21 22:35 ` [PATCH 8/8] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
      [irrelevant] ` <20180921223558.65055-2-sbeller@google.com>
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-21 22:35 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

Currently when git-fetch is asked to recurse into submodules, it dispatches
a plain "git-fetch -C <submodule-dir>" (with some submodule related options
such as prefix and recusing strategy, but) without any information of the
remote or the tip that should be fetched.

This works surprisingly well in some workflows (such as using submodules
as a third party library), while not so well in other scenarios, such
as in a Gerrit topic-based workflow, that can tie together changes
(potentially across repositories) on the server side. One of the parts
of such a Gerrit workflow is to download a change when wanting to examine
it, and you'd want to have its submodule changes that are in the same
topic downloaded as well. However these submodule changes reside in their
own repository in their own ref (refs/changes/<int>).

Retry fetching a submodule by object name if the object id that the
superproject points to, cannot be found.

This retrying does not happen when the "git fetch" done at the
superproject is not storing the fetched results in remote
tracking branches (i.e. instead just recording them to
FETCH_HEAD) in this step. A later patch will fix this.

builtin/fetch used to only inspect submodules when they were fetched
"on-demand", as in either on/off case it was clear whether the submodule
needs to be fetched. However to know whether we need to try fetching the
object ids, we need to identify the object names, which is done in this
function check_for_new_submodule_commits(), so we'll also run that code
in case the submodule recursion is set to "on".

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c             |   9 +-
 submodule.c                 | 182 ++++++++++++++++++++++++++++++------
 t/t5526-fetch-submodules.sh |  16 ++++
 3 files changed, 174 insertions(+), 33 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 0696abfc2a1..e3b03ad3bd3 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -707,8 +707,7 @@ static int update_local_ref(struct ref *ref,
 			what = _("[new ref]");
 		}
 
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref(msg, ref, 0);
 		format_display(display, r ? '!' : '*', what,
@@ -723,8 +722,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "..");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("fast-forward", ref, 1);
 		format_display(display, r ? '!' : ' ', quickref.buf,
@@ -738,8 +736,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "...");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("forced-update", ref, 1);
 		format_display(display, r ? '!' : '+', quickref.buf,
diff --git a/submodule.c b/submodule.c
index 8829e8d4eff..ba98a94d4db 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1128,8 +1128,12 @@ struct submodule_parallel_fetch {
 	int result;
 
 	struct string_list changed_submodule_names;
+	struct get_next_submodule_task **retry;
+	int retry_nr, retry_alloc;
 };
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \
+		  STRING_LIST_INIT_DUP, \
+		  NULL, 0, 0}
 
 static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
@@ -1234,6 +1238,56 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+struct get_next_submodule_task {
+	struct repository *repo;
+	const struct submodule *sub;
+	unsigned free_sub : 1; /* Do we need to free the submodule? */
+	struct oid_array *commits;
+};
+
+static const struct submodule *get_default_submodule(const char *path)
+{
+	struct submodule *ret = NULL;
+	const char *name = default_name_or_path(path);
+
+	if (!name)
+		return NULL;
+
+	ret = xmalloc(sizeof(*ret));
+	memset(ret, 0, sizeof(*ret));
+	ret->path = name;
+	ret->name = name;
+
+	return (const struct submodule *) ret;
+}
+
+static struct get_next_submodule_task *get_next_submodule_task_create(
+	struct repository *r, const char *path)
+{
+	struct get_next_submodule_task *task = xmalloc(sizeof(*task));
+	memset(task, 0, sizeof(*task));
+
+	task->sub = submodule_from_path(r, &null_oid, path);
+	if (!task->sub) {
+		task->sub = get_default_submodule(path);
+		task->free_sub = 1;
+	}
+
+	return task;
+}
+
+static void get_next_submodule_task_release(struct get_next_submodule_task *p)
+{
+	if (p->free_sub)
+		free((void*)p->sub);
+	p->free_sub = 0;
+	p->sub = NULL;
+
+	if (p->repo)
+		repo_clear(p->repo);
+	FREE_AND_NULL(p->repo);
+}
+
 static struct repository *get_submodule_repo_for(struct repository *r,
 						 const char *path)
 {
@@ -1256,39 +1310,35 @@ static struct repository *get_submodule_repo_for(struct repository *r,
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
-	int ret = 0;
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_prefix = STRBUF_INIT;
+		int recurse_config;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
 		const char *default_argv;
-		const struct submodule *submodule;
-		struct repository *repo;
-		struct submodule default_submodule = SUBMODULE_INIT;
+		struct get_next_submodule_task *task;
 
 		if (!S_ISGITLINK(ce->ce_mode))
 			continue;
 
-		submodule = submodule_from_path(spf->r, &null_oid, ce->name);
-		if (!submodule) {
-			const char *name = default_name_or_path(ce->name);
-			if (name) {
-				default_submodule.path = name;
-				default_submodule.name = name;
-				submodule = &default_submodule;
-			}
+		task = get_next_submodule_task_create(spf->r, ce->name);
+
+		if (!task->sub) {
+			free(task);
+			continue;
 		}
 
-		switch (get_fetch_recurse_config(submodule, spf))
+		recurse_config = get_fetch_recurse_config(task->sub, spf);
+
+		switch (recurse_config)
 		{
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule ||
+			if (!task->sub ||
 			    !string_list_lookup(
 					&spf->changed_submodule_names,
-					submodule->name))
+					task->sub->name))
 				continue;
 			default_argv = "on-demand";
 			break;
@@ -1299,12 +1349,13 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		repo = get_submodule_repo_for(spf->r, ce->name);
-		if (repo) {
+		task->repo = get_submodule_repo_for(spf->r,
+						       task->sub->path);
+		if (task->repo) {
+			struct strbuf submodule_prefix = STRBUF_INIT;
 			child_process_init(cp);
 			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
-			cp->dir = xstrdup(repo->gitdir);
+			cp->dir = task->repo->gitdir;
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1313,18 +1364,51 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_pushv(&cp->args, spf->args.argv);
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
+
+			strbuf_addf(&submodule_prefix, "%s%s/",
+						       spf->prefix,
+						       task->sub->path);
 			argv_array_push(&cp->args, submodule_prefix.buf);
 
-			repo_clear(repo);
-			free(repo);
-			ret = 1;
-		}
-		strbuf_release(&submodule_prefix);
-		if (ret) {
 			spf->count++;
+			*task_cb = task;
+
+			strbuf_release(&submodule_prefix);
 			return 1;
+		} else {
+			get_next_submodule_task_release(task);
+			free(task);
 		}
 	}
+
+	if (spf->retry_nr) {
+		struct get_next_submodule_task *task = spf->retry[spf->retry_nr - 1];
+		struct strbuf submodule_prefix = STRBUF_INIT;
+		spf->retry_nr--;
+
+		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, task->sub->path);
+
+		child_process_init(cp);
+		prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+		cp->git_cmd = 1;
+		cp->dir = task->repo->gitdir;
+
+		argv_array_init(&cp->args);
+		argv_array_pushv(&cp->args, spf->args.argv);
+		argv_array_push(&cp->args, "on-demand");
+		argv_array_push(&cp->args, "--submodule-prefix");
+		argv_array_push(&cp->args, submodule_prefix.buf);
+
+		/* NEEDSWORK: have get_default_remote from s--h */
+		argv_array_push(&cp->args, "origin");
+		oid_array_for_each_unique(task->commits,
+					  append_oid_to_argv, &cp->args);
+
+		*task_cb = task;
+		strbuf_release(&submodule_prefix);
+		return 1;
+	}
+
 	return 0;
 }
 
@@ -1332,20 +1416,64 @@ static int fetch_start_failure(struct strbuf *err,
 			       void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct get_next_submodule_task *task = task_cb;
 
 	spf->result = 1;
 
+	get_next_submodule_task_release(task);
 	return 0;
 }
 
+static int commit_exists_in_sub(const struct object_id *oid, void *data)
+{
+	struct repository *subrepo = data;
+
+	enum object_type type = oid_object_info(subrepo, oid, NULL);
+
+	return type != OBJ_COMMIT;
+}
+
 static int fetch_finish(int retvalue, struct strbuf *err,
 			void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct get_next_submodule_task *task = task_cb;
+	const struct submodule *sub;
+
+	struct string_list_item *it;
+	struct oid_array *commits;
 
 	if (retvalue)
 		spf->result = 1;
 
+	if (!task)
+		return 0;
+
+	sub = task->sub;
+	if (!sub)
+		goto out;
+
+	it = string_list_lookup(&spf->changed_submodule_names, sub->name);
+	if (!it)
+		goto out;
+
+	commits = it->util;
+	oid_array_filter(commits,
+			 commit_exists_in_sub,
+			 task->repo);
+
+	trace_printf("checking for submodule: needs %d more commits", commits->nr);
+	if (commits->nr) {
+		task->commits = commits;
+		ALLOC_GROW(spf->retry, spf->retry_nr + 1, spf->retry_alloc);
+		spf->retry[spf->retry_nr] = task;
+		spf->retry_nr++;
+		return 0;
+	}
+
+out:
+	get_next_submodule_task_release(task);
+
 	return 0;
 }
 
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 42692219a1a..af12c50e7dd 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -605,4 +605,20 @@ test_expect_success "fetch new commits when submodule got renamed" '
 	test_cmp expect actual
 '
 
+test_expect_success "fetch new commits on-demand when they are not reachable" '
+	git checkout --detach &&
+	C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
+	git -C submodule update-ref refs/changes/1 $C &&
+	git update-index --cacheinfo 160000 $C submodule &&
+	git commit -m "updated submodule outside of refs/heads" &&
+	D=$(git rev-parse HEAD) &&
+	git update-ref refs/changes/2 $D &&
+	(
+		cd downstream &&
+		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git -C submodule cat-file -t $C &&
+		git checkout --recurse-submodules FETCH_HEAD
+	)
+'
+
 test_done
-- 
2.19.0.444.g18242da7ef-goog


^ permalink raw reply	[relevance 24%]

* [PATCH 8/8] builtin/fetch: check for submodule updates for non branch fetches
  2018-09-21 22:35 [PATCHv3 0/8] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (5 preceding siblings ...)
  2018-09-21 22:35 ` [PATCH 7/8] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
@ 2018-09-21 22:35 ` Stefan Beller
      [irrelevant] ` <20180921223558.65055-2-sbeller@google.com>
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-21 22:35 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

Gerrit, the code review tool, has a different workflow than our mailing
list based approach. Usually users upload changes to a Gerrit server and
continuous integration and testing happens by bots. Sometimes however a
user wants to checkout a change locally and look at it locally. For this
use case, Gerrit offers a command line snippet to copy and paste to your
terminal, which looks like

  git fetch https://<host>/gerrit refs/changes/<id> &&
  git checkout FETCH_HEAD

For Gerrit changes that contain changing submodule gitlinks, it would be
easy to extend both the fetch and checkout with the '--recurse-submodules'
flag, such that this command line snippet would produce the state of a
change locally.

However the functionality added in the previous patch, which would
ensure that we fetch the objects in the submodule that the gitlink pointed
at, only works for remote tracking branches so far, not for FETCH_HEAD.

Make sure that fetching a superproject to its FETCH_HEAD, also respects
the existence checks for objects in the submodule recursion.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c             | 5 ++++-
 t/t5526-fetch-submodules.sh | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index e3b03ad3bd3..f2d9e548bf0 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -894,11 +894,14 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
 				rc |= update_local_ref(ref, what, rm, &note,
 						       summary_width);
 				free(ref);
-			} else
+			} else {
+				check_for_new_submodule_commits(&rm->old_oid);
 				format_display(&note, '*',
 					       *kind ? kind : "branch", NULL,
 					       *what ? what : "HEAD",
 					       "FETCH_HEAD", summary_width);
+			}
+
 			if (note.len) {
 				if (verbosity >= 0 && !shown_url) {
 					fprintf(stderr, _("From %.*s\n"),
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index af12c50e7dd..a509eabb044 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -615,7 +615,7 @@ test_expect_success "fetch new commits on-demand when they are not reachable" '
 	git update-ref refs/changes/2 $D &&
 	(
 		cd downstream &&
-		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git fetch --recurse-submodules origin refs/changes/2 &&
 		git -C submodule cat-file -t $C &&
 		git checkout --recurse-submodules FETCH_HEAD
 	)
-- 
2.19.0.444.g18242da7ef-goog


^ permalink raw reply	[relevance 22%]

* Re: "git rev-parse --show-superproject-working-tree" fails with "fatal: BUG: returned path string doesn't match cwd?" if supermodule has unmerged changes of the submodule reference
      [irrelevant] <EE3D88E4-EF86-44E2-811D-535C8F19C51A@gmail.com>
@ 2018-09-24 19:15 ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-24 19:15 UTC (permalink / raw)
  To: sammck; +Cc: git

On Mon, Sep 24, 2018 at 10:59 AM Sam McKelvie <sammck@gmail.com> wrote:
>
> I experienced this problem using git 2.17.1; however, from inspection of the next branch, function get_superproject_working_tree() in submodule.c has not changed in 2 years.
>
> I believe the problem is related to the fact that when a merge of the submodule reference is in progress, "git --stage —full-name <submodule-relative-path>” returns three seperate entries for the submodule (one for each stage) rather than a single entry; e.g.,
>
> $ git ls-files --stage --full-name submodule-child-test
> 160000 dbbd2766fa330fa741ea59bb38689fcc2d283ac5 1       submodule-child-test
> 160000 f174d1dbfe863a59692c3bdae730a36f2a788c51 2       submodule-child-test
> 160000 e6178f3a58b958543952e12824aa2106d560f21d 3       submodule-child-test
>
> The code in get_superproject_working_tree() uses the “-z” option on ls-files, so it expects null-byte termination between entries. However, the computation of super_sub_len:
>
>                 super_sub_len = sb.buf + sb.len - super_sub - 1;
>
> will only work when there is exactly one entry returned. If this line is changed to:
>
>                 super_sub_len = strlen(super_sub);
>
> then only the first returned entry is used, and the bug is resolved.
>
> strlen() should be safe to use here because strbuf_read ensures the result buffer is null-terminated.

This is good analysis of the issue. Thanks for writing it up!
Would you also mind to send a patch fixing the problem?

I agree that using strlen should work. I do not recall why I
did not use it at the time of writing it.

Thanks,
Stefan

^ permalink raw reply	[relevance 5%]

* Re: [PATCH v5 9/9] submodule: support reading .gitmodules when it's not in the working tree
      [irrelevant]     ` <20180924122031.9dbec6b4c2e2a8c1bff3365b@ao2.it>
@ 2018-09-24 21:00       ` Stefan Beller
  2018-09-27 14:44         ` Antonio Ospite
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-09-24 21:00 UTC (permalink / raw)
  To: Antonio Ospite
  Cc: SZEDER Gábor, Junio C Hamano, git, Brandon Williams,
	Daniel Graña, Jonathan Nieder, Richard Hartmann,
	Ævar Arnfjörð Bjarmason

On Mon, Sep 24, 2018 at 3:20 AM Antonio Ospite <ao2@ao2.it> wrote:

> > The third call, however, looks at the nested submodule at
> > 'submodule/sub', which doesn't contain a '.gitmodules' file.  So this
> > function goes on with the second condition and calls
> > get_oid(GITMODULES_INDEX, &oid), which then appears to find the blob
> > in the _superproject's_ index.
> >
>
> You are correct.
>
> This is a limitation of the object store in git, there is no equivalent
> of get_oid() to get the oid from a specific repository and this affects
> config_with_options too when the config source is a blob.

Not yet, as there is a big push to pass-through an object-store object
or similar recently and rely less on global variables.
I am not sure I get to this code, though.

> This does not affect commands called via "git -C submodule_dir cmd"
> because in that case the chdir happens before the_repository is set up,
> for instance "git-submodule $SOMETHING --recursive" commands seem to
> change the working directory before the recursion.

For this it may be worth looking into the option
       --super-prefix=<path>
  Currently for internal use only. Set a prefix which gives a
  path from above a repository down to its root. One use is
  to give submodules context about the superproject that
  invoked it.

the whole motion of moving to in-process deprecates this clunky
API to pass around strings to subprocesses.

> The test suite passes even after removing repo_read_gitmodules()
> entirely from builtin/grep.c, but I am still not confident that I get
> all the implication of why that call was originally added in commit
> f9ee2fcdfa (grep: recurse in-process using 'struct repository',
> 2017-08-02).

If you checkout that commit and remove the call to repo_read_gitmodules
and then call git-grep in a superproject with nested submodules, you
get a segfault.

On master (and deleting out that line) you do not get the segfault,
I think praise goes to ff6f1f564c4 (submodule-config: lazy-load a
repository's .gitmodules file, 2017-08-03) which happened shortly
after f9ee2fcdfa.

It showcased that it worked by converting ls-files, but left out grep.

So I think based on ff6f1f564c4 it is safe to remove all calls to
repo_read_gitmodules.

> Anyways, even if we removed the call we would prevent the problem from
> happening in the test suite, but not in the real world, in case non-leaf
> submodules without .gitmodules in their working tree.

Quite frankly I think grep was just overlooked in review of
https://public-inbox.org/git/20170803182000.179328-14-bmwill@google.com/

Stefan

^ permalink raw reply	[relevance 7%]

* Re: [PATCH v5 1/9] submodule: add a print_config_from_gitmodules() helper
      [irrelevant]   ` <20180924122502.f932da9d6b71c1f81341040a@ao2.it>
@ 2018-09-24 23:06     ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-24 23:06 UTC (permalink / raw)
  To: Antonio Ospite
  Cc: Junio C Hamano, git, Brandon Williams, Daniel Graña,
	Jonathan Nieder, Richard Hartmann,
	Ævar Arnfjörð Bjarmason

> > +int print_config_from_gitmodules(const char *key)
>
> I am thinking about adding  a "struct repository" argument to this
> function

Sounds like a good idea.

^ permalink raw reply	[relevance 5%]

* Re: [PATCH] submodule.c: Make get_superproject_working_tree() work when supermodule has unmerged changes of the submodule reference
      [irrelevant] <20180924212352.41909-1-smckelvie@xevo.com>
@ 2018-09-25  1:24 ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-25  1:24 UTC (permalink / raw)
  To: sammck; +Cc: git, smckelvie

On Mon, Sep 24, 2018 at 2:25 PM Sam McKelvie <sammck@gmail.com> wrote:

Thanks for writing a patch!
I wonder if we can shorten the subject and make it a bit more concise,
how about

    submodule: Alllow staged changes for get_superproject_working_tree

or

    rev-parse: allow staged submodule in --show-superproject-working-tree


>     Previously, "fatal: BUG: returned path string doesn't match cwd?" is displayed and
>     git-rev-parse aborted.

Usually it is hard to read, continuing from the commit title to the body
of the commit message, as the title may be displayed differently or
somewhere else, so it would be good to restate it or rather state the
command that is broken, which we are trying to fix.

    Invoking 'git rev-parse --show-superproject-working-tree' exits with

        fatal: BUG ...

    instead of displaying the superproject working tree when ....

>     The problem is due to the fact that when a merge of the submodule reference is in progress,
>     "git --stage —full-name <submodule-relative-path>” returns three seperate entries for the

"ls-files" is missing in that git invocation?

>     submodule (one for each stage) rather than a single entry; e.g.,
>
>     $ git ls-files --stage --full-name submodule-child-test
>     160000 dbbd2766fa330fa741ea59bb38689fcc2d283ac5 1       submodule-child-test
>     160000 f174d1dbfe863a59692c3bdae730a36f2a788c51 2       submodule-child-test
>     160000 e6178f3a58b958543952e12824aa2106d560f21d 3       submodule-child-test
>
>     The code in get_superproject_working_tree() expected exactly one entry to be returned;
>     this patch makes it use the first entry if multiple entries are returned.

The code looks good.

I wonder if we want to add a test for it, see t/t1500-rev-parse.sh
(at the very end 'showing the superproject correctly').

>
>     Signed-off-by: Sam McKelvie <smckelvie@xevo.com>

Side note: the commit message seems to be indented,
but the patch applies fine, which makes me curious:
How did you produce&send the patch?
(Usually a patch would not apply as white spaces are
mangled in the code for example in some email setups)
(If I had to guess I could imagine that it is the output of
git-show as that indents the commit message usually,
see git-format-patch for a better tool)

Thanks for writing the patch!
Stefan

^ permalink raw reply	[relevance 7%]

* Re: [PATCH 2/8] Add a place for (not) sharing stuff between worktrees
      [irrelevant]     ` <CACsJy8BKTkbc=ZgMnO7Yuk0eaqzZnifo80tnR872_T8b02biqg@mail.gmail.com>
@ 2018-09-25 16:24       ` Stefan Beller
  2018-09-25 16:55         ` Duy Nguyen
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-09-25 16:24 UTC (permalink / raw)
  To: Duy Nguyen; +Cc: git, Elijah Newren, Jeff King

On Tue, Sep 25, 2018 at 8:36 AM Duy Nguyen <pclouds@gmail.com> wrote:
>
> On Tue, Sep 25, 2018 at 4:35 AM Stefan Beller <sbeller@google.com> wrote:
> >
> > On Sat, Sep 22, 2018 at 11:05 AM Nguyễn Thái Ngọc Duy <pclouds@gmail.com> wrote:
> > >
> > > When multiple worktrees are used, we need rules to determine if
> > > something belongs to one worktree or all of them. Instead of keeping
> > > adding rules when new stuff comes, have a generic rule:
> > >
> > > - Inside $GIT_DIR, which is per-worktree by default, add
> > >   $GIT_DIR/common which is always shared. New features that want to
> > >   share stuff should put stuff under this directory.
> >
> > So that /common is a directory and you have to use it specifically
> > in new code? That would be easy to overlook when coming up
> > with $GIT_DIR/foo for implementing the git-foo.
>
> There's no easy way out. I have to do _something_ if you want to share
> $GIT_DIR/foo to all worktrees. Either we have to update path.c and add
> "foo" which is not even an option for external commands, or we put
> "foo" in a common place, e.g. $GIT_DIR/common/foo.
>
> > > - Inside refs/, which is shared by default except refs/bisect, add
> > >   refs/local/ which is per-worktree. We may eventually move
> > >   refs/bisect to this new location and remove the exception in refs
> > >   code.
> >
> > That sounds dangerous to me. There is already a concept of
> > local and remote-tracking branches. So I would think that local
> > may soon become an overused word, (just like "index" today or
> > "recursive" to a lesser extend).
> >
> > Could this special area be more explicit?
> > (refs/worktree-local/ ? or after peeking at the docs below
> > refs/un-common/ ?)
>
> refs/un-common sounds really "uncommon" :D. If refs/local is bad, I
> guess we could go with either refs/worktree-local, refs/worktree,
> refs/private, refs/per-worktree... My vote is on refs/worktree. I

refs/worktree sounds good to me (I do not object), but I am not
overly enthused either, as when I think further worktrees and
submodules are both features with a very similar nature in that
they touch a lot of core concepts in Git, but seem to be a niche
feature for the masses for now.

For example I could think of submodules following this addressing
mode as well: submodule/<path>/master sounds similar to the
originally proposed worktree/<name>/<branch> convention.
For now it is not quite clear to me why you would want to have
access to the submodule refs in the superproject, but maybe
the use case will come later.

And with that said, I wonder if the "local" part should be feature agnostic,
or if we want to be "local" for worktrees, "local" for remotes, "local"
for submodules (i.e. our own refs vs submodule refs).

> think as long as the word "worktree" is in there, people would notice
> the difference.

That makes sense. But is refs/worktree shared or local? It's not quite
obvious to me, as I could have refs/worktree/<worktree-name>/master
instead when it is shared, so I tend to favor refs/local-worktree/ a bit
more, but that is more typing. :/

==
As we grow the worktree feature, do we ever expect the need to
reference the current worktree?

For example when there is a ref "test" that could be unique per
repo and in the common area, so refs/heads/test would describe
it and "test" would get there in DWIM mode.

But then I could also delete the common ref and recreate a "test"
ref in worktree A, in worktree B however DWIMming "test" could still
refer to A's "test" as it is unique (so far) in the repository.
And maybe I would want to check if test exists locally, so I'd
want to ask for "self/test" (with "self" == "B" as that is my cwd).

Stefan

^ permalink raw reply	[relevance 5%]

* Re: [PATCH 2/8] Add a place for (not) sharing stuff between worktrees
  2018-09-25 16:24       ` [PATCH 2/8] Add a place for (not) sharing stuff between worktrees Stefan Beller
@ 2018-09-25 16:55         ` Duy Nguyen
  2018-09-25 17:56           ` Stefan Beller
  0 siblings, 1 reply; 200+ results
From: Duy Nguyen @ 2018-09-25 16:55 UTC (permalink / raw)
  To: Stefan Beller; +Cc: Git Mailing List, Elijah Newren, Jeff King

On Tue, Sep 25, 2018 at 6:24 PM Stefan Beller <sbeller@google.com> wrote:
> > > That sounds dangerous to me. There is already a concept of
> > > local and remote-tracking branches. So I would think that local
> > > may soon become an overused word, (just like "index" today or
> > > "recursive" to a lesser extend).
> > >
> > > Could this special area be more explicit?
> > > (refs/worktree-local/ ? or after peeking at the docs below
> > > refs/un-common/ ?)
> >
> > refs/un-common sounds really "uncommon" :D. If refs/local is bad, I
> > guess we could go with either refs/worktree-local, refs/worktree,
> > refs/private, refs/per-worktree... My vote is on refs/worktree. I
>
> refs/worktree sounds good to me (I do not object), but I am not
> overly enthused either, as when I think further worktrees and
> submodules are both features with a very similar nature in that
> they touch a lot of core concepts in Git, but seem to be a niche
> feature for the masses for now.

I think the similarity is partly because submodule feature also has to
manage worktrees. My view is at some point, this "git worktree" would
be good enough that it can handle submodules as well (for the worktree
part only of course)

> For example I could think of submodules following this addressing
> mode as well: submodule/<path>/master sounds similar to the
> originally proposed worktree/<name>/<branch> convention.
> For now it is not quite clear to me why you would want to have
> access to the submodule refs in the superproject, but maybe
> the use case will come later.

Yeah. In theory we could "mount" the submodule ref store to a
superproject's ref store. I think it may be needed just for the same
reason it's needed for worktree: error reporting. If you peek into a
submodule and say "HEAD has an error", the user will get confused
whether it's superproject's HEAD or a submodule's HEAD.

> And with that said, I wonder if the "local" part should be feature agnostic,
> or if we want to be "local" for worktrees, "local" for remotes, "local"
> for submodules (i.e. our own refs vs submodule refs).

You lost me here.

>
> > think as long as the word "worktree" is in there, people would notice
> > the difference.
>
> That makes sense. But is refs/worktree shared or local? It's not quite
> obvious to me, as I could have refs/worktree/<worktree-name>/master
> instead when it is shared, so I tend to favor refs/local-worktree/ a bit
> more, but that is more typing. :/

OK I think mixing the two patches will different purposes messes you
(or me) up ;-)

refs/worktrees/xxx (and refs/main/xxx) are about visibility from other
worktrees. Or like Eric put it, they are simply aliases. These refs
are not shared because if they are, you can already see them without
new "ref mount points" like this.

refs/worktree (previously refs/local) is also per-worktree but it's
specifically because you can't have per-worktree inside "refs/" (the
only exception so far is refs/bisect which is hard coded). You can
have refs outside "refs/" (like HEAD or FETCH_HEAD) and they will not
be shared, but they cannot be iterated while those inside refs/ can
be. This is more about deciding what to share and I believe is really
worktree-specific and only matters to _current_ worktree.

Since refs/worktree is per-worktree, you can also view them from a
different worktree via refs/worktrees/. E.g. if you have
refs/worktree/foo then another worktree can see it via
refs/worktrees/xxx/refs/worktree/foo (besides pseudo refs like
refs/worktrees/xxx/HEAD)

> As we grow the worktree feature, do we ever expect the need to
> reference the current worktree?
>
> For example when there is a ref "test" that could be unique per
> repo and in the common area, so refs/heads/test would describe
> it and "test" would get there in DWIM mode.
>
> But then I could also delete the common ref and recreate a "test"
> ref in worktree A, in worktree B however DWIMming "test" could still
> refer to A's "test" as it is unique (so far) in the repository.
> And maybe I would want to check if test exists locally, so I'd
> want to ask for "self/test" (with "self" == "B" as that is my cwd).

You probably lost me again. In theory we must be able to detect
ambiguity and stop DWIMing. If you want to be ambiguity-free, you
specify full ref name, starting with "refs/" which should function
like "self/" because worktree design so far is always about the
current worktree's view.
-- 
Duy

^ permalink raw reply	[relevance 5%]

* Re: [PATCH 2/8] Add a place for (not) sharing stuff between worktrees
  2018-09-25 16:55         ` Duy Nguyen
@ 2018-09-25 17:56           ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-25 17:56 UTC (permalink / raw)
  To: Duy Nguyen; +Cc: git, Elijah Newren, Jeff King

On Tue, Sep 25, 2018 at 9:55 AM Duy Nguyen <pclouds@gmail.com> wrote:

> > And with that said, I wonder if the "local" part should be feature agnostic,
> > or if we want to be "local" for worktrees, "local" for remotes, "local"
> > for submodules (i.e. our own refs vs submodule refs).
>
> You lost me here.

Yeah, me too after rereading. :P

I think the "local" part always implies that there is a part that is
not local and depending on the feature you call it remote or other
worktree.

When writing this comment I briefly wondered if we want to combine
the local aspects of the various features.
However the "local" part really depends on the feature
(e.g. a ref on a different worktree is still local from the here/remote
perspective or from the superproject/submodule perspective),
so I think I was misguided.

> > > think as long as the word "worktree" is in there, people would notice
> > > the difference.
> >
> > That makes sense. But is refs/worktree shared or local? It's not quite
> > obvious to me, as I could have refs/worktree/<worktree-name>/master
> > instead when it is shared, so I tend to favor refs/local-worktree/ a bit
> > more, but that is more typing. :/
>
> OK I think mixing the two patches will different purposes messes you
> (or me) up ;-)

possible.

>
> refs/worktrees/xxx (and refs/main/xxx) are about visibility from other
> worktrees. Or like Eric put it, they are simply aliases. These refs
> are not shared because if they are, you can already see them without
> new "ref mount points" like this.
>
> refs/worktree (previously refs/local) is also per-worktree but it's
> specifically because you can't have per-worktree inside "refs/" (the
> only exception so far is refs/bisect which is hard coded). You can
> have refs outside "refs/" (like HEAD or FETCH_HEAD) and they will not
> be shared, but they cannot be iterated while those inside refs/ can
> be. This is more about deciding what to share and I believe is really
> worktree-specific and only matters to _current_ worktree.
>
> Since refs/worktree is per-worktree, you can also view them from a
> different worktree via refs/worktrees/. E.g. if you have
> refs/worktree/foo then another worktree can see it via
> refs/worktrees/xxx/refs/worktree/foo (besides pseudo refs like
> refs/worktrees/xxx/HEAD)

Ah. now I seem to understand, thanks for explaining.

^ permalink raw reply	[relevance 4%]

* Git Test Coverage Report (Tuesday, Sept 25)
@ 2018-09-25 18:42 Derrick Stolee
  0 siblings, 0 replies; 200+ results
From: Derrick Stolee @ 2018-09-25 18:42 UTC (permalink / raw)
  To: Git List

In an effort to ensure new code is reasonably covered by the test suite, 
we now have contrib/coverage-diff.sh to combine the gcov output from 
'make coverage-test ; make coverage-report' with the output from 'git 
diff A B' to discover _new_ lines of code that are not covered.

This report takes the output of these results after running on four 
branches:

         pu: 80e728fa913dc3a1165b6dec9a7afa6052a86325

        jch: 0c10634844314ab89666ed0a1c7d36dde7ac9689

       next: 76f2f5c1e34c4dbef1029e2984c2892894c444ce

     master: fe8321ec057f9231c26c29b364721568e58040f7

master@{1}: 2d3b1c576c85b7f5db1f418907af00ab88e0c303

I ran the test suite on each of these branches on an Ubuntu Linux VM, 
and I'm missing some dependencies (like apache, svn, and perforce) so 
not all tests are run.

I submit this output without comment. I'm taking a look especially at my 
own lines to see where coverage can be improved.

Thanks,

-Stolee

---

master@{1}..master:

builtin/remote.c
5025425dfff     (   Shulhan     2018-09-13 20:18:33 +0700 
864)            return error(_("No such remote: '%s'"), name);
commit-reach.c
b67f6b26e35     (Derrick Stolee 2018-09-21 08:05:26 -0700 
559)                    continue;
b67f6b26e35     (Derrick Stolee 2018-09-21 08:05:26 -0700 
569)                    from->objects[i].item->flags |= assign_flag;
b67f6b26e35     (Derrick Stolee 2018-09-21 08:05:26 -0700 
570)                    continue;
b67f6b26e35     (Derrick Stolee 2018-09-21 08:05:26 -0700 
576)                    result = 0;
b67f6b26e35     (Derrick Stolee 2018-09-21 08:05:26 -0700 
577)                    goto cleanup;
cat: compat#mingw.c.gcov: No such file or directory
ll-merge.c
d64324cb60e     (Torsten Bögershausen   2018-09-12 21:32:02 +0200       
379)                    marker_size = DEFAULT_CONFLICT_MARKER_SIZE;
remote-curl.c
c3b9bc94b9b     (Elijah Newren  2018-09-05 10:03:07 -0700 
181)            options.filter = xstrdup(value);

master..next:

fsck.c
fb8952077df     (René Scharfe   2018-09-03 14:49:26 +0000 
212)            die_errno("Could not read '%s'", path);
midx.c
56ee7ff1565     (Derrick Stolee 2018-09-13 11:02:13 -0700 
949)            return 0;
cc6af73c029     (Derrick Stolee 2018-09-13 11:02:25 -0700 
990)                    midx_report(_("failed to load pack-index for 
packfile %s"),
cc6af73c029     (Derrick Stolee 2018-09-13 11:02:25 -0700 
991)                                e.p->pack_name);
cc6af73c029     (Derrick Stolee 2018-09-13 11:02:25 -0700 
992)                    break;

next..jch:

blame.c
a470beea39b     (Nguyễn Thái Ngọc Duy   2018-09-21 17:57:21 +0200       
113)             !strcmp(r->index->cache[-1 - pos]->name, path))
a470beea39b     (Nguyễn Thái Ngọc Duy   2018-09-21 17:57:21 +0200       
272)            int pos = index_name_pos(r->index, path, len);
a470beea39b     (Nguyễn Thái Ngọc Duy   2018-09-21 17:57:21 +0200       
274)                    mode = r->index->cache[pos]->ce_mode;
commit-graph.c
5cef295f283     (Derrick Stolee 2018-08-20 18:24:32 +0000 
67)             return 0;
20fd6d57996     (Derrick Stolee 2018-08-20 18:24:30 +0000 
79)             return 0;
config.c
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2301)           if (is_bool)
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2302)                   return val ? 0 : 1;
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2304)                   return val;
diff.c
b78ea5fc357     (Nguyễn Thái Ngọc Duy   2018-09-21 17:57:19 +0200       
4117) add_external_diff_name(o->repo, &argv, other, two);
http-push.c
2abf3503854     (Nguyễn Thái Ngọc Duy   2018-09-21 17:57:38 +0200       
1928)           repo_init_revisions(the_repository, &revs, 
setup_git_directory());
list-objects-filter-options.c
7c8a0cecc49     (Matthew DeVore 2018-09-21 13:32:04 -0700 
55)                     if (errbuf) {
7c8a0cecc49     (Matthew DeVore 2018-09-21 13:32:04 -0700 
56)                             strbuf_addstr(
7c8a0cecc49     (Matthew DeVore 2018-09-21 13:32:04 -0700 
60)                     return 1;
ba72cca605f     (Matthew DeVore 2018-09-21 13:32:03 -0700 86)     if 
(errbuf)
list-objects-filter.c
22e9b63e620     (Matthew DeVore 2018-09-21 13:32:02 -0700 
47)             BUG("unknown filter_situation: %d", filter_situation);
7c8a0cecc49     (Matthew DeVore 2018-09-21 13:32:04 -0700 100)    default:
7c8a0cecc49     (Matthew DeVore 2018-09-21 13:32:04 -0700 
101)            BUG("unknown filter_situation: %d", filter_situation);
22e9b63e620     (Matthew DeVore 2018-09-21 13:32:02 -0700 
152)            BUG("unknown filter_situation: %d", filter_situation);
22e9b63e620     (Matthew DeVore 2018-09-21 13:32:02 -0700 
257)            BUG("unknown filter_situation: %d", filter_situation);
22e9b63e620     (Matthew DeVore 2018-09-21 13:32:02 -0700 
438)            BUG("invalid list-objects filter choice: %d",
list-objects.c
f447a499dbb     (Matthew DeVore 2018-08-13 11:14:28 -0700 
197)                    ctx->show_object(obj, base->buf, ctx->show_data);
preload-index.c
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
73)                     struct progress_data *pd = p->progress;
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
75) pthread_mutex_lock(&pd->mutex);
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
76)                     pd->n += last_nr - nr;
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
77) display_progress(pd->progress, pd->n);
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
78) pthread_mutex_unlock(&pd->mutex);
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
79)                     last_nr = nr;
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
93)             struct progress_data *pd = p->progress;
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
95) pthread_mutex_lock(&pd->mutex);
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
96)             display_progress(pd->progress, pd->n + last_nr);
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
97) pthread_mutex_unlock(&pd->mutex);
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
128)            pd.progress = start_delayed_progress(_("Refreshing 
index"), index->cache_nr);
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
129)            pthread_mutex_init(&pd.mutex, NULL);
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
140)                    p->progress = &pd;
prio-queue.c
2d181390f3c     (Derrick Stolee 2018-09-17 21:08:43 -0700 
94)             return queue->array[queue->nr - 1].data;
read-cache.c
ae9af12287b     (Nguyễn Thái Ngọc Duy   2018-09-15 19:56:04 +0200       
1535)                   display_progress(progress, i);
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1754)   strbuf_setlen(name, name->len - len);
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1755)   ep = cp + strlen((const char *)cp);
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1792)           const unsigned char *cp = (const unsigned char *)name;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1795)           previous_len = previous_ce ? previous_ce->ce_namelen : 0;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1796)           strip_len = decode_varint(&cp);
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1797)           if (previous_len < strip_len) {
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1798)                   if (previous_ce)
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1799)                           die(_("malformed name field in the 
index, near path '%s'"),
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1800) previous_ce->name);
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1802)                           die(_("malformed name field in the index 
in the first path"));
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1804)           copy_len = previous_len - strip_len;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1805)           name = (const char *)cp;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1829)           memcpy(ce->name, previous_ce->name, copy_len);
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1830)           memcpy(ce->name + copy_len, name, len + 1 - copy_len);
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
1831)           *ent_size = (name - ((char *)ondisk)) + len + 1 - copy_len;
9928a34be6b     ( Ben Peart     2018-09-12 16:18:59 +0000 
1954)                   munmap((void *)p->mmap, p->mmap_size);
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
1955)                   die("index file corrupt");
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
1996)           mem_pool_init(&istate->ce_mem_pool,
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2034)static 
void *load_cache_entries_thread(void *_data)
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2036)   struct 
load_cache_entries_thread_data *p = _data;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2038)   
p->consumed += load_cache_entry_block(p->istate, p->ce_mem_pool,
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2039)           p->offset, p->nr, p->mmap, p->start_offset, p->previous_ce);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2040)   return 
NULL;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2043)static 
unsigned long load_cache_entries_threaded(int nr_threads, struct 
index_state *istate,
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2046)   struct 
strbuf previous_name_buf = STRBUF_INIT, *previous_name;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2053)   if 
(istate->name_hash_initialized)
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2054)           BUG("the name hash isn't thread safe");
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2056)   
mem_pool_init(&istate->ce_mem_pool, 0);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2057)   if 
(istate->version == 4)
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2058)           previous_name = &previous_name_buf;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2060)           previous_name = NULL;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2062)   
ce_per_thread = DIV_ROUND_UP(istate->cache_nr, nr_threads);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2063)   data = 
xcalloc(nr_threads, sizeof(struct load_cache_entries_thread_data));
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2070)   
consumed = thread = 0;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2071)   for (i 
= 0; ; i++) {
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2080)           if (i % ce_per_thread == 0) {
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2081)                   struct load_cache_entries_thread_data *p = 
&data[thread];
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2083)                   p->istate = istate;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2084)                   p->offset = i;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2085)                   p->nr = ce_per_thread < istate->cache_nr - i ? 
ce_per_thread : istate->cache_nr - i;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2086)                   p->mmap = mmap;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2087)                   p->start_offset = src_offset;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2090)                   if (istate->version == 4) {
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2091) 
mem_pool_init(&p->ce_mem_pool,
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2092) 
estimate_cache_size_from_compressed(p->nr));
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2095)                           if (previous_name->len) {
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2096) p->previous_ce = mem_pool__ce_alloc(p->ce_mem_pool, 
previous_name->len);
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2097) p->previous_ce->ce_namelen = previous_name->len;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2098) memcpy(p->previous_ce->name, previous_name->buf, previous_name->len);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2101) 
mem_pool_init(&p->ce_mem_pool,
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2102) 
estimate_cache_size(mmap_size, p->nr));
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2105)                   if (pthread_create(&p->pthread, NULL, 
load_cache_entries_thread, p))
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2106)                           die("unable to create 
load_cache_entries_thread");
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2109)                   if (++thread == nr_threads)
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2110)                           break;
9928a34be6b     ( Ben Peart     2018-09-12 16:18:59 +0000 
2113)           ondisk = (struct ondisk_cache_entry *)(mmap + src_offset);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2116)           flags = get_be16(&ondisk->flags);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2118)           if (flags & CE_EXTENDED) {
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2120)                   ondisk2 = (struct ondisk_cache_entry_extended 
*)ondisk;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2121)                   name = ondisk2->name;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2123)                   name = ondisk->name;
13e1b78757a     (Nguyễn Thái Ngọc Duy   2018-09-12 16:18:57 +0000       
2125)           if (istate->version != 4) {
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2129)                   len = flags & CE_NAMEMASK;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2130)                   if (len == CE_NAMEMASK)
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2131)                           len = strlen(name);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2132)                   src_offset += (flags & CE_EXTENDED) ?
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2133) 
ondisk_cache_entry_extended_size(len) :
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2134)                           ondisk_cache_entry_size(len);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2136)                   src_offset += (name - ((char *)ondisk)) + 
expand_name_field(previous_name, name);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2139)   for (i 
= 0; i < nr_threads; i++) {
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2140)           struct load_cache_entries_thread_data *p = data + i;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2141)           if (pthread_join(p->pthread, NULL))
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2142)                   die("unable to join load_cache_entries_thread");
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2143)           mem_pool_combine(istate->ce_mem_pool, p->ce_mem_pool);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2144)           consumed += p->consumed;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2147)   
free(data);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2148)   
strbuf_release(&previous_name_buf);
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 2150)   return 
consumed;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2217)                   nr_threads = cpus;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2222)           nr_threads = 3;
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2225)           extension_offset = read_eoie_extension(mmap, mmap_size);
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2226)           if (extension_offset) {
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2228)                   p.src_offset = extension_offset;
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2229)                   if (pthread_create(&p.pthread, NULL, 
load_index_extensions, &p))
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2230)                           die(_("unable to create 
load_index_extensions_thread"));
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2231)                   nr_threads--;
ad570c680bc     ( Ben Peart     2018-09-12 16:18:56 +0000 
2235)           src_offset += load_cache_entries_threaded(nr_threads, 
istate, mmap, mmap_size, src_offset);
4ee45bacad0     ( Ben Peart     2018-09-12 16:18:55 +0000 
2248)           die(_("unable to join load_index_extensions_thread"));
9928a34be6b     ( Ben Peart     2018-09-12 16:18:59 +0000 3288)static 
unsigned long read_eoie_extension(const char *mmap, size_t mmap_size)
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3306)   if 
(mmap_size < sizeof(struct cache_header) + EOIE_SIZE_WITH_HEADER + 
the_hash_algo->rawsz)
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3307)           return 0;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3310)   index 
= eoie = mmap + mmap_size - EOIE_SIZE_WITH_HEADER - the_hash_algo->rawsz;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3311)   if 
(CACHE_EXT(index) != CACHE_EXT_ENDOFINDEXENTRIES)
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3312)           return 0;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3313)   index 
+= sizeof(uint32_t);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3316)   
extsize = get_be32(index);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3317)   if 
(extsize != EOIE_SIZE)
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3318)           return 0;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3319)   index 
+= sizeof(uint32_t);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3325)   offset 
= get_be32(index);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3326)   if 
(mmap + offset < mmap + sizeof(struct cache_header))
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3327)           return 0;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3328)   if 
(mmap + offset >= eoie)
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3329)           return 0;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3330)   index 
+= sizeof(uint32_t);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3341)   
src_offset = offset;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3342)   
the_hash_algo->init_fn(&c);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3343)   while 
(src_offset < mmap_size - the_hash_algo->rawsz - EOIE_SIZE_WITH_HEADER) {
9928a34be6b     ( Ben Peart     2018-09-12 16:18:59 +0000 
3351)           extsize = get_be32(mmap + src_offset + 4);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3354)           if (src_offset + 8 + extsize < src_offset)
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3355)                   return 0;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3357)           the_hash_algo->update_fn(&c, mmap + src_offset, 8);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3359)           src_offset += 8;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3360)           src_offset += extsize;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3362)   
the_hash_algo->final_fn(hash, &c);
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3363)   if 
(hashcmp(hash, (const unsigned char *)index))
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3364)           return 0;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3367)   if 
(src_offset != mmap_size - the_hash_algo->rawsz - EOIE_SIZE_WITH_HEADER)
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 
3368)           return 0;
8d44e69943e     ( Ben Peart     2018-09-12 16:18:53 +0000 3370)   return 
offset;
rebase-interactive.c
64a43cbd5da     (Alban Gruin    2018-08-10 18:51:31 +0200 
61)             return error_errno(_("could not read '%s'."), todo_file);
64a43cbd5da     (Alban Gruin    2018-08-10 18:51:31 +0200 
65)             strbuf_release(&buf);
64a43cbd5da     (Alban Gruin    2018-08-10 18:51:31 +0200 
66)             return -1;
a9f5476fbca     (Alban Gruin    2018-08-10 18:51:35 +0200 
74)             return error_errno(_("could not read '%s'."), todo_file);
a9f5476fbca     (Alban Gruin    2018-08-10 18:51:35 +0200 
78)             strbuf_release(&buf);
a9f5476fbca     (Alban Gruin    2018-08-10 18:51:35 +0200 
79)             return -1;
64a43cbd5da     (Alban Gruin    2018-08-10 18:51:31 +0200 
85)             return -1;
refs.c
4a6067cda51     (Stefan Beller  2018-08-20 18:24:16 +0000 
1405)           return 0;
revision.c
2abf3503854     (Nguyễn Thái Ngọc Duy   2018-09-21 17:57:38 +0200       
1529)           if (ce_path_match(istate, ce, &revs->prune_data, NULL)) {
2abf3503854     (Nguyễn Thái Ngọc Duy   2018-09-21 17:57:38 +0200       
1535)           while ((i+1 < istate->cache_nr) &&
2abf3503854     (Nguyễn Thái Ngọc Duy   2018-09-21 17:57:38 +0200       
1536)                  ce_same_name(ce, istate->cache[i+1]))
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
2931)           return;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
2934)           return;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
2937)           c->object.flags |= UNINTERESTING;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
2940)           return;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
2943)           mark_parents_uninteresting(c);
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
2966)           return;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
2969)           return;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
2974)           return;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 3018)   case 
REV_SORT_BY_COMMIT_DATE:
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
3019)           info->topo_queue.compare = compare_commits_by_commit_date;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
3020)           break;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 3021)   case 
REV_SORT_BY_AUTHOR_DATE:
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 3022) 
init_author_date_slab(&info->author_date);
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
3023)           info->topo_queue.compare = compare_commits_by_author_date;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
3024)           info->topo_queue.cb_data = &info->author_date;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
3025)           break;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
3038)                   continue;
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 3048) 
record_author_date(&info->author_date, c);
6c04ff30019     (Derrick Stolee 2018-09-21 10:39:32 -0700 
3086)           if (!revs->ignore_missing_links)
6c04ff30019     (Derrick Stolee 2018-09-21 10:39:32 -0700 
3087)                   die("Failed to traverse parents of commit %s",
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 3088) 
oid_to_hex(&commit->object.oid));
4943d288495     (Derrick Stolee 2018-09-21 10:39:36 -0700 
3096)                   continue;
sequencer.c
65850686cf0     (Alban Gruin    2018-08-28 14:10:40 +0200 
2276)           return;
65850686cf0     (Alban Gruin    2018-08-28 14:10:40 +0200 
2373)           write_file(rebase_path_quiet(), "%s\n", quiet);
2c58483a598     (Alban Gruin    2018-08-10 18:51:33 +0200 
3371)                   return error(_("could not checkout %s"), commit);
4df66c40b08     (Alban Gruin    2018-08-10 18:51:34 +0200 
3385)           return error(_("%s: not a valid OID"), orig_head);
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4750)           return -1;
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4753)           return -1;
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4759)           return error_errno(_("could not read '%s'."), todo_file);
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4762)           todo_list_release(&todo_list);
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4763)           return error(_("unusable todo list: '%s'"), todo_file);
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4782)           todo_list_release(&todo_list);
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4783)           return -1;
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4787)           return error(_("could not copy '%s' to '%s'."), todo_file,
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4791)           return error(_("could not transform the todo list"));
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4820)           return error(_("could not transform the todo list"));
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4823)           return error(_("could not skip unnecessary pick commands"));
b97e1873649     (Alban Gruin    2018-08-28 14:10:36 +0200 
4829)           return -1;
strbuf.c
f95736288a3     (Pratik Karki   2018-08-08 20:51:16 +0545 
127)                    --sb->len;
wt-status.c
f3bd35fa0dd     (Stephen P. Smith       2018-09-05 17:53:29 -0700       
671)                    s->committable = 1;

jch..pu:

builtin/bisect--helper.c
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
43)             free((void *) terms->term_good);
3d3237b0e6b     (Pranit Bauva   2017-10-27 15:06:37 +0000 
162)            if (get_oid_commit(commit, &oid))
3d3237b0e6b     (Pranit Bauva   2017-10-27 15:06:37 +0000 
163)                    return error(_("'%s' is not a valid commit"), 
commit);
3d3237b0e6b     (Pranit Bauva   2017-10-27 15:06:37 +0000 
164)            strbuf_addstr(&branch, commit);
3d3237b0e6b     (Pranit Bauva   2017-10-27 15:06:37 +0000 
172)                    error(_("Could not check out original HEAD '%s'. 
Try "
3d3237b0e6b     (Pranit Bauva   2017-10-27 15:06:37 +0000 
174)                    strbuf_release(&branch);
3d3237b0e6b     (Pranit Bauva   2017-10-27 15:06:37 +0000 
175)                    argv_array_clear(&argv);
3d3237b0e6b     (Pranit Bauva   2017-10-27 15:06:37 +0000 
176)                    return -1;
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
215)            error(_("Bad bisect_write argument: %s"), state);
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
216)            goto fail;
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
220)            error(_("couldn't get the oid of the rev '%s'"), rev);
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
221)            goto fail;
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
226)            goto fail;
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
230)            error_errno(_("couldn't open the file '%s'"), 
git_path_bisect_log());
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
231)            goto fail;
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 242)fail:
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 243)    retval 
= -1;
a919f328ba3     (Pranit Bauva   2017-10-27 15:06:37 +0000 
323)            yesno = git_prompt(_("Are you sure [Y/n]? "), PROMPT_ECHO);
a919f328ba3     (Pranit Bauva   2017-10-27 15:06:37 +0000 
324)            if (starts_with(yesno, "N") || starts_with(yesno, "n"))
a919f328ba3     (Pranit Bauva   2017-10-27 15:06:37 +0000 
327)            goto finish;
a919f328ba3     (Pranit Bauva   2017-10-27 15:06:37 +0000 
336)            error(_("You need to start by \"git bisect start\". You "
a919f328ba3     (Pranit Bauva   2017-10-27 15:06:37 +0000 
341)            goto fail;
a919f328ba3     (Pranit Bauva   2017-10-27 15:06:37 +0000 345)fail:
35f7ca528ae     (Pranit Bauva   2017-10-27 15:06:37 +0000 
387)            return error(_("--bisect-term requires exactly one 
argument"));
35f7ca528ae     (Pranit Bauva   2017-10-27 15:06:37 +0000 
400)                    error(_("BUG: invalid argument %s for 'git 
bisect terms'.\n"
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
416)            return -1;
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
419)            goto fail;
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
423)            goto fail;
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 427)fail:
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 428)    retval 
= -1;
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
466)                    no_checkout = 1;
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
488)                     !one_of(arg, "--term-good", "--term-bad", NULL)) {
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
489)                    return error(_("unrecognised option: '%s'"), arg);
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
523)            if (get_oid("HEAD", &head_oid))
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
524)                    return error(_("Bad HEAD - I need a HEAD"));
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
539)                            error(_("checking out '%s' failed. Try 
'git "
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
559)                            return error(_("won't bisect on 
cg-seek'ed tree"));
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
562)                    return error(_("Bad HEAD - strange symbolic ref"));
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
570)            return -1;
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
588)                    goto fail;
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
598)                    goto fail;
5dfeec316b8     (Pranit Bauva   2017-10-27 15:06:37 +0000 
606)            goto fail;
3d3237b0e6b     (Pranit Bauva   2017-10-27 15:06:37 +0000 
686)                    return error(_("--bisect-reset requires either 
no argument or a commit"));
0b1f0fd910c     (Pranit Bauva   2017-10-27 15:06:37 +0000 
690)                    return error(_("--bisect-write requires either 4 
or 5 arguments"));
20edf353b72     (Pranit Bauva   2017-10-27 15:06:37 +0000 
697)                    return error(_("--check-and-set-terms requires 3 
arguments"));
a919f328ba3     (Pranit Bauva   2017-10-27 15:06:37 +0000 
703)                    return error(_("--bisect-next-check requires 2 
or 3 arguments"));
builtin/blame.c
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 922)    case 
DATE_HUMAN:
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
924)            blame_date_width = sizeof("Thu Oct 19 16:00");
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
925)            break;
builtin/gc.c
3029970275b     (Jonathan Nieder        2018-07-16 23:57:40 -0700       
461)            ret = error_errno(_("cannot stat '%s'"), gc_log_path);
3029970275b     (Jonathan Nieder        2018-07-16 23:57:40 -0700       
462)            goto done;
3029970275b     (Jonathan Nieder        2018-07-16 23:57:40 -0700       
470)            ret = error_errno(_("cannot read '%s'"), gc_log_path);
3029970275b     (Jonathan Nieder        2018-07-16 23:57:40 -0700       
585)                            exit(128);
builtin/submodule--helper.c
df255b8cac7     (Brandon Williams       2018-08-08 15:33:22 -0700       
914)            usage(_("git submodule--helper gitdir <name>"));
date.c
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
113)            die("Timestamp too large for this system: %"PRItime, time);
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
216)            if (tm->tm_mon == human_tm->tm_mon) {
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
217)                    if (tm->tm_mday > human_tm->tm_mday) {
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
219)                    } else if (tm->tm_mday == human_tm->tm_mday) {
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
220)                            hide.date = hide.wday = 1;
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
221)                    } else if (tm->tm_mday + 5 > human_tm->tm_mday) {
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
223)                            hide.date = 1;
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
231)            gettimeofday(&now, NULL);
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
232)            show_date_relative(time, tz, &now, buf);
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
233)            return;
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
246)            hide.seconds = 1;
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
247)            hide.tz |= !hide.date;
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
248)            hide.wday = hide.time = !hide.year;
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
262)            strbuf_rtrim(buf);
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
287)            gettimeofday(&now, NULL);
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
290)            human_tz = local_time_tzoffset(now.tv_sec, &human_tm);
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 886)static int 
auto_date_style(void)
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 888)    return 
(isatty(1) || pager_in_use()) ? DATE_HUMAN : DATE_NORMAL;
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
909)            return DATE_HUMAN;
74e8221b523     (Linus Torvalds 2018-07-07 15:02:35 -0700 
911)            return auto_date_style();
sha1-array.c
7007a318a68     (Stefan Beller  2018-09-11 16:49:44 -0700 
90)                             oidcpy(&array->oid[dst], &array->oid[src]);
string-list.c
6fecf7cd01a     (Stefan Beller  2018-09-11 16:49:43 -0700 
86)             BUG("tried to remove an item from empty string list");
6fecf7cd01a     (Stefan Beller  2018-09-11 16:49:43 -0700 
89)             free(list->items[list->nr - 1].string);
6fecf7cd01a     (Stefan Beller  2018-09-11 16:49:43 -0700 
92)             free(list->items[list->nr - 1].util);
submodule-config.c
cc3e30d6aea     (Antonio Ospite 2018-09-17 16:09:32 +0200 
716)            return CONFIG_INVALID_KEY;
89039393db8     (Antonio Ospite 2018-09-17 16:09:33 +0200 
731)            warning(_("Could not update .gitmodules entry %s"), key);
submodule.c
bab609b4dc1     (Stefan Beller  2018-09-11 16:49:50 -0700 
1369)                   string_list_pop(&spf->retry, 0);
bab609b4dc1     (Stefan Beller  2018-09-11 16:49:50 -0700 
1370)                   goto retry_next;
bab609b4dc1     (Stefan Beller  2018-09-11 16:49:50 -0700 
1429)           warning(_("Could not get submodule repository for 
submodule '%s' in repository '%s'"),
bab609b4dc1     (Stefan Beller  2018-09-11 16:49:50 -0700 
1430)                     sub->path, spf->r->worktree);
df255b8cac7     (Brandon Williams       2018-08-08 15:33:22 -0700       
1884)           die(_("could not create directory '%s'"), new_gitdir.buf);
wrapper.c
7e621449185     (Pranit Bauva   2017-10-27 15:06:37 +0000 
701)            die_errno(_("could not stat %s"), filename);
wt-status.c
1c623c066c2     (Junio C Hamano 2018-09-07 15:32:24 -0700 
1949)                   if (s->state.rebase_in_progress ||
1c623c066c2     (Junio C Hamano 2018-09-07 15:32:24 -0700 1950) 
s->state.rebase_interactive_in_progress)
1c623c066c2     (Junio C Hamano 2018-09-07 15:32:24 -0700 
1951)                           branch_name = s->state.onto;
1c623c066c2     (Junio C Hamano 2018-09-07 15:32:24 -0700 
1952)                   else if (s->state.detached_from)
1c623c066c2     (Junio C Hamano 2018-09-07 15:32:24 -0700 
1953)                           branch_name = s->state.detached_from;


^ permalink raw reply	[relevance 4%]

* [PATCH v4 0/9] fetch: make sure submodule oids are fetched
@ 2018-09-25 19:47 Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 2/9] submodule.c: fix indentation Stefan Beller
                   ` (7 more replies)
  0 siblings, 8 replies; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

v4:
* Per Ævars comment, moved the docs for oid_array_filter into
  Documentation/technical/...
* addressed all outstanding comment as noted in "What's cooking" email,
* below is a range-diff against the currently queued version of the
  series.

v3:
* I discovered some issues with v2 after sending,
  which is why I rewrote the later patches completely
  and now we pass around a "task" struct that contains everything to know
  about the things to work on and what needs free()ing afterwards.
* as it is no longer string list based, this drops adding string_list_{pop, last}

v2:
* extended commit messages,
* plugged a memory leak
* rewrote the patch "sha1-array: provide oid_array_filter" to be much more like 
  object_array_fiter
* fixed a typo pointed out by Ramsay.

The range diff is below.
  
Thanks,
Stefan

v1:
Currently when git-fetch is asked to recurse into submodules, it dispatches
a plain "git-fetch -C <submodule-dir>" (and some submodule related options
such as prefix and recusing strategy, but) without any information of the
remote or the tip that should be fetched.

This works surprisingly well in some workflows, not so well in others,
which this series aims to fix.

The first patches provide new basic functionality and do some refactoring;
the interesting part is in the two last patches.

This was discussed in
https://public-inbox.org/git/20180808221752.195419-1-sbeller@google.com/
and I think I addressed all feedback so far.

Stefan Beller (9):
  sha1-array: provide oid_array_filter
  submodule.c: fix indentation
  submodule.c: sort changed_submodule_names before searching it
  submodule: move global changed_submodule_names into fetch submodule
    struct
  submodule.c: do not copy around submodule list
  repository: repo_submodule_init to take a submodule struct
  submodule: fetch in submodules git directory instead of in worktree
  fetch: retry fetching submodules if needed objects were not fetched
  builtin/fetch: check for submodule updates for non branch fetches

 Documentation/technical/api-oid-array.txt |   5 +
 builtin/fetch.c                           |  14 +-
 builtin/grep.c                            |  17 +-
 builtin/ls-files.c                        |  12 +-
 builtin/submodule--helper.c               |   2 +-
 repository.c                              |  27 +--
 repository.h                              |  11 +-
 sha1-array.c                              |  17 ++
 sha1-array.h                              |   3 +
 submodule.c                               | 275 +++++++++++++++++-----
 t/t5526-fetch-submodules.sh               |  23 +-
 11 files changed, 311 insertions(+), 95 deletions(-)

git range-diff  origin/sb/submodule-recursive-fetch-gets-the-tip...

  1:  6fecf7cd01a <   -:  ----------- string-list: add string_list_{pop, last} functions
  2:  7007a318a68 <   -:  ----------- sha1-array: provide oid_array_filter

    [ ... snip ... I rebased onto: ]

  -:  ----------- > 215:  fe8321ec057 Second batch post 2.19
  -:  ----------- > 216:  a9b49d4cfe9 sha1-array: provide oid_array_filter
  3:  807429234ac ! 217:  813205700d1 submodule.c: fix indentation
    @@ -6,7 +6,6 @@
         Fix it while we are here.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
      diff --git a/submodule.c b/submodule.c
      --- a/submodule.c
  4:  f6fa5273af9 ! 218:  b4aa77f72ba submodule.c: sort changed_submodule_names before searching it
    @@ -12,7 +12,6 @@
         appropriate.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
      diff --git a/submodule.c b/submodule.c
      --- a/submodule.c
  5:  adf7a2fd203 ! 219:  df4da0e18e4 submodule: move global changed_submodule_names into fetch submodule struct
    @@ -6,13 +6,12 @@
         part of the struct that is passed around for fetching submodules.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
      diff --git a/submodule.c b/submodule.c
      --- a/submodule.c
      +++ b/submodule.c
     @@
    - #include "object-store.h"
    + #include "commit-reach.h"
      
      static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
     -static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
  6:  56c9398589a ! 220:  b9f5d06c134 submodule.c: do not copy around submodule list
    @@ -9,12 +9,11 @@
         Instead use the result list directly and prune items afterwards
         using string_list_remove_empty_items.
     
    -    By doin so we'll have access to the util pointer for longer that
    +    By doing so we'll have access to the util pointer for longer that
         contains the commits that we need to fetch, which will be
         useful in a later patch.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
      diff --git a/submodule.c b/submodule.c
      --- a/submodule.c
  -:  ----------- > 221:  e5ff287a0a2 repository: repo_submodule_init to take a submodule struct
  7:  9f70a5f32c9 ! 222:  8acd734b5f5 submodule: fetch in submodules git directory instead of in worktree
    @@ -3,14 +3,24 @@
         submodule: fetch in submodules git directory instead of in worktree
     
         This patch started as a refactoring to make 'get_next_submodule' more
    -    readable, but upon doing so, I realized that git-fetch actually doesn't
    -    need to be run in the worktree. So let's run it in the git dir instead.
    +    readable, but upon doing so, I realized that "git fetch" of the submodule
    +    actually doesn't need to be run in the submodules worktree. So let's run
    +    it in its git dir instead.
     
         That should pave the way towards fetching submodules that are currently
         not checked out.
     
    +    This patch leaks the cp->dir in get_next_submodule, as any further
    +    callback in run_processes_parallel doesn't have access to the child
    +    process any more. In an early iteration of this patch, the function
    +    get_submodule_repo_for directly returned the string containing the
    +    git directory, which would be a better design choice for this patch.
    +
    +    However the next patch both fixes the memory leak of cp->dir and also has
    +    a use case for using the full repository handle of the submodule, so
    +    it makes sense to introduce the get_submodule_repo_for here already.
    +
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
      diff --git a/submodule.c b/submodule.c
      --- a/submodule.c
    @@ -32,24 +42,26 @@
      	return spf->default_option;
      }
      
    -+static const char *get_submodule_git_dir(struct repository *r, const char *path)
    ++static struct repository *get_submodule_repo_for(struct repository *r,
    ++						 const struct submodule *sub)
     +{
    -+	struct repository subrepo;
    -+	const char *ret;
    ++	struct repository *ret = xmalloc(sizeof(*ret));
     +
    -+	if (repo_submodule_init(&subrepo, r, path)) {
    -+		/* no entry in .gitmodules? */
    ++	if (repo_submodule_init(ret, r, sub)) {
    ++		/*
    ++		 * No entry in .gitmodules? Technically not a submodule,
    ++		 * but historically we supported repositories that happen to be
    ++		 * in-place where a gitlink is. Keep supporting them.
    ++		 */
     +		struct strbuf gitdir = STRBUF_INIT;
    -+		strbuf_repo_worktree_path(&gitdir, r, "%s/.git", path);
    -+		if (repo_init(&subrepo, gitdir.buf, NULL)) {
    ++		strbuf_repo_worktree_path(&gitdir, r, "%s/.git", sub->path);
    ++		if (repo_init(ret, gitdir.buf, NULL)) {
     +			strbuf_release(&gitdir);
     +			return NULL;
     +		}
    ++		strbuf_release(&gitdir);
     +	}
     +
    -+	ret = xstrdup(subrepo.gitdir);
    -+	repo_clear(&subrepo);
    -+
     +	return ret;
     +}
     +
    @@ -64,7 +76,13 @@
     -		struct strbuf submodule_git_dir = STRBUF_INIT;
      		struct strbuf submodule_prefix = STRBUF_INIT;
      		const struct cache_entry *ce = spf->r->index->cache[spf->count];
    - 		const char *git_dir, *default_argv;
    +-		const char *git_dir, *default_argv;
    ++		const char *default_argv;
    + 		const struct submodule *submodule;
    ++		struct repository *repo;
    + 		struct submodule default_submodule = SUBMODULE_INIT;
    + 
    + 		if (!S_ISGITLINK(ce->ce_mode))
     @@
      			continue;
      		}
    @@ -76,18 +94,23 @@
     -		if (!git_dir)
     -			git_dir = submodule_git_dir.buf;
     -		if (is_directory(git_dir)) {
    -+		git_dir = get_submodule_git_dir(spf->r, ce->name);
    -+		if (git_dir) {
    ++		repo = get_submodule_repo_for(spf->r, submodule);
    ++		if (repo) {
      			child_process_init(cp);
     -			cp->dir = strbuf_detach(&submodule_path, NULL);
     -			prepare_submodule_repo_env(&cp->env_array);
     +			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
    -+			cp->dir = git_dir;
    ++			cp->dir = xstrdup(repo->gitdir);
      			cp->git_cmd = 1;
      			if (!spf->quiet)
      				strbuf_addf(err, "Fetching submodule %s%s\n",
     @@
    + 			argv_array_push(&cp->args, default_argv);
    + 			argv_array_push(&cp->args, "--submodule-prefix");
      			argv_array_push(&cp->args, submodule_prefix.buf);
    ++
    ++			repo_clear(repo);
    ++			free(repo);
      			ret = 1;
      		}
     -		strbuf_release(&submodule_path);
  8:  bab609b4dc1 ! 223:  5752ba212a7 fetch: retry fetching submodules if sha1 were not fetched
    @@ -1,9 +1,9 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    fetch: retry fetching submodules if sha1 were not fetched
    +    fetch: retry fetching submodules if needed objects were not fetched
     
         Currently when git-fetch is asked to recurse into submodules, it dispatches
    -    a plain "git-fetch -C <submodule-dir>" (and some submodule related options
    +    a plain "git-fetch -C <submodule-dir>" (with some submodule related options
         such as prefix and recusing strategy, but) without any information of the
         remote or the tip that should be fetched.
     
    @@ -16,16 +16,22 @@
         topic downloaded as well. However these submodule changes reside in their
         own repository in their own ref (refs/changes/<int>).
     
    -    Retry fetching a submodule if the object id that the superproject points
    -    to, cannot be found.
    +    Retry fetching a submodule by object name if the object id that the
    +    superproject points to, cannot be found.
     
    -    This doesn't support fetching to FETCH_HEAD yet, but only into a local
    -    branch. To make fetching into FETCH_HEAD work, we need some refactoring
    -    in builtin/fetch.c to adjust the calls to 'check_for_new_submodule_commits'
    -    that is coming in the next patch.
    +    This retrying does not happen when the "git fetch" done at the
    +    superproject is not storing the fetched results in remote
    +    tracking branches (i.e. instead just recording them to
    +    FETCH_HEAD) in this step. A later patch will fix this.
    +
    +    builtin/fetch used to only inspect submodules when they were fetched
    +    "on-demand", as in either on/off case it was clear whether the submodule
    +    needs to be fetched. However to know whether we need to try fetching the
    +    object ids, we need to identify the object names, which is done in this
    +    function check_for_new_submodule_commits(), so we'll also run that code
    +    in case the submodule recursion is set to "on".
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
      diff --git a/builtin/fetch.c b/builtin/fetch.c
      --- a/builtin/fetch.c
    @@ -68,68 +74,181 @@
      	int result;
      
      	struct string_list changed_submodule_names;
    -+	struct string_list retry;
    ++	struct get_next_submodule_task **retry;
    ++	int retry_nr, retry_alloc;
      };
     -#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
     +#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \
     +		  STRING_LIST_INIT_DUP, \
    -+		  STRING_LIST_INIT_NODUP}
    ++		  NULL, 0, 0}
      
      static void calculate_changed_submodule_paths(
      	struct submodule_parallel_fetch *spf)
     @@
    + 	return spf->default_option;
    + }
    + 
    ++struct get_next_submodule_task {
    ++	struct repository *repo;
    ++	const struct submodule *sub;
    ++	unsigned free_sub : 1; /* Do we need to free the submodule? */
    ++	struct oid_array *commits;
    ++};
    ++
    ++static const struct submodule *get_default_submodule(const char *path)
    ++{
    ++	struct submodule *ret = NULL;
    ++	const char *name = default_name_or_path(path);
    ++
    ++	if (!name)
    ++		return NULL;
    ++
    ++	ret = xmalloc(sizeof(*ret));
    ++	memset(ret, 0, sizeof(*ret));
    ++	ret->path = name;
    ++	ret->name = name;
    ++
    ++	return (const struct submodule *) ret;
    ++}
    ++
    ++static struct get_next_submodule_task *get_next_submodule_task_create(
    ++	struct repository *r, const char *path)
    ++{
    ++	struct get_next_submodule_task *task = xmalloc(sizeof(*task));
    ++	memset(task, 0, sizeof(*task));
    ++
    ++	task->sub = submodule_from_path(r, &null_oid, path);
    ++	if (!task->sub) {
    ++		task->sub = get_default_submodule(path);
    ++		task->free_sub = 1;
    ++	}
    ++
    ++	return task;
    ++}
    ++
    ++static void get_next_submodule_task_release(struct get_next_submodule_task *p)
    ++{
    ++	if (p->free_sub)
    ++		free((void*)p->sub);
    ++	p->free_sub = 0;
    ++	p->sub = NULL;
    ++
    ++	if (p->repo)
    ++		repo_clear(p->repo);
    ++	FREE_AND_NULL(p->repo);
    ++}
    ++
    + static struct repository *get_submodule_repo_for(struct repository *r,
    + 						 const struct submodule *sub)
    + {
    +@@
    + static int get_next_submodule(struct child_process *cp,
    + 			      struct strbuf *err, void *data, void **task_cb)
      {
    - 	int ret = 0;
    +-	int ret = 0;
      	struct submodule_parallel_fetch *spf = data;
    -+	struct string_list_item *it;
      
      	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
    +-		struct strbuf submodule_prefix = STRBUF_INIT;
     +		int recurse_config;
    - 		struct strbuf submodule_prefix = STRBUF_INIT;
      		const struct cache_entry *ce = spf->r->index->cache[spf->count];
    - 		const char *git_dir, *default_argv;
    -@@
    - 			}
    + 		const char *default_argv;
    +-		const struct submodule *submodule;
    +-		struct repository *repo;
    +-		struct submodule default_submodule = SUBMODULE_INIT;
    ++		struct get_next_submodule_task *task;
    + 
    + 		if (!S_ISGITLINK(ce->ce_mode))
    + 			continue;
    + 
    +-		submodule = submodule_from_path(spf->r, &null_oid, ce->name);
    +-		if (!submodule) {
    +-			const char *name = default_name_or_path(ce->name);
    +-			if (name) {
    +-				default_submodule.path = name;
    +-				default_submodule.name = name;
    +-				submodule = &default_submodule;
    +-			}
    ++		task = get_next_submodule_task_create(spf->r, ce->name);
    ++
    ++		if (!task->sub) {
    ++			free(task);
    ++			continue;
      		}
      
     -		switch (get_fetch_recurse_config(submodule, spf))
    -+		recurse_config = get_fetch_recurse_config(submodule, spf);
    ++		recurse_config = get_fetch_recurse_config(task->sub, spf);
     +
     +		switch (recurse_config)
      		{
      		default:
      		case RECURSE_SUBMODULES_DEFAULT:
    + 		case RECURSE_SUBMODULES_ON_DEMAND:
    +-			if (!submodule ||
    ++			if (!task->sub ||
    + 			    !string_list_lookup(
    + 					&spf->changed_submodule_names,
    +-					submodule->name))
    ++					task->sub->name))
    + 				continue;
    + 			default_argv = "on-demand";
    + 			break;
    +@@
    + 			continue;
    + 		}
    + 
    +-		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
    +-		repo = get_submodule_repo_for(spf->r, submodule);
    +-		if (repo) {
    ++		task->repo = get_submodule_repo_for(spf->r, task->sub);
    ++		if (task->repo) {
    ++			struct strbuf submodule_prefix = STRBUF_INIT;
    + 			child_process_init(cp);
    + 			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
    +-			cp->dir = xstrdup(repo->gitdir);
    ++			cp->dir = task->repo->gitdir;
    + 			cp->git_cmd = 1;
    + 			if (!spf->quiet)
    + 				strbuf_addf(err, "Fetching submodule %s%s\n",
     @@
    - 		strbuf_release(&submodule_prefix);
    - 		if (ret) {
    + 			argv_array_pushv(&cp->args, spf->args.argv);
    + 			argv_array_push(&cp->args, default_argv);
    + 			argv_array_push(&cp->args, "--submodule-prefix");
    ++
    ++			strbuf_addf(&submodule_prefix, "%s%s/",
    ++						       spf->prefix,
    ++						       task->sub->path);
    + 			argv_array_push(&cp->args, submodule_prefix.buf);
    + 
    +-			repo_clear(repo);
    +-			free(repo);
    +-			ret = 1;
    +-		}
    +-		strbuf_release(&submodule_prefix);
    +-		if (ret) {
      			spf->count++;
    -+			if (submodule != &default_submodule)
    -+				/* discard const-ness: */
    -+				*task_cb = (void*)submodule;
    ++			*task_cb = task;
    ++
    ++			strbuf_release(&submodule_prefix);
      			return 1;
    ++		} else {
    ++			get_next_submodule_task_release(task);
    ++			free(task);
      		}
      	}
     +
    -+retry_next:
    -+
    -+	if (spf->retry.nr) {
    ++	if (spf->retry_nr) {
    ++		struct get_next_submodule_task *task = spf->retry[spf->retry_nr - 1];
     +		struct strbuf submodule_prefix = STRBUF_INIT;
    -+		const struct submodule *sub;
    ++		spf->retry_nr--;
     +
    -+		it = string_list_last(&spf->retry);
    -+		sub = submodule_from_name(spf->r, &null_oid,
    -+					  it->string);
    ++		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, task->sub->path);
     +
     +		child_process_init(cp);
    -+		cp->dir = get_submodule_git_dir(spf->r, sub->path);
    -+		if (!cp->dir) {
    -+			string_list_pop(&spf->retry, 0);
    -+			goto retry_next;
    -+		}
     +		prepare_submodule_repo_env_in_gitdir(&cp->env_array);
     +		cp->git_cmd = 1;
    ++		cp->dir = task->repo->gitdir;
     +
    -+		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, sub->path);
     +		argv_array_init(&cp->args);
     +		argv_array_pushv(&cp->args, spf->args.argv);
     +		argv_array_push(&cp->args, "on-demand");
    @@ -138,12 +257,11 @@
     +
     +		/* NEEDSWORK: have get_default_remote from s--h */
     +		argv_array_push(&cp->args, "origin");
    -+		oid_array_for_each_unique(it->util,
    ++		oid_array_for_each_unique(task->commits,
     +					  append_oid_to_argv, &cp->args);
     +
    -+		*task_cb = NULL; /* make sure we do not recurse forever */
    ++		*task_cb = task;
     +		strbuf_release(&submodule_prefix);
    -+		string_list_pop(&spf->retry, 0);
     +		return 1;
     +	}
     +
    @@ -151,6 +269,14 @@
      }
      
     @@
    + 			       void *cb, void *task_cb)
    + {
    + 	struct submodule_parallel_fetch *spf = cb;
    ++	struct get_next_submodule_task *task = task_cb;
    + 
    + 	spf->result = 1;
    + 
    ++	get_next_submodule_task_release(task);
      	return 0;
      }
      
    @@ -167,35 +293,46 @@
      			void *cb, void *task_cb)
      {
      	struct submodule_parallel_fetch *spf = cb;
    -+	struct submodule *sub = task_cb;
    -+	struct repository subrepo;
    ++	struct get_next_submodule_task *task = task_cb;
    ++	const struct submodule *sub;
    ++
    ++	struct string_list_item *it;
    ++	struct oid_array *commits;
      
      	if (retvalue)
      		spf->result = 1;
      
    -+	if (!sub)
    ++	if (!task)
     +		return 0;
     +
    -+	if (repo_submodule_init(&subrepo, spf->r, sub->path) < 0)
    -+		warning(_("Could not get submodule repository for submodule '%s' in repository '%s'"),
    -+			  sub->path, spf->r->worktree);
    -+	else {
    -+		struct string_list_item *it;
    -+		struct oid_array *commits;
    ++	sub = task->sub;
    ++	if (!sub)
    ++		goto out;
     +
    -+		it = string_list_lookup(&spf->changed_submodule_names, sub->name);
    -+		if (!it)
    -+			return 0;
    ++	it = string_list_lookup(&spf->changed_submodule_names, sub->name);
    ++	if (!it)
    ++		goto out;
    ++
    ++	commits = it->util;
    ++	oid_array_filter(commits,
    ++			 commit_exists_in_sub,
    ++			 task->repo);
     +
    -+		commits = it->util;
    -+		oid_array_filter(commits,
    -+				 commit_exists_in_sub,
    -+				 &subrepo);
    ++	/* Are there commits that do not exist? */
    ++	if (commits->nr) {
    ++		/* We already tried fetching them, do not try again. */
    ++		if (task->commits)
    ++			return 0;
     +
    -+		if (commits->nr)
    -+			string_list_append(&spf->retry, sub->name)
    -+				->util = commits;
    ++		task->commits = commits;
    ++		ALLOC_GROW(spf->retry, spf->retry_nr + 1, spf->retry_alloc);
    ++		spf->retry[spf->retry_nr] = task;
    ++		spf->retry_nr++;
    ++		return 0;
     +	}
    ++
    ++out:
    ++	get_next_submodule_task_release(task);
     +
      	return 0;
      }
  9:  c16d21313f6 ! 224:  d02ee9ef485 builtin/fetch: check for submodule updates for non branch fetches
    @@ -2,11 +2,29 @@
     
         builtin/fetch: check for submodule updates for non branch fetches
     
    -    For Gerrit users that use submodules the invocation of fetch without a
    -    branch is their main use case.
    +    Gerrit, the code review tool, has a different workflow than our mailing
    +    list based approach. Usually users upload changes to a Gerrit server and
    +    continuous integration and testing happens by bots. Sometimes however a
    +    user wants to checkout a change locally and look at it locally. For this
    +    use case, Gerrit offers a command line snippet to copy and paste to your
    +    terminal, which looks like
    +
    +      git fetch https://<host>/gerrit refs/changes/<id> &&
    +      git checkout FETCH_HEAD
    +
    +    For Gerrit changes that contain changing submodule gitlinks, it would be
    +    easy to extend both the fetch and checkout with the '--recurse-submodules'
    +    flag, such that this command line snippet would produce the state of a
    +    change locally.
    +
    +    However the functionality added in the previous patch, which would
    +    ensure that we fetch the objects in the submodule that the gitlink pointed
    +    at, only works for remote tracking branches so far, not for FETCH_HEAD.
    +
    +    Make sure that fetching a superproject to its FETCH_HEAD, also respects
    +    the existence checks for objects in the submodule recursion.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
      diff --git a/builtin/fetch.c b/builtin/fetch.c
      --- a/builtin/fetch.c

^ permalink raw reply	[relevance 8%]

* [PATCH v4 2/9] submodule.c: fix indentation
  2018-09-25 19:47 [PATCH v4 0/9] fetch: make sure submodule oids are fetched Stefan Beller
@ 2018-09-25 19:47 ` Stefan Beller
  2018-09-26 22:18   ` Junio C Hamano
  2018-09-25 19:47 ` [PATCH v4 3/9] submodule.c: sort changed_submodule_names before searching it Stefan Beller
                   ` (6 subsequent siblings)
  7 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

The submodule subsystem is really bad at staying within 80 characters.
Fix it while we are here.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/submodule.c b/submodule.c
index b53cb6e9c47..0de9e2800ad 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1244,7 +1244,8 @@ static int get_next_submodule(struct child_process *cp,
 		if (!submodule) {
 			const char *name = default_name_or_path(ce->name);
 			if (name) {
-				default_submodule.path = default_submodule.name = name;
+				default_submodule.path = name;
+				default_submodule.name = name;
 				submodule = &default_submodule;
 			}
 		}
@@ -1254,8 +1255,10 @@ static int get_next_submodule(struct child_process *cp,
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names,
-							 submodule->name))
+			if (!submodule ||
+			    !unsorted_string_list_lookup(
+					&changed_submodule_names,
+					submodule->name))
 				continue;
 			default_argv = "on-demand";
 			break;
-- 
2.19.0.605.g01d371f741-goog


^ permalink raw reply	[relevance 26%]

* [PATCH v4 3/9] submodule.c: sort changed_submodule_names before searching it
  2018-09-25 19:47 [PATCH v4 0/9] fetch: make sure submodule oids are fetched Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 2/9] submodule.c: fix indentation Stefan Beller
@ 2018-09-25 19:47 ` Stefan Beller
  2018-09-26 22:14   ` Junio C Hamano
  2018-09-25 19:47 ` [PATCH v4 4/9] submodule: move global changed_submodule_names into fetch submodule struct Stefan Beller
                   ` (5 subsequent siblings)
  7 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

We can string_list_insert() to maintain sorted-ness of the
list as we find new items, or we can string_list_append() to
build an unsorted list and sort it at the end just once.

To pick which one is more appropriate, we notice the fact
that we discover new items more or less in the already
sorted order.  That makes "append then sort" more
appropriate.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/submodule.c b/submodule.c
index 0de9e2800ad..22c64bd8559 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1256,7 +1256,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
-			    !unsorted_string_list_lookup(
+			    !string_list_lookup(
 					&changed_submodule_names,
 					submodule->name))
 				continue;
@@ -1350,6 +1350,7 @@ int fetch_populated_submodules(struct repository *r,
 	/* default value, "--submodule-prefix" and its value are added later */
 
 	calculate_changed_submodule_paths();
+	string_list_sort(&changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
-- 
2.19.0.605.g01d371f741-goog


^ permalink raw reply	[relevance 17%]

* [PATCH v4 4/9] submodule: move global changed_submodule_names into fetch submodule struct
  2018-09-25 19:47 [PATCH v4 0/9] fetch: make sure submodule oids are fetched Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 2/9] submodule.c: fix indentation Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 3/9] submodule.c: sort changed_submodule_names before searching it Stefan Beller
@ 2018-09-25 19:47 ` Stefan Beller
  2018-09-26 22:19   ` Junio C Hamano
  2018-09-25 19:47 ` [PATCH v4 5/9] submodule.c: do not copy around submodule list Stefan Beller
                   ` (4 subsequent siblings)
  7 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

The `changed_submodule_names` are only used for fetching, so let's make it
part of the struct that is passed around for fetching submodules.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/submodule.c b/submodule.c
index 22c64bd8559..17103379ba4 100644
--- a/submodule.c
+++ b/submodule.c
@@ -25,7 +25,7 @@
 #include "commit-reach.h"
 
 static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
-static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
+
 static int initialized_fetch_ref_tips;
 static struct oid_array ref_tips_before_fetch;
 static struct oid_array ref_tips_after_fetch;
@@ -1110,7 +1110,22 @@ void check_for_new_submodule_commits(struct object_id *oid)
 	oid_array_append(&ref_tips_after_fetch, oid);
 }
 
-static void calculate_changed_submodule_paths(void)
+struct submodule_parallel_fetch {
+	int count;
+	struct argv_array args;
+	struct repository *r;
+	const char *prefix;
+	int command_line_option;
+	int default_option;
+	int quiet;
+	int result;
+
+	struct string_list changed_submodule_names;
+};
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+
+static void calculate_changed_submodule_paths(
+	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
 	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
@@ -1148,7 +1163,8 @@ static void calculate_changed_submodule_paths(void)
 			continue;
 
 		if (!submodule_has_commits(path, commits))
-			string_list_append(&changed_submodule_names, name->string);
+			string_list_append(&spf->changed_submodule_names,
+					   name->string);
 	}
 
 	free_submodules_oids(&changed_submodules);
@@ -1185,18 +1201,6 @@ int submodule_touches_in_range(struct object_id *excl_oid,
 	return ret;
 }
 
-struct submodule_parallel_fetch {
-	int count;
-	struct argv_array args;
-	struct repository *r;
-	const char *prefix;
-	int command_line_option;
-	int default_option;
-	int quiet;
-	int result;
-};
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
-
 static int get_fetch_recurse_config(const struct submodule *submodule,
 				    struct submodule_parallel_fetch *spf)
 {
@@ -1257,7 +1261,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
 			    !string_list_lookup(
-					&changed_submodule_names,
+					&spf->changed_submodule_names,
 					submodule->name))
 				continue;
 			default_argv = "on-demand";
@@ -1349,8 +1353,8 @@ int fetch_populated_submodules(struct repository *r,
 	argv_array_push(&spf.args, "--recurse-submodules-default");
 	/* default value, "--submodule-prefix" and its value are added later */
 
-	calculate_changed_submodule_paths();
-	string_list_sort(&changed_submodule_names);
+	calculate_changed_submodule_paths(&spf);
+	string_list_sort(&spf.changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
@@ -1359,7 +1363,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&changed_submodule_names, 1);
+	string_list_clear(&spf.changed_submodule_names, 1);
 	return spf.result;
 }
 
-- 
2.19.0.605.g01d371f741-goog


^ permalink raw reply	[relevance 17%]

* [PATCH v4 5/9] submodule.c: do not copy around submodule list
  2018-09-25 19:47 [PATCH v4 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (2 preceding siblings ...)
  2018-09-25 19:47 ` [PATCH v4 4/9] submodule: move global changed_submodule_names into fetch submodule struct Stefan Beller
@ 2018-09-25 19:47 ` Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 6/9] repository: repo_submodule_init to take a submodule struct Stefan Beller
                   ` (3 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

'calculate_changed_submodule_paths' uses a local list to compute the
changed submodules, and then produces the result by copying appropriate
items into the result list.

Instead use the result list directly and prune items afterwards
using string_list_remove_empty_items.

By doing so we'll have access to the util pointer for longer that
contains the commits that we need to fetch, which will be
useful in a later patch.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/submodule.c b/submodule.c
index 17103379ba4..dd478ed70bf 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1128,8 +1128,7 @@ static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
-	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
-	const struct string_list_item *name;
+	struct string_list_item *name;
 
 	/* No need to check if there are no submodules configured */
 	if (!submodule_from_path(the_repository, NULL, NULL))
@@ -1146,9 +1145,9 @@ static void calculate_changed_submodule_paths(
 	 * Collect all submodules (whether checked out or not) for which new
 	 * commits have been recorded upstream in "changed_submodule_names".
 	 */
-	collect_changed_submodules(&changed_submodules, &argv);
+	collect_changed_submodules(&spf->changed_submodule_names, &argv);
 
-	for_each_string_list_item(name, &changed_submodules) {
+	for_each_string_list_item(name, &spf->changed_submodule_names) {
 		struct oid_array *commits = name->util;
 		const struct submodule *submodule;
 		const char *path = NULL;
@@ -1162,12 +1161,14 @@ static void calculate_changed_submodule_paths(
 		if (!path)
 			continue;
 
-		if (!submodule_has_commits(path, commits))
-			string_list_append(&spf->changed_submodule_names,
-					   name->string);
+		if (submodule_has_commits(path, commits)) {
+			oid_array_clear(commits);
+			*name->string = '\0';
+		}
 	}
 
-	free_submodules_oids(&changed_submodules);
+	string_list_remove_empty_items(&spf->changed_submodule_names, 1);
+
 	argv_array_clear(&argv);
 	oid_array_clear(&ref_tips_before_fetch);
 	oid_array_clear(&ref_tips_after_fetch);
@@ -1363,7 +1364,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&spf.changed_submodule_names, 1);
+	free_submodules_oids(&spf.changed_submodule_names);
 	return spf.result;
 }
 
-- 
2.19.0.605.g01d371f741-goog


^ permalink raw reply	[relevance 17%]

* [PATCH v4 6/9] repository: repo_submodule_init to take a submodule struct
  2018-09-25 19:47 [PATCH v4 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (3 preceding siblings ...)
  2018-09-25 19:47 ` [PATCH v4 5/9] submodule.c: do not copy around submodule list Stefan Beller
@ 2018-09-25 19:47 ` Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 7/9] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
                   ` (2 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

When constructing a struct repository for a submodule for some revision
of the superproject where the submodule is not contained in the index,
it may not be present in the working tree currently either. In that
siutation giving a 'path' argument is not useful. Upgrade the
repo_submodule_init function to take a struct submodule instead.

While we are at it, overhaul the repo_submodule_init function by renaming
the submodule repository struct, which is to be initialized, to a name
that is not confused with the struct submodule as easily.

Also move its documentation into the header file.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/grep.c              | 17 ++++++++++-------
 builtin/ls-files.c          | 12 +++++++-----
 builtin/submodule--helper.c |  2 +-
 repository.c                | 27 ++++++++++-----------------
 repository.h                | 11 +++++++++--
 5 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/builtin/grep.c b/builtin/grep.c
index 601f801158f..81c53c862b1 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -418,16 +418,19 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 			  const struct object_id *oid,
 			  const char *filename, const char *path)
 {
-	struct repository submodule;
+	struct repository subrepo;
+	const struct submodule *sub = submodule_from_path(superproject,
+							  &null_oid, path);
+
 	int hit;
 
 	if (!is_submodule_active(superproject, path))
 		return 0;
 
-	if (repo_submodule_init(&submodule, superproject, path))
+	if (repo_submodule_init(&subrepo, superproject, sub))
 		return 0;
 
-	repo_read_gitmodules(&submodule);
+	repo_read_gitmodules(&subrepo);
 
 	/*
 	 * NEEDSWORK: This adds the submodule's object directory to the list of
@@ -440,7 +443,7 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 	 * object.
 	 */
 	grep_read_lock();
-	add_to_alternates_memory(submodule.objects->objectdir);
+	add_to_alternates_memory(subrepo.objects->objectdir);
 	grep_read_unlock();
 
 	if (oid) {
@@ -465,14 +468,14 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 
 		init_tree_desc(&tree, data, size);
 		hit = grep_tree(opt, pathspec, &tree, &base, base.len,
-				object->type == OBJ_COMMIT, &submodule);
+				object->type == OBJ_COMMIT, &subrepo);
 		strbuf_release(&base);
 		free(data);
 	} else {
-		hit = grep_cache(opt, &submodule, pathspec, 1);
+		hit = grep_cache(opt, &subrepo, pathspec, 1);
 	}
 
-	repo_clear(&submodule);
+	repo_clear(&subrepo);
 	return hit;
 }
 
diff --git a/builtin/ls-files.c b/builtin/ls-files.c
index 7f9919a3623..4d1649c1b3a 100644
--- a/builtin/ls-files.c
+++ b/builtin/ls-files.c
@@ -206,17 +206,19 @@ static void show_files(struct repository *repo, struct dir_struct *dir);
 static void show_submodule(struct repository *superproject,
 			   struct dir_struct *dir, const char *path)
 {
-	struct repository submodule;
+	struct repository subrepo;
+	const struct submodule *sub = submodule_from_path(superproject,
+							  &null_oid, path);
 
-	if (repo_submodule_init(&submodule, superproject, path))
+	if (repo_submodule_init(&subrepo, superproject, sub))
 		return;
 
-	if (repo_read_index(&submodule) < 0)
+	if (repo_read_index(&subrepo) < 0)
 		die("index file corrupt");
 
-	show_files(&submodule, dir);
+	show_files(&subrepo, dir);
 
-	repo_clear(&submodule);
+	repo_clear(&subrepo);
 }
 
 static void show_ce(struct repository *repo, struct dir_struct *dir,
diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 40844870cfb..1164e0979bd 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -2037,7 +2037,7 @@ static int ensure_core_worktree(int argc, const char **argv, const char *prefix)
 	if (!sub)
 		BUG("We could get the submodule handle before?");
 
-	if (repo_submodule_init(&subrepo, the_repository, path))
+	if (repo_submodule_init(&subrepo, the_repository, sub))
 		die(_("could not get a repository handle for submodule '%s'"), path);
 
 	if (!repo_config_get_string(&subrepo, "core.worktree", &cw)) {
diff --git a/repository.c b/repository.c
index 5dd14867181..aabe64ee5d9 100644
--- a/repository.c
+++ b/repository.c
@@ -166,30 +166,23 @@ int repo_init(struct repository *repo,
 	return -1;
 }
 
-/*
- * Initialize 'submodule' as the submodule given by 'path' in parent repository
- * 'superproject'.
- * Return 0 upon success and a non-zero value upon failure.
- */
-int repo_submodule_init(struct repository *submodule,
+int repo_submodule_init(struct repository *subrepo,
 			struct repository *superproject,
-			const char *path)
+			const struct submodule *sub)
 {
-	const struct submodule *sub;
 	struct strbuf gitdir = STRBUF_INIT;
 	struct strbuf worktree = STRBUF_INIT;
 	int ret = 0;
 
-	sub = submodule_from_path(superproject, &null_oid, path);
 	if (!sub) {
 		ret = -1;
 		goto out;
 	}
 
-	strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", path);
-	strbuf_repo_worktree_path(&worktree, superproject, "%s", path);
+	strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", sub->path);
+	strbuf_repo_worktree_path(&worktree, superproject, "%s", sub->path);
 
-	if (repo_init(submodule, gitdir.buf, worktree.buf)) {
+	if (repo_init(subrepo, gitdir.buf, worktree.buf)) {
 		/*
 		 * If initilization fails then it may be due to the submodule
 		 * not being populated in the superproject's worktree.  Instead
@@ -201,16 +194,16 @@ int repo_submodule_init(struct repository *submodule,
 		strbuf_repo_git_path(&gitdir, superproject,
 				     "modules/%s", sub->name);
 
-		if (repo_init(submodule, gitdir.buf, NULL)) {
+		if (repo_init(subrepo, gitdir.buf, NULL)) {
 			ret = -1;
 			goto out;
 		}
 	}
 
-	submodule->submodule_prefix = xstrfmt("%s%s/",
-					      superproject->submodule_prefix ?
-					      superproject->submodule_prefix :
-					      "", path);
+	subrepo->submodule_prefix = xstrfmt("%s%s/",
+					    superproject->submodule_prefix ?
+					    superproject->submodule_prefix :
+					    "", sub->path);
 
 out:
 	strbuf_release(&gitdir);
diff --git a/repository.h b/repository.h
index 9f16c42c1ed..a9c7a5baa54 100644
--- a/repository.h
+++ b/repository.h
@@ -116,9 +116,16 @@ void repo_set_worktree(struct repository *repo, const char *path);
 void repo_set_hash_algo(struct repository *repo, int algo);
 void initialize_the_repository(void);
 int repo_init(struct repository *r, const char *gitdir, const char *worktree);
-int repo_submodule_init(struct repository *submodule,
+
+/*
+ * Initialize the repository 'subrepo' as the submodule given by the
+ * struct submodule 'sub' in parent repository 'superproject'.
+ * Return 0 upon success and a non-zero value upon failure.
+ */
+struct submodule;
+int repo_submodule_init(struct repository *subrepo,
 			struct repository *superproject,
-			const char *path);
+			const struct submodule *sub);
 void repo_clear(struct repository *repo);
 
 /*
-- 
2.19.0.605.g01d371f741-goog


^ permalink raw reply	[relevance 26%]

* [PATCH v4 7/9] submodule: fetch in submodules git directory instead of in worktree
  2018-09-25 19:47 [PATCH v4 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (4 preceding siblings ...)
  2018-09-25 19:47 ` [PATCH v4 6/9] repository: repo_submodule_init to take a submodule struct Stefan Beller
@ 2018-09-25 19:47 ` Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 8/9] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 9/9] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

This patch started as a refactoring to make 'get_next_submodule' more
readable, but upon doing so, I realized that "git fetch" of the submodule
actually doesn't need to be run in the submodules worktree. So let's run
it in its git dir instead.

That should pave the way towards fetching submodules that are currently
not checked out.

This patch leaks the cp->dir in get_next_submodule, as any further
callback in run_processes_parallel doesn't have access to the child
process any more. In an early iteration of this patch, the function
get_submodule_repo_for directly returned the string containing the
git directory, which would be a better design choice for this patch.

However the next patch both fixes the memory leak of cp->dir and also has
a use case for using the full repository handle of the submodule, so
it makes sense to introduce the get_submodule_repo_for here already.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c                 | 51 +++++++++++++++++++++++++++----------
 t/t5526-fetch-submodules.sh |  7 ++++-
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/submodule.c b/submodule.c
index dd478ed70bf..3f791f22771 100644
--- a/submodule.c
+++ b/submodule.c
@@ -481,6 +481,12 @@ void prepare_submodule_repo_env(struct argv_array *out)
 			 DEFAULT_GIT_DIR_ENVIRONMENT);
 }
 
+static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out)
+{
+	prepare_submodule_repo_env_no_git_dir(out);
+	argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
+}
+
 /* Helper function to display the submodule header line prior to the full
  * summary output. If it can locate the submodule objects directory it will
  * attempt to lookup both the left and right commits and put them into the
@@ -1227,6 +1233,29 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+static struct repository *get_submodule_repo_for(struct repository *r,
+						 const struct submodule *sub)
+{
+	struct repository *ret = xmalloc(sizeof(*ret));
+
+	if (repo_submodule_init(ret, r, sub)) {
+		/*
+		 * No entry in .gitmodules? Technically not a submodule,
+		 * but historically we supported repositories that happen to be
+		 * in-place where a gitlink is. Keep supporting them.
+		 */
+		struct strbuf gitdir = STRBUF_INIT;
+		strbuf_repo_worktree_path(&gitdir, r, "%s/.git", sub->path);
+		if (repo_init(ret, gitdir.buf, NULL)) {
+			strbuf_release(&gitdir);
+			return NULL;
+		}
+		strbuf_release(&gitdir);
+	}
+
+	return ret;
+}
+
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
@@ -1234,12 +1263,11 @@ static int get_next_submodule(struct child_process *cp,
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_path = STRBUF_INIT;
-		struct strbuf submodule_git_dir = STRBUF_INIT;
 		struct strbuf submodule_prefix = STRBUF_INIT;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
-		const char *git_dir, *default_argv;
+		const char *default_argv;
 		const struct submodule *submodule;
+		struct repository *repo;
 		struct submodule default_submodule = SUBMODULE_INIT;
 
 		if (!S_ISGITLINK(ce->ce_mode))
@@ -1274,16 +1302,12 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_repo_worktree_path(&submodule_path, spf->r, "%s", ce->name);
-		strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
 		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		git_dir = read_gitfile(submodule_git_dir.buf);
-		if (!git_dir)
-			git_dir = submodule_git_dir.buf;
-		if (is_directory(git_dir)) {
+		repo = get_submodule_repo_for(spf->r, submodule);
+		if (repo) {
 			child_process_init(cp);
-			cp->dir = strbuf_detach(&submodule_path, NULL);
-			prepare_submodule_repo_env(&cp->env_array);
+			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+			cp->dir = xstrdup(repo->gitdir);
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1293,10 +1317,11 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
 			argv_array_push(&cp->args, submodule_prefix.buf);
+
+			repo_clear(repo);
+			free(repo);
 			ret = 1;
 		}
-		strbuf_release(&submodule_path);
-		strbuf_release(&submodule_git_dir);
 		strbuf_release(&submodule_prefix);
 		if (ret) {
 			spf->count++;
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 6c2f9b2ba26..42692219a1a 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -566,7 +566,12 @@ test_expect_success 'fetching submodule into a broken repository' '
 
 	test_must_fail git -C dst status &&
 	test_must_fail git -C dst diff &&
-	test_must_fail git -C dst fetch --recurse-submodules
+
+	# git-fetch cannot find the git directory of the submodule,
+	# so it will do nothing, successfully, as it cannot distinguish between
+	# this broken submodule and a submodule that was just set active but
+	# not cloned yet
+	git -C dst fetch --recurse-submodules
 '
 
 test_expect_success "fetch new commits when submodule got renamed" '
-- 
2.19.0.605.g01d371f741-goog


^ permalink raw reply	[relevance 28%]

* [PATCH v4 8/9] fetch: retry fetching submodules if needed objects were not fetched
  2018-09-25 19:47 [PATCH v4 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (5 preceding siblings ...)
  2018-09-25 19:47 ` [PATCH v4 7/9] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
@ 2018-09-25 19:47 ` Stefan Beller
  2018-09-25 19:47 ` [PATCH v4 9/9] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

Currently when git-fetch is asked to recurse into submodules, it dispatches
a plain "git-fetch -C <submodule-dir>" (with some submodule related options
such as prefix and recusing strategy, but) without any information of the
remote or the tip that should be fetched.

This works surprisingly well in some workflows (such as using submodules
as a third party library), while not so well in other scenarios, such
as in a Gerrit topic-based workflow, that can tie together changes
(potentially across repositories) on the server side. One of the parts
of such a Gerrit workflow is to download a change when wanting to examine
it, and you'd want to have its submodule changes that are in the same
topic downloaded as well. However these submodule changes reside in their
own repository in their own ref (refs/changes/<int>).

Retry fetching a submodule by object name if the object id that the
superproject points to, cannot be found.

This retrying does not happen when the "git fetch" done at the
superproject is not storing the fetched results in remote
tracking branches (i.e. instead just recording them to
FETCH_HEAD) in this step. A later patch will fix this.

builtin/fetch used to only inspect submodules when they were fetched
"on-demand", as in either on/off case it was clear whether the submodule
needs to be fetched. However to know whether we need to try fetching the
object ids, we need to identify the object names, which is done in this
function check_for_new_submodule_commits(), so we'll also run that code
in case the submodule recursion is set to "on".

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c             |   9 +-
 submodule.c                 | 185 ++++++++++++++++++++++++++++++------
 t/t5526-fetch-submodules.sh |  16 ++++
 3 files changed, 177 insertions(+), 33 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 0696abfc2a1..e3b03ad3bd3 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -707,8 +707,7 @@ static int update_local_ref(struct ref *ref,
 			what = _("[new ref]");
 		}
 
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref(msg, ref, 0);
 		format_display(display, r ? '!' : '*', what,
@@ -723,8 +722,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "..");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("fast-forward", ref, 1);
 		format_display(display, r ? '!' : ' ', quickref.buf,
@@ -738,8 +736,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "...");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("forced-update", ref, 1);
 		format_display(display, r ? '!' : '+', quickref.buf,
diff --git a/submodule.c b/submodule.c
index 3f791f22771..05799362e05 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1127,8 +1127,12 @@ struct submodule_parallel_fetch {
 	int result;
 
 	struct string_list changed_submodule_names;
+	struct get_next_submodule_task **retry;
+	int retry_nr, retry_alloc;
 };
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \
+		  STRING_LIST_INIT_DUP, \
+		  NULL, 0, 0}
 
 static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
@@ -1233,6 +1237,56 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+struct get_next_submodule_task {
+	struct repository *repo;
+	const struct submodule *sub;
+	unsigned free_sub : 1; /* Do we need to free the submodule? */
+	struct oid_array *commits;
+};
+
+static const struct submodule *get_default_submodule(const char *path)
+{
+	struct submodule *ret = NULL;
+	const char *name = default_name_or_path(path);
+
+	if (!name)
+		return NULL;
+
+	ret = xmalloc(sizeof(*ret));
+	memset(ret, 0, sizeof(*ret));
+	ret->path = name;
+	ret->name = name;
+
+	return (const struct submodule *) ret;
+}
+
+static struct get_next_submodule_task *get_next_submodule_task_create(
+	struct repository *r, const char *path)
+{
+	struct get_next_submodule_task *task = xmalloc(sizeof(*task));
+	memset(task, 0, sizeof(*task));
+
+	task->sub = submodule_from_path(r, &null_oid, path);
+	if (!task->sub) {
+		task->sub = get_default_submodule(path);
+		task->free_sub = 1;
+	}
+
+	return task;
+}
+
+static void get_next_submodule_task_release(struct get_next_submodule_task *p)
+{
+	if (p->free_sub)
+		free((void*)p->sub);
+	p->free_sub = 0;
+	p->sub = NULL;
+
+	if (p->repo)
+		repo_clear(p->repo);
+	FREE_AND_NULL(p->repo);
+}
+
 static struct repository *get_submodule_repo_for(struct repository *r,
 						 const struct submodule *sub)
 {
@@ -1259,39 +1313,35 @@ static struct repository *get_submodule_repo_for(struct repository *r,
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
-	int ret = 0;
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_prefix = STRBUF_INIT;
+		int recurse_config;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
 		const char *default_argv;
-		const struct submodule *submodule;
-		struct repository *repo;
-		struct submodule default_submodule = SUBMODULE_INIT;
+		struct get_next_submodule_task *task;
 
 		if (!S_ISGITLINK(ce->ce_mode))
 			continue;
 
-		submodule = submodule_from_path(spf->r, &null_oid, ce->name);
-		if (!submodule) {
-			const char *name = default_name_or_path(ce->name);
-			if (name) {
-				default_submodule.path = name;
-				default_submodule.name = name;
-				submodule = &default_submodule;
-			}
+		task = get_next_submodule_task_create(spf->r, ce->name);
+
+		if (!task->sub) {
+			free(task);
+			continue;
 		}
 
-		switch (get_fetch_recurse_config(submodule, spf))
+		recurse_config = get_fetch_recurse_config(task->sub, spf);
+
+		switch (recurse_config)
 		{
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule ||
+			if (!task->sub ||
 			    !string_list_lookup(
 					&spf->changed_submodule_names,
-					submodule->name))
+					task->sub->name))
 				continue;
 			default_argv = "on-demand";
 			break;
@@ -1302,12 +1352,12 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		repo = get_submodule_repo_for(spf->r, submodule);
-		if (repo) {
+		task->repo = get_submodule_repo_for(spf->r, task->sub);
+		if (task->repo) {
+			struct strbuf submodule_prefix = STRBUF_INIT;
 			child_process_init(cp);
 			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
-			cp->dir = xstrdup(repo->gitdir);
+			cp->dir = task->repo->gitdir;
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1316,18 +1366,51 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_pushv(&cp->args, spf->args.argv);
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
+
+			strbuf_addf(&submodule_prefix, "%s%s/",
+						       spf->prefix,
+						       task->sub->path);
 			argv_array_push(&cp->args, submodule_prefix.buf);
 
-			repo_clear(repo);
-			free(repo);
-			ret = 1;
-		}
-		strbuf_release(&submodule_prefix);
-		if (ret) {
 			spf->count++;
+			*task_cb = task;
+
+			strbuf_release(&submodule_prefix);
 			return 1;
+		} else {
+			get_next_submodule_task_release(task);
+			free(task);
 		}
 	}
+
+	if (spf->retry_nr) {
+		struct get_next_submodule_task *task = spf->retry[spf->retry_nr - 1];
+		struct strbuf submodule_prefix = STRBUF_INIT;
+		spf->retry_nr--;
+
+		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, task->sub->path);
+
+		child_process_init(cp);
+		prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+		cp->git_cmd = 1;
+		cp->dir = task->repo->gitdir;
+
+		argv_array_init(&cp->args);
+		argv_array_pushv(&cp->args, spf->args.argv);
+		argv_array_push(&cp->args, "on-demand");
+		argv_array_push(&cp->args, "--submodule-prefix");
+		argv_array_push(&cp->args, submodule_prefix.buf);
+
+		/* NEEDSWORK: have get_default_remote from s--h */
+		argv_array_push(&cp->args, "origin");
+		oid_array_for_each_unique(task->commits,
+					  append_oid_to_argv, &cp->args);
+
+		*task_cb = task;
+		strbuf_release(&submodule_prefix);
+		return 1;
+	}
+
 	return 0;
 }
 
@@ -1335,20 +1418,68 @@ static int fetch_start_failure(struct strbuf *err,
 			       void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct get_next_submodule_task *task = task_cb;
 
 	spf->result = 1;
 
+	get_next_submodule_task_release(task);
 	return 0;
 }
 
+static int commit_exists_in_sub(const struct object_id *oid, void *data)
+{
+	struct repository *subrepo = data;
+
+	enum object_type type = oid_object_info(subrepo, oid, NULL);
+
+	return type != OBJ_COMMIT;
+}
+
 static int fetch_finish(int retvalue, struct strbuf *err,
 			void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct get_next_submodule_task *task = task_cb;
+	const struct submodule *sub;
+
+	struct string_list_item *it;
+	struct oid_array *commits;
 
 	if (retvalue)
 		spf->result = 1;
 
+	if (!task)
+		return 0;
+
+	sub = task->sub;
+	if (!sub)
+		goto out;
+
+	it = string_list_lookup(&spf->changed_submodule_names, sub->name);
+	if (!it)
+		goto out;
+
+	commits = it->util;
+	oid_array_filter(commits,
+			 commit_exists_in_sub,
+			 task->repo);
+
+	/* Are there commits that do not exist? */
+	if (commits->nr) {
+		/* We already tried fetching them, do not try again. */
+		if (task->commits)
+			return 0;
+
+		task->commits = commits;
+		ALLOC_GROW(spf->retry, spf->retry_nr + 1, spf->retry_alloc);
+		spf->retry[spf->retry_nr] = task;
+		spf->retry_nr++;
+		return 0;
+	}
+
+out:
+	get_next_submodule_task_release(task);
+
 	return 0;
 }
 
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 42692219a1a..af12c50e7dd 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -605,4 +605,20 @@ test_expect_success "fetch new commits when submodule got renamed" '
 	test_cmp expect actual
 '
 
+test_expect_success "fetch new commits on-demand when they are not reachable" '
+	git checkout --detach &&
+	C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
+	git -C submodule update-ref refs/changes/1 $C &&
+	git update-index --cacheinfo 160000 $C submodule &&
+	git commit -m "updated submodule outside of refs/heads" &&
+	D=$(git rev-parse HEAD) &&
+	git update-ref refs/changes/2 $D &&
+	(
+		cd downstream &&
+		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git -C submodule cat-file -t $C &&
+		git checkout --recurse-submodules FETCH_HEAD
+	)
+'
+
 test_done
-- 
2.19.0.605.g01d371f741-goog


^ permalink raw reply	[relevance 25%]

* [PATCH v4 9/9] builtin/fetch: check for submodule updates for non branch fetches
  2018-09-25 19:47 [PATCH v4 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (6 preceding siblings ...)
  2018-09-25 19:47 ` [PATCH v4 8/9] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
@ 2018-09-25 19:47 ` Stefan Beller
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-25 19:47 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

Gerrit, the code review tool, has a different workflow than our mailing
list based approach. Usually users upload changes to a Gerrit server and
continuous integration and testing happens by bots. Sometimes however a
user wants to checkout a change locally and look at it locally. For this
use case, Gerrit offers a command line snippet to copy and paste to your
terminal, which looks like

  git fetch https://<host>/gerrit refs/changes/<id> &&
  git checkout FETCH_HEAD

For Gerrit changes that contain changing submodule gitlinks, it would be
easy to extend both the fetch and checkout with the '--recurse-submodules'
flag, such that this command line snippet would produce the state of a
change locally.

However the functionality added in the previous patch, which would
ensure that we fetch the objects in the submodule that the gitlink pointed
at, only works for remote tracking branches so far, not for FETCH_HEAD.

Make sure that fetching a superproject to its FETCH_HEAD, also respects
the existence checks for objects in the submodule recursion.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c             | 5 ++++-
 t/t5526-fetch-submodules.sh | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index e3b03ad3bd3..f2d9e548bf0 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -894,11 +894,14 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
 				rc |= update_local_ref(ref, what, rm, &note,
 						       summary_width);
 				free(ref);
-			} else
+			} else {
+				check_for_new_submodule_commits(&rm->old_oid);
 				format_display(&note, '*',
 					       *kind ? kind : "branch", NULL,
 					       *what ? what : "HEAD",
 					       "FETCH_HEAD", summary_width);
+			}
+
 			if (note.len) {
 				if (verbosity >= 0 && !shown_url) {
 					fprintf(stderr, _("From %.*s\n"),
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index af12c50e7dd..a509eabb044 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -615,7 +615,7 @@ test_expect_success "fetch new commits on-demand when they are not reachable" '
 	git update-ref refs/changes/2 $D &&
 	(
 		cd downstream &&
-		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git fetch --recurse-submodules origin refs/changes/2 &&
 		git -C submodule cat-file -t $C &&
 		git checkout --recurse-submodules FETCH_HEAD
 	)
-- 
2.19.0.605.g01d371f741-goog


^ permalink raw reply	[relevance 22%]

* Re: [PATCH v4 3/9] submodule.c: sort changed_submodule_names before searching it
  2018-09-25 19:47 ` [PATCH v4 3/9] submodule.c: sort changed_submodule_names before searching it Stefan Beller
@ 2018-09-26 22:14   ` Junio C Hamano
  0 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-09-26 22:14 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

Stefan Beller <sbeller@google.com> writes:

> We can string_list_insert() to maintain sorted-ness of the
> list as we find new items, or we can string_list_append() to
> build an unsorted list and sort it at the end just once.
>
> To pick which one is more appropriate, we notice the fact
> that we discover new items more or less in the already
> sorted order.  That makes "append then sort" more
> appropriate.

I somehow thought that we agreed that the second paragraph above did
not make much sense in the previous review round.

    ... goes and looks ...

https://public-inbox.org/git/CAGZ79kbavjVbTqXsmtjW6=jhkq47_p3mc6=92xOp4_mfhqDtvw@mail.gmail.com/

That was two review cycles ago, I guess.

^ permalink raw reply	[relevance 5%]

* Re: [PATCH v4 2/9] submodule.c: fix indentation
  2018-09-25 19:47 ` [PATCH v4 2/9] submodule.c: fix indentation Stefan Beller
@ 2018-09-26 22:18   ` Junio C Hamano
  0 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-09-26 22:18 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

Stefan Beller <sbeller@google.com> writes:

> The submodule subsystem is really bad at staying within 80 characters.
> Fix it while we are here.
>
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---

Makes sense.

>  submodule.c | 9 ++++++---
>  1 file changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/submodule.c b/submodule.c
> index b53cb6e9c47..0de9e2800ad 100644
> --- a/submodule.c
> +++ b/submodule.c
> @@ -1244,7 +1244,8 @@ static int get_next_submodule(struct child_process *cp,
>  		if (!submodule) {
>  			const char *name = default_name_or_path(ce->name);
>  			if (name) {
> -				default_submodule.path = default_submodule.name = name;
> +				default_submodule.path = name;
> +				default_submodule.name = name;
>  				submodule = &default_submodule;
>  			}
>  		}
> @@ -1254,8 +1255,10 @@ static int get_next_submodule(struct child_process *cp,
>  		default:
>  		case RECURSE_SUBMODULES_DEFAULT:
>  		case RECURSE_SUBMODULES_ON_DEMAND:
> -			if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names,
> -							 submodule->name))
> +			if (!submodule ||
> +			    !unsorted_string_list_lookup(
> +					&changed_submodule_names,
> +					submodule->name))
>  				continue;
>  			default_argv = "on-demand";
>  			break;

^ permalink raw reply	[relevance 5%]

* Re: [PATCH v4 4/9] submodule: move global changed_submodule_names into fetch submodule struct
  2018-09-25 19:47 ` [PATCH v4 4/9] submodule: move global changed_submodule_names into fetch submodule struct Stefan Beller
@ 2018-09-26 22:19   ` Junio C Hamano
  0 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-09-26 22:19 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

Stefan Beller <sbeller@google.com> writes:

> The `changed_submodule_names` are only used for fetching, so let's make it
> part of the struct that is passed around for fetching submodules.
>
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
>  submodule.c | 42 +++++++++++++++++++++++-------------------
>  1 file changed, 23 insertions(+), 19 deletions(-)

Yup, the less file-scope static we have and the more of them moved
to a struct, the closer we get to be able to use multiple of them at
the same time, which is a very nice step in the right direction.

>
> diff --git a/submodule.c b/submodule.c
> index 22c64bd8559..17103379ba4 100644
> --- a/submodule.c
> +++ b/submodule.c
> @@ -25,7 +25,7 @@
>  #include "commit-reach.h"
>  
>  static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
> -static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
> +
>  static int initialized_fetch_ref_tips;
>  static struct oid_array ref_tips_before_fetch;
>  static struct oid_array ref_tips_after_fetch;
> @@ -1110,7 +1110,22 @@ void check_for_new_submodule_commits(struct object_id *oid)
>  	oid_array_append(&ref_tips_after_fetch, oid);
>  }
>  
> -static void calculate_changed_submodule_paths(void)
> +struct submodule_parallel_fetch {
> +	int count;
> +	struct argv_array args;
> +	struct repository *r;
> +	const char *prefix;
> +	int command_line_option;
> +	int default_option;
> +	int quiet;
> +	int result;
> +
> +	struct string_list changed_submodule_names;
> +};
> +#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
> +
> +static void calculate_changed_submodule_paths(
> +	struct submodule_parallel_fetch *spf)
>  {
>  	struct argv_array argv = ARGV_ARRAY_INIT;
>  	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
> @@ -1148,7 +1163,8 @@ static void calculate_changed_submodule_paths(void)
>  			continue;
>  
>  		if (!submodule_has_commits(path, commits))
> -			string_list_append(&changed_submodule_names, name->string);
> +			string_list_append(&spf->changed_submodule_names,
> +					   name->string);
>  	}
>  
>  	free_submodules_oids(&changed_submodules);
> @@ -1185,18 +1201,6 @@ int submodule_touches_in_range(struct object_id *excl_oid,
>  	return ret;
>  }
>  
> -struct submodule_parallel_fetch {
> -	int count;
> -	struct argv_array args;
> -	struct repository *r;
> -	const char *prefix;
> -	int command_line_option;
> -	int default_option;
> -	int quiet;
> -	int result;
> -};
> -#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
> -
>  static int get_fetch_recurse_config(const struct submodule *submodule,
>  				    struct submodule_parallel_fetch *spf)
>  {
> @@ -1257,7 +1261,7 @@ static int get_next_submodule(struct child_process *cp,
>  		case RECURSE_SUBMODULES_ON_DEMAND:
>  			if (!submodule ||
>  			    !string_list_lookup(
> -					&changed_submodule_names,
> +					&spf->changed_submodule_names,
>  					submodule->name))
>  				continue;
>  			default_argv = "on-demand";
> @@ -1349,8 +1353,8 @@ int fetch_populated_submodules(struct repository *r,
>  	argv_array_push(&spf.args, "--recurse-submodules-default");
>  	/* default value, "--submodule-prefix" and its value are added later */
>  
> -	calculate_changed_submodule_paths();
> -	string_list_sort(&changed_submodule_names);
> +	calculate_changed_submodule_paths(&spf);
> +	string_list_sort(&spf.changed_submodule_names);
>  	run_processes_parallel(max_parallel_jobs,
>  			       get_next_submodule,
>  			       fetch_start_failure,
> @@ -1359,7 +1363,7 @@ int fetch_populated_submodules(struct repository *r,
>  
>  	argv_array_clear(&spf.args);
>  out:
> -	string_list_clear(&changed_submodule_names, 1);
> +	string_list_clear(&spf.changed_submodule_names, 1);
>  	return spf.result;
>  }

^ permalink raw reply	[relevance 5%]

* Re: On shipping more of our technical docs as manpages
      [irrelevant]                   ` <878t3oj8em.fsf@evledraar.gmail.com>
@ 2018-09-26 23:21                     ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-26 23:21 UTC (permalink / raw)
  To: Ævar Arnfjörð Bjarmason; +Cc: Jeff King, Junio C Hamano, git

On Wed, Sep 26, 2018 at 1:44 PM Ævar Arnfjörð Bjarmason
<avarab@gmail.com> wrote:

> > And if we were going to generate something external, would it make more
> > sense to write in a structured format like doxygen? I am not a big fan
> > of it myself, but at least from there you can generate a more richly
> > interconnected set of documentation.
>
> It's useful to have a single authoritative source for all documentation
> that's easy to search through.

If that is the case I would propose to keep it all in header files and
organize the headers.

> That includes stuff like perl585delta(1) which we'd stick in
> Documentation/RelNotes, and "Internals and C Language Interface". Most
> of what we'd put in Documentation/technical/api-* & headers is in
> perlapi(1).

This seems cool, but was also a recent introduction?
perl400delta seems to yield nothing for me (which may be because
I do not have an old version of perl installed?)

>
> Sometimes it's obvious where to look, but as someone who uses most of
> these APIs very occasionally (because I contribute less) I keep
> (re-)discovering various internal APIs.

Sometimes I have the same feeling. Maybe more structure in the
source files would help (e.g. datastructures/{strbuf, string-list}.h
and objects/{packfile.h, sha1*} ?

> Every perl installation also ships perlapi and a MB or so of obscure
> internal docs to everyone, which makes it easier to find via Google et
> al, which I find useful. So I guess I'm more on the side fence of
> dropping a hypothetical gitapi-oid-array into /usr/share/man than you
> are.
>

Personally I would not want to ship our internal docs everywhere
as it seems like overly wasteful. But then, only my early days
of childhood were guided by Internet that is not available anywhere
and everywhere. Today I would just take for granted that I can lookup
things in github/git/git when I am in not at my regular desk.

> ANYWAY
>
> This E-mail is much longer than I intended, sorry about that. I didn't
> just mean to bitch about how we ship docs, but I was wondering if there
> was a desire to move to something like what I've outlined above, or
> whether the status quo was mostly by design and intended.
>
> I just thought I'd write this rather lengthy E-Mails because this is one
> of the rare areas where you can fully describe an idea in E-Mail without
> writing any patches.
>
> If the consensus is that something like the exhaustive index "perldoc
> perl" provides wouldn't be useful for git I can just drop this, but if
> people are enthusiastic about having that it would be useful to work on
> it...

I agree with Junio, as that the discoverability is unrelated to where to store
the actual docs, Documentation/technical/* has the advantage that it
only has "good" stuff, i.e. some topic that someone cared enough to
write about and it is easy to guess if it is relevant to your need.
In *.h, we have a lot of false positives (xdiff-interface.h or cache.h
just pollute the searching space when looking for suitable storage
classes.)

So I wonder if we'd want to have a list (similar as in
command-list.txt) that describes the header files and gives
some classification of them, for example one class could be the
data structures (strbuf, stringlist, hashmap), algorithms
(diff, range-diff), OS specific stuff (run-command, trace, alloc)
or Git specific things (apply, fetch, submodule)

^ permalink raw reply	[relevance 4%]

* Re: [PATCH v5 9/9] submodule: support reading .gitmodules when it's not in the working tree
  2018-09-24 21:00       ` [PATCH v5 9/9] submodule: support reading .gitmodules when it's not in the working tree Stefan Beller
@ 2018-09-27 14:44         ` Antonio Ospite
  2018-09-27 18:00           ` Stefan Beller
  0 siblings, 1 reply; 200+ results
From: Antonio Ospite @ 2018-09-27 14:44 UTC (permalink / raw)
  To: Stefan Beller
  Cc: SZEDER Gábor, Junio C Hamano, git, Brandon Williams,
	Daniel Graña, Jonathan Nieder, Richard Hartmann,
	Ævar Arnfjörð Bjarmason

Hi Stefan,

On Mon, 24 Sep 2018 14:00:50 -0700
Stefan Beller <sbeller@google.com> wrote:

> On Mon, Sep 24, 2018 at 3:20 AM Antonio Ospite <ao2@ao2.it> wrote:
> 
[...]
> > This is a limitation of the object store in git, there is no equivalent
> > of get_oid() to get the oid from a specific repository and this affects
> > config_with_options too when the config source is a blob.
> 
> Not yet, as there is a big push to pass-through an object-store object
> or similar recently and rely less on global variables.
> I am not sure I get to this code, though.
>

If you end up touching get_oid() please CC me.

> > This does not affect commands called via "git -C submodule_dir cmd"
> > because in that case the chdir happens before the_repository is set up,
> > for instance "git-submodule $SOMETHING --recursive" commands seem to
> > change the working directory before the recursion.
> 
> For this it may be worth looking into the option
>        --super-prefix=<path>
>   Currently for internal use only. Set a prefix which gives a
>   path from above a repository down to its root. One use is
>   to give submodules context about the superproject that
>   invoked it.
>

My comment wanted to highlight that there are NO problems in the
mentioned cases:

  - git -C submodule_dir cmd
  - git submodule cmd --recursive

Are you suggesting to look into super-prefix for any reason in
particular?

[...]
> > The test suite passes even after removing repo_read_gitmodules()
> > entirely from builtin/grep.c, but I am still not confident that I get
> > all the implication of why that call was originally added in commit
> > f9ee2fcdfa (grep: recurse in-process using 'struct repository',
> > 2017-08-02).
> 
> If you checkout that commit and remove the call to repo_read_gitmodules
> and then call git-grep in a superproject with nested submodules, you
> get a segfault.
> 
> On master (and deleting out that line) you do not get the segfault,
> I think praise goes to ff6f1f564c4 (submodule-config: lazy-load a
> repository's .gitmodules file, 2017-08-03) which happened shortly
> after f9ee2fcdfa.
> 
> It showcased that it worked by converting ls-files, but left out grep.
> 
> So I think based on ff6f1f564c4 it is safe to remove all calls to
> repo_read_gitmodules.
>

Thanks for confirming.

> > Anyways, even if we removed the call we would prevent the problem from
> > happening in the test suite, but not in the real world, in case non-leaf
> > submodules without .gitmodules in their working tree.
> 
> Quite frankly I think grep was just overlooked in review of
> https://public-inbox.org/git/20170803182000.179328-14-bmwill@google.com/
> 

OK, so the plan for v6 is:

  - avoid the corruption issues spotted by Gábor by removing the call
    to repo_read_gitmodules in builtin/grep.c (this still does not fix
    the potential problem with nested submodules).

  - add a new test-tool which better exercises the new
    config_from_gitmodules code,

  - add also a test_expect_failure test to document the use case that
    cannot be supported yet: nested submodules without .gitmodules in
    their working tree.

Thanks,
   Antonio

-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 7%]

* Re: [PATCH v5 9/9] submodule: support reading .gitmodules when it's not in the working tree
      [irrelevant]       ` <xmqq5zyyyg9q.fsf@gitster-ct.c.googlers.com>
@ 2018-09-27 14:49         ` Antonio Ospite
  0 siblings, 0 replies; 200+ results
From: Antonio Ospite @ 2018-09-27 14:49 UTC (permalink / raw)
  To: Junio C Hamano
  Cc: SZEDER Gábor, git, Brandon Williams, Daniel Graña,
	Jonathan Nieder, Richard Hartmann, Stefan Beller,
	Ævar Arnfjörð Bjarmason

On Fri, 21 Sep 2018 09:19:45 -0700
Junio C Hamano <gitster@pobox.com> wrote:

> Antonio Ospite <ao2@ao2.it> writes:
> 
> > Protecting the problematic submodules function could work for now, but
> > I'd like to have more comments, my proposal is:
> >
> > diff --git a/builtin/grep.c b/builtin/grep.c
> > index 601f801158..52b45de749 100644
> > --- a/builtin/grep.c
> > +++ b/builtin/grep.c
> > @@ -427,6 +427,11 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
> >         if (repo_submodule_init(&submodule, superproject, path))
> >                 return 0;
> >
> > +       grep_read_lock();
> > +       /*
> > +        * NEEDSWORK: repo_read_gitmodules accesses the object store which is
> > +        * global, thus it needs to be protected.
> > +        */
> >         repo_read_gitmodules(&submodule);
> >
> >         /*
> > @@ -439,7 +444,6 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
> >          * store is no longer global and instead is a member of the repository
> >          * object.
> >          */
> > -       grep_read_lock();
> >         add_to_alternates_memory(submodule.objects->objectdir);
> >         grep_read_unlock();
> 
> I think this is in line with how the grep codepath protects itself
> when doing anything that accesses the object store.
> 

Thanks for the comment.

However, after confirming with Stefan Beller, I think we are going to
solve the corruption issue by removing this call to
repo_read_gitmodules(), which is not strictly necessary:

https://public-inbox.org/git/CAGZ79kZaomuE3p1puznM1x+hu-w4O+ZqeGUODBDj=-R3Z1hDzg@mail.gmail.com/

Thanks,
   Antonio

-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 4%]

* Re: [PATCH v5 9/9] submodule: support reading .gitmodules when it's not in the working tree
  2018-09-27 14:44         ` Antonio Ospite
@ 2018-09-27 18:00           ` Stefan Beller
  2018-10-01 15:45             ` Antonio Ospite
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-09-27 18:00 UTC (permalink / raw)
  To: Antonio Ospite
  Cc: SZEDER Gábor, Junio C Hamano, git, Brandon Williams,
	Daniel Graña, Jonathan Nieder, Richard Hartmann,
	Ævar Arnfjörð Bjarmason

On Thu, Sep 27, 2018 at 7:44 AM Antonio Ospite <ao2@ao2.it> wrote:
>
> If you end up touching get_oid() please CC me.

noted. I am not sure I'll touch it anytime soon, though.

>
> Are you suggesting to look into super-prefix for any reason in
> particular?

No, I misread the intent of that part of your message

> >
> > So I think based on ff6f1f564c4 it is safe to remove all calls to
> > repo_read_gitmodules.
> >
>
> Thanks for confirming.
>

> OK, so the plan for v6 is:
>
>   - avoid the corruption issues spotted by Gábor by removing the call
>     to repo_read_gitmodules in builtin/grep.c (this still does not fix
>     the potential problem with nested submodules).
>
>   - add a new test-tool which better exercises the new
>     config_from_gitmodules code,

Sounds good.

>
>   - add also a test_expect_failure test to document the use case that
>     cannot be supported yet: nested submodules without .gitmodules in
>     their working tree.

Personally I would want to live in a world where we don't *have* to nor
*want* to support submodules without .gitmodules in the respective
superproject.

We did support some use cases historically that I would make sure to
continue to support, but I am not sure how much effort we want to spend
on supporting further use cases of incomplete submodules.

Feel free to do so, as such tests help to document the boundaries.

Stefan

^ permalink raw reply	[relevance 7%]

* Re: [PATCH] submodule: Alllow staged changes for get_superproject_working_tree
      [irrelevant] <20180927181054.25802-1-sammck@gmail.com>
@ 2018-09-27 19:42 ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-27 19:42 UTC (permalink / raw)
  To: Sam McKelvie; +Cc: git

On Thu, Sep 27, 2018 at 11:12 AM Sam McKelvie <sammck@gmail.com> wrote:
>
> Invoking 'git rev-parse --show-superproject-working-tree' exits with
>
>     "fatal: BUG: returned path string doesn't match cwd?"
>
> when the superproject has an unmerged entry for the current submodule,
> instead of displaying the superproject's working tree.
>
> The problem is due to the fact that when a merge of the submodule reference
> is in progress, "git ls-files --stage —full-name <submodule-relative-path>”
> returns three seperate entries for the submodule (one for each stage) rather
> than a single entry; e.g.,
>
> $ git ls-files --stage --full-name submodule-child-test
> 160000 dbbd2766fa330fa741ea59bb38689fcc2d283ac5 1       submodule-child-test
> 160000 f174d1dbfe863a59692c3bdae730a36f2a788c51 2       submodule-child-test
> 160000 e6178f3a58b958543952e12824aa2106d560f21d 3       submodule-child-test
>
> The code in get_superproject_working_tree() expected exactly one entry to
> be returned; this patch makes it use the first entry if multiple entries
> are returned.
>
> Test t1500-rev-parse is extended to cover this case.
>
> Signed-off-by: Sam McKelvie <sammck@gmail.com>

Thanks for following up, this patch is
Reviewed-by: Stefan Beller <sbeller@google.com>

Thanks for adding the test as well!
Stefan

^ permalink raw reply	[relevance 7%]

* [PATCH] FYI / RFC: submodules: introduce repo-like workflow
@ 2018-09-27 22:16 Stefan Beller
  2018-09-27 22:27 ` Jonathan Nieder
  2018-09-28 19:26 ` Ævar Arnfjörð Bjarmason
  0 siblings, 2 replies; 200+ results
From: Stefan Beller @ 2018-09-27 22:16 UTC (permalink / raw)
  To: git, git-core; +Cc: jrnieder, Stefan Beller, Jonathan Nieder

Internally we have rolled out this as an experiment for
"submodules replacing the repo tool[1]". The repo tool is described as:

    Repo unifies Git repositories when necessary, performs uploads to the
    Gerrit revision control system, and automates parts of the Android
    development workflow. Repo is not meant to replace Git, only to make
    it easier to work with Git in the context of Android. The repo command
    is an executable Python script that you can put anywhere in your path.

    In working with the Android source files, you use Repo for
    across-network operations. For example, with a single Repo command you
    can download files from multiple repositories into your local working
    directory.

    In most situations, you can use Git instead of Repo, or mix Repo and
    Git commands to form complex commands.

[1] https://source.android.com/setup/develop/

Submodules can also be understood as unifying Git repositories, even more,
in the superproject the submodules have relationships between their
versions, which the repo tool only provides in releases.

The repo tool does not provide these relationships between versions, but
all the repositories (in case of Android think of ~1000 git repositories)
are put in their place without depending on each other.

This comes with a couple of advantages and disadvantages:

* Many users are familiar with Git, but not submodules. Each repository
  can be used independently with Git and there is no need to update the
  superproject or the repo manifest for a change in a repository.
* It is easy to work with repositories with no version-control-dependencies
  if there are dependencies in the code. In case of Android the
  repositories are bound at natural boundaries. For example the linux
  kernel is one repository, as then upstream work is made easy for this
  repository. So it is desirable to keep an easy-as-repo workflow.
* Fetching changes ("repo sync") needs to fetch all repositories, as there
  is no central place that tracks what has changed. In a superproject
  git fetch can determine which submodules need fetching.  In Androids
  case the daily change is only in a few repositories (think 10s), so
  migrating to a superproject would save an order of magnitude in fetch
  traffic for daily updates of developers.
* Sometimes when the dependencies are not on a clear repository boundary
  one would like to have git-bisect available across the different
  repositories, which repo cannot provide due to its design.

Internally we have the Gerrit as a central point, where the source of
truth is found for a given repository.

This patch adds a new mode to submodule handling, where the superproject
controls the existence of the submodule (just as current submodule
handling), but the submodule HEAD is not detached, but following the same
branch name as the superproject.

Current situation visualized:

  superproject
  HEAD -> "<branch name>" -> OID
                              |
  submodule                   v
  HEAD --------------------> OID

The OID in the submodule is controlled via the HEAD in the submodule that
is set accordingly to the gitlink in the superproject. Confusion can arise
when the (detached) HEAD in the submodule doesn't match the superprojects
gitlink.

This patch visualized:

  superproject
  HEAD -> "<branch name>" -> OID
                 |
  submodule      v
  HEAD -> "<branch name>" -> OID

As there is a central point of truth in our setup (our Gerrit installation)
which keeps the superproject and the submodule branches in sync, this
ought to look the same for the user; removing the "detached HEAD" in the
submodule. git-status will still notice if there is an OID mismatch between
the gitlink and the submodules branch, but that is a race condition and
should be caught by Gerrit.

This changes the following commands in the superproject:

  checkout -B/-b create branches in subs, too
  checkout (-f): update branch in submodule (create if needed) and check
                 it out; Pass the argument down literally if it is a branch
                 name (e.g. "checkout -f master" will run a
                            "checkout -f master" in the submodule as well)
  clone: see checkout
  reset --hard: see checkout

Change-Id: I69b361e5bd9d57226a0976fd36968cf9aeb52ae0
Signed-off-by: Jonathan Nieder <jrn@google.com>
Signed-off-by: Stefan Beller <sbeller@google.com>
---
 Documentation/git-checkout.txt | 13 +++--
 Makefile                       |  1 +
 branch.c                       | 84 +++++++++++++++++++++++++++++++++
 builtin.h                      |  1 +
 builtin/branch.c               | 35 ++++++++++++++
 builtin/checkout.c             | 76 ++++++++++++++++++++++++------
 builtin/clone.c                | 12 ++++-
 builtin/reset.c                | 41 ++++++++++++++--
 entry.c                        | 43 +++++++++++++----
 git-submodule.sh               | 24 +++++++++-
 git.c                          |  1 +
 submodule-move-head.c          | 81 ++++++++++++++++++++++++++++++++
 submodule-move-head.h          | 22 +++++++++
 submodule.c                    | 14 ++++++
 submodule.h                    |  9 +++-
 t/lib-submodule-update.sh      | 86 +++++++++++++++++++++++++++++++++-
 t/t1013-read-tree-submodule.sh |  1 +
 t/t2013-checkout-submodule.sh  |  4 ++
 t/t7112-reset-submodule.sh     |  5 ++
 unpack-trees.c                 | 20 ++++++--
 unpack-trees.h                 | 10 ++++
 21 files changed, 538 insertions(+), 45 deletions(-)
 create mode 100644 submodule-move-head.c
 create mode 100644 submodule-move-head.h

diff --git a/Documentation/git-checkout.txt b/Documentation/git-checkout.txt
index 9db02928c4..19b0c2a272 100644
--- a/Documentation/git-checkout.txt
+++ b/Documentation/git-checkout.txt
@@ -268,13 +268,12 @@ section of linkgit:git-add[1] to learn how to operate the `--patch` mode.
 	worktree.
 
 --[no-]recurse-submodules::
-	Using --recurse-submodules will update the content of all initialized
-	submodules according to the commit recorded in the superproject. If
-	local modifications in a submodule would be overwritten the checkout
-	will fail unless `-f` is used. If nothing (or --no-recurse-submodules)
-	is used, the work trees of submodules will not be updated.
-	Just like linkgit:git-submodule[1], this will detach the
-	submodules HEAD.
+	Using --recurse-submodules will update the content and current
+	branch of all initialized submodules in addition to the
+	superproject. If local modifications in a submodule would be
+	overwritten, the checkout will fail unless `-f` is used. If
+	nothing (or --no-recurse-submodules) is used, the work trees of
+	submodules will not be updated.
 
 <branch>::
 	Branch to checkout; if it refers to a branch (i.e., a name that,
diff --git a/Makefile b/Makefile
index 13e1c52478..6824dbcccf 100644
--- a/Makefile
+++ b/Makefile
@@ -975,6 +975,7 @@ LIB_OBJS += streaming.o
 LIB_OBJS += string-list.o
 LIB_OBJS += submodule.o
 LIB_OBJS += submodule-config.o
+LIB_OBJS += submodule-move-head.o
 LIB_OBJS += sub-process.o
 LIB_OBJS += symlinks.o
 LIB_OBJS += tag.o
diff --git a/branch.c b/branch.c
index 776f55fc66..e4538f2b36 100644
--- a/branch.c
+++ b/branch.c
@@ -1,9 +1,13 @@
 #include "git-compat-util.h"
 #include "cache.h"
 #include "config.h"
+#include "repository.h"
+#include "submodule.h"
 #include "branch.h"
 #include "refs.h"
 #include "refspec.h"
+#include "tree-walk.h"
+#include "run-command.h"
 #include "remote.h"
 #include "commit.h"
 #include "worktree.h"
@@ -242,6 +246,76 @@ N_("\n"
 "will track its remote counterpart, you may want to use\n"
 "\"git push -u\" to set the upstream config as you push.");
 
+static void create_branch_in_submodule(const char *name, const char *start_name,
+		   const char *start_ref, const struct object_id *start_oid,
+		   int force, int reflog, int clobber_head,
+		   int quiet, enum branch_track track,
+		   int checking_out_branch,
+		   const char *sub_path, const struct object_id *entry_oid)
+{
+	struct child_process cp = CHILD_PROCESS_INIT;
+
+	trace_printf("create_branch_in_submodule %s", sub_path);
+
+	prepare_submodule_repo_env(&cp.env_array);
+	cp.git_cmd = 1;
+	cp.dir = sub_path;
+	argv_array_push(&cp.args, "branch--helper");
+	argv_array_pushf(&cp.args, "--name=%s", name);
+	if (checking_out_branch) {
+		argv_array_pushf(&cp.args, "--start_name=%s",
+				 start_ref);
+	} else {
+		argv_array_pushf(&cp.args, "--start_name=%s",
+				 oid_to_hex(entry_oid));
+	}
+	argv_array_pushf(&cp.args, "--force=%d", force);
+	argv_array_pushf(&cp.args, "--reflog=%d", reflog);
+	argv_array_pushf(&cp.args, "--clobber_head=%d", clobber_head);
+	argv_array_pushf(&cp.args, "--quiet=%d", quiet);
+	argv_array_pushf(&cp.args, "--track=%d", track);
+
+	if (run_command(&cp))
+		fprintf(stderr, "process for submodule '%s' failed", sub_path);
+	child_process_clear(&cp);
+}
+
+static void create_branch_in_submodules(const char *name, const char *start_name,
+		   const char *start_ref, const struct object_id *start_oid,
+		   int force, int reflog, int clobber_head,
+		   int quiet, enum branch_track track, struct strbuf *rec_path)
+{
+
+	int checking_out_branch = start_ref && starts_with(start_ref, "refs/heads/");
+	void *buf;
+	struct tree_desc tree;
+	struct name_entry entry;
+	int rec_path_len = rec_path->len;
+
+	buf = fill_tree_descriptor(&tree, start_oid);
+	if (!buf)
+		die("could not read %s for checkout", start_name);
+
+	while (tree_entry(&tree, &entry)) {
+
+		if (rec_path->len > 0)
+			strbuf_addch(rec_path, '/');
+		strbuf_addstr(rec_path, entry.path);
+
+		trace_printf("create_branch_in_submodules %s %o", rec_path->buf, entry.mode);
+
+		if (S_ISGITLINK(entry.mode) && is_submodule_active(the_repository, rec_path->buf))
+			create_branch_in_submodule(name, start_name, start_ref,
+						start_oid, force, reflog, clobber_head,
+						quiet, track, checking_out_branch, rec_path->buf, entry.oid);
+		else if (S_ISDIR(entry.mode)) {
+			create_branch_in_submodules(name, start_name, start_ref, entry.oid, force, reflog, clobber_head, quiet, track, rec_path);
+		}
+		strbuf_setlen(rec_path, rec_path_len);
+	}
+	free(buf);
+}
+
 void create_branch(const char *name, const char *start_name,
 		   int force, int clobber_head_ok, int reflog,
 		   int quiet, enum branch_track track)
@@ -250,6 +324,7 @@ void create_branch(const char *name, const char *start_name,
 	struct object_id oid;
 	char *real_ref;
 	struct strbuf ref = STRBUF_INIT;
+	struct strbuf sub_path = STRBUF_INIT;
 	int forcing = 0;
 	int dont_change_ref = 0;
 	int explicit_tracking = 0;
@@ -307,6 +382,14 @@ void create_branch(const char *name, const char *start_name,
 	if (reflog)
 		log_all_ref_updates = LOG_REFS_NORMAL;
 
+	/*
+	 * NEEDSWORK: Doesn't handle errors part-way through very well.
+	 */
+	trace_printf("create_branch need to update subs: %d", should_update_submodules());
+	if (behave_google_repo_like() && should_update_submodules())
+		create_branch_in_submodules(name, start_name, real_ref, &oid,
+					    force, reflog, clobber_head_ok, quiet, track, &sub_path);
+
 	if (!dont_change_ref) {
 		struct ref_transaction *transaction;
 		struct strbuf err = STRBUF_INIT;
@@ -333,6 +416,7 @@ void create_branch(const char *name, const char *start_name,
 		setup_tracking(ref.buf + 11, real_ref, track, quiet);
 
 	strbuf_release(&ref);
+	strbuf_release(&sub_path);
 	free(real_ref);
 }
 
diff --git a/builtin.h b/builtin.h
index 962f0489ab..31030f248f 100644
--- a/builtin.h
+++ b/builtin.h
@@ -135,6 +135,7 @@ extern int cmd_archive(int argc, const char **argv, const char *prefix);
 extern int cmd_bisect__helper(int argc, const char **argv, const char *prefix);
 extern int cmd_blame(int argc, const char **argv, const char *prefix);
 extern int cmd_branch(int argc, const char **argv, const char *prefix);
+extern int cmd_branch_helper(int argc, const char **argv, const char *prefix);
 extern int cmd_bundle(int argc, const char **argv, const char *prefix);
 extern int cmd_cat_file(int argc, const char **argv, const char *prefix);
 extern int cmd_checkout(int argc, const char **argv, const char *prefix);
diff --git a/builtin/branch.c b/builtin/branch.c
index c396c41533..b6db7ec5bf 100644
--- a/builtin/branch.c
+++ b/builtin/branch.c
@@ -36,6 +36,11 @@ static const char * const builtin_branch_usage[] = {
 	NULL
 };
 
+static const char * const builtin_branch_helper_usage[] = {
+	N_("git branch--helper"),
+	NULL
+};
+
 static const char *head;
 static struct object_id head_oid;
 
@@ -578,6 +583,36 @@ static int edit_branch_description(const char *branch_name)
 	return 0;
 }
 
+int cmd_branch_helper(int argc, const char **argv, const char *prefix)
+{
+	const char *name = NULL, *start_name = NULL;
+	int force = 0, reflog = 0, clobber_head = 0, quiet = 0;
+	enum branch_track track = BRANCH_TRACK_NEVER;
+
+	struct option options[] = {
+		OPT_STRING(0, "name", &name, N_(""), N_("")),
+		OPT_STRING(0, "start_name", &start_name, N_(""), N_("")),
+
+		OPT_INTEGER(0, "force", &force, N_("")),
+		OPT_INTEGER(0, "reflog", &reflog, N_("")),
+		OPT_INTEGER(0, "clobber_head", &clobber_head, N_("")),
+		OPT_INTEGER(0, "quiet", &quiet, N_("")),
+
+		/* implicit int -> enum conversion */
+		OPT_INTEGER(0, "track", &track, N_("")),
+		OPT_END(),
+	};
+
+	argc = parse_options(argc, argv, prefix, options,
+			     builtin_branch_helper_usage, 0);
+	if (argc > 0)
+		die (_("branchhelper doesn't know about %s"), argv[0]);
+
+	create_branch(name, start_name, force, clobber_head, reflog,
+		      quiet, track);
+	return 0;
+}
+
 int cmd_branch(int argc, const char **argv, const char *prefix)
 {
 	int delete = 0, rename = 0, copy = 0, force = 0, list = 0;
diff --git a/builtin/checkout.c b/builtin/checkout.c
index b30b48767e..054d0ef60e 100644
--- a/builtin/checkout.c
+++ b/builtin/checkout.c
@@ -3,6 +3,7 @@
 #include "checkout.h"
 #include "lockfile.h"
 #include "parse-options.h"
+#include "repository.h"
 #include "refs.h"
 #include "object-store.h"
 #include "commit.h"
@@ -22,6 +23,7 @@
 #include "ll-merge.h"
 #include "resolve-undo.h"
 #include "submodule-config.h"
+#include "submodule-move-head.h"
 #include "submodule.h"
 #include "advice.h"
 
@@ -421,11 +423,24 @@ static void describe_detached_head(const char *msg, struct commit *commit)
 	strbuf_release(&sb);
 }
 
-static int reset_tree(struct tree *tree, const struct checkout_opts *o,
+struct branch_info {
+	const char *name; /* The short name used */
+	const char *path; /* The full name of a real branch */
+	struct commit *commit; /* The named commit */
+	/*
+	 * if not null the branch is detached because it's already
+	 * checked out in this checkout
+	 */
+	char *checkout;
+};
+
+static int reset_tree(struct branch_info *b, const struct checkout_opts *o,
 		      int worktree, int *writeout_error)
 {
 	struct unpack_trees_options opts;
+	struct submodule_move_head_options move_head_opts;
 	struct tree_desc tree_desc;
+	struct tree *tree = get_commit_tree(b->commit);
 
 	memset(&opts, 0, sizeof(opts));
 	opts.head_idx = -1;
@@ -437,6 +452,16 @@ static int reset_tree(struct tree *tree, const struct checkout_opts *o,
 	opts.verbose_update = o->show_progress;
 	opts.src_index = &the_index;
 	opts.dst_index = &the_index;
+
+	if (behave_google_repo_like()) {
+		opts.move_head = unpack_trees_move_head;
+		memset(&move_head_opts, 0, sizeof(move_head_opts));
+		move_head_opts.force = 1;
+		move_head_opts.new_ref = b->path;
+		move_head_opts.target_ref = o->force_detach ? NULL : b->path;
+		opts.unpack_data = &move_head_opts;
+	}
+
 	parse_tree(tree);
 	init_tree_desc(&tree_desc, tree->buffer, tree->size);
 	switch (unpack_trees(1, &tree_desc, &opts)) {
@@ -456,17 +481,6 @@ static int reset_tree(struct tree *tree, const struct checkout_opts *o,
 	}
 }
 
-struct branch_info {
-	const char *name; /* The short name used */
-	const char *path; /* The full name of a real branch */
-	struct commit *commit; /* The named commit */
-	/*
-	 * if not null the branch is detached because it's already
-	 * checked out in this checkout
-	 */
-	char *checkout;
-};
-
 static void setup_branch_path(struct branch_info *branch)
 {
 	struct strbuf buf = STRBUF_INIT;
@@ -584,7 +598,7 @@ static int merge_working_tree(const struct checkout_opts *opts,
 
 	resolve_undo_clear();
 	if (opts->force) {
-		ret = reset_tree(get_commit_tree(new_branch_info->commit),
+		ret = reset_tree(new_branch_info,
 				 opts, 1, writeout_error);
 		if (ret)
 			return ret;
@@ -592,6 +606,7 @@ static int merge_working_tree(const struct checkout_opts *opts,
 		struct tree_desc trees[2];
 		struct tree *tree;
 		struct unpack_trees_options topts;
+		struct submodule_move_head_options mopts;
 
 		memset(&topts, 0, sizeof(topts));
 		topts.head_idx = -1;
@@ -607,6 +622,16 @@ static int merge_working_tree(const struct checkout_opts *opts,
 			return 1;
 		}
 
+		if (behave_google_repo_like()) {
+			topts.move_head = unpack_trees_move_head;
+			memset(&mopts, 0, sizeof(mopts));
+			mopts.old_ref = old_branch_info->path;
+			mopts.new_ref = new_branch_info->path;
+			mopts.target_ref = opts->force_detach
+					? NULL : new_branch_info->path;
+			topts.unpack_data = &mopts;
+		}
+
 		/* 2-way merge to the new branch */
 		topts.initial_checkout = is_cache_unborn();
 		topts.update = 1;
@@ -672,7 +697,7 @@ static int merge_working_tree(const struct checkout_opts *opts,
 			o.verbosity = 0;
 			work = write_tree_from_memory(&o);
 
-			ret = reset_tree(get_commit_tree(new_branch_info->commit),
+			ret = reset_tree(new_branch_info,
 					 opts, 1,
 					 writeout_error);
 			if (ret)
@@ -687,7 +712,7 @@ static int merge_working_tree(const struct checkout_opts *opts,
 					  &result);
 			if (ret < 0)
 				exit(128);
-			ret = reset_tree(get_commit_tree(new_branch_info->commit),
+			ret = reset_tree(new_branch_info,
 					 opts, 0,
 					 writeout_error);
 			strbuf_release(&o.obuf);
@@ -722,6 +747,18 @@ static void report_tracking(struct branch_info *new_branch_info)
 	strbuf_release(&sb);
 }
 
+static void create_symref_in_submodules(const char *symref, const char *target, const char *logmsg)
+{
+	int i = 0;
+	for (i = 0; i < active_nr; i++) {
+		const struct cache_entry *ce = active_cache[i];
+		if (!S_ISGITLINK(ce->ce_mode) || !is_submodule_active(the_repository, ce->name))
+			continue;
+
+		create_symref_in_submodule(ce->name, symref, target, logmsg);
+	}
+}
+
 static void update_refs_for_switch(const struct checkout_opts *opts,
 				   struct branch_info *old_branch_info,
 				   struct branch_info *new_branch_info)
@@ -732,6 +769,9 @@ static void update_refs_for_switch(const struct checkout_opts *opts,
 		if (opts->new_orphan_branch) {
 			char *refname;
 
+			if (should_update_submodules())
+				die("--orphan --recurse-submodules is not implemented");
+
 			refname = mkpathdup("refs/heads/%s", opts->new_orphan_branch);
 			if (opts->new_branch_log &&
 			    !should_autocreate_reflog(refname)) {
@@ -784,6 +824,12 @@ static void update_refs_for_switch(const struct checkout_opts *opts,
 			describe_detached_head(_("HEAD is now at"), new_branch_info->commit);
 		}
 	} else if (new_branch_info->path) {	/* Switch branches. */
+		/*
+		 * NEEDSWORK: We don't handle attachment on checkout <branch>
+		 * yet.
+		 */
+		if (opts->new_branch && should_update_submodules() && behave_google_repo_like())
+			create_symref_in_submodules("HEAD", new_branch_info->path, msg.buf);
 		if (create_symref("HEAD", new_branch_info->path, msg.buf) < 0)
 			die(_("unable to update HEAD"));
 		if (!opts->quiet) {
diff --git a/builtin/clone.c b/builtin/clone.c
index 15b142d646..d5936ea1c7 100644
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -28,6 +28,7 @@
 #include "run-command.h"
 #include "connected.h"
 #include "packfile.h"
+#include "submodule.h"
 #include "list-objects-filter-options.h"
 #include "object-store.h"
 
@@ -738,7 +739,6 @@ static int checkout(int submodule_progress)
 		if (!starts_with(head, "refs/heads/"))
 			die(_("HEAD not found below refs/heads!"));
 	}
-	free(head);
 
 	/* We need to be in the new work tree for the checkout */
 	setup_work_tree();
@@ -767,8 +767,17 @@ static int checkout(int submodule_progress)
 			   oid_to_hex(&oid), "1", NULL);
 
 	if (!err && (option_recurse_submodules.nr > 0)) {
+		const char *branch;
 		struct argv_array args = ARGV_ARRAY_INIT;
 		argv_array_pushl(&args, "submodule", "update", "--init", "--recursive", NULL);
+		if (behave_google_repo_like()) {
+			if (!strcmp(head, "HEAD"))
+				; /* detach HEAD in submodules, too. */
+			else if (skip_prefix(head, "refs/heads/", &branch))
+				argv_array_pushl(&args, "--checkout-branch", branch, NULL);
+			else
+				BUG("HEAD not found below refs/heads!");
+		}
 
 		if (option_shallow_submodules == 1)
 			argv_array_push(&args, "--depth=1");
@@ -786,6 +795,7 @@ static int checkout(int submodule_progress)
 		argv_array_clear(&args);
 	}
 
+	free(head);
 	return err;
 }
 
diff --git a/builtin/reset.c b/builtin/reset.c
index 11cd0dcb8c..9cc8c4db52 100644
--- a/builtin/reset.c
+++ b/builtin/reset.c
@@ -24,6 +24,7 @@
 #include "cache-tree.h"
 #include "submodule.h"
 #include "submodule-config.h"
+#include "submodule-move-head.h"
 
 static const char * const git_reset_usage[] = {
 	N_("git reset [--mixed | --soft | --hard | --merge | --keep] [-q] [<commit>]"),
@@ -42,12 +43,16 @@ static inline int is_merge(void)
 	return !access(git_path_merge_head(the_repository), F_OK);
 }
 
-static int reset_index(const struct object_id *oid, int reset_type, int quiet)
+static int reset_index(const char *rev, const struct object_id *oid, int reset_type, int quiet)
 {
-	int i, nr = 0;
+	int i, nr = 0, flags = 0;
 	struct tree_desc desc[2];
 	struct tree *tree;
 	struct unpack_trees_options opts;
+	struct submodule_move_head_options mopts;
+	char *current_branch = NULL;
+	struct object_id discard;
+	char *new_ref = NULL;
 	int ret = -1;
 
 	memset(&opts, 0, sizeof(opts));
@@ -65,6 +70,31 @@ static int reset_index(const struct object_id *oid, int reset_type, int quiet)
 		break;
 	case HARD:
 		opts.update = 1;
+
+		if (behave_google_repo_like()) {
+			/*
+			 * Submodule handling:
+			 * - unless we are detached, attach HEAD in submodules
+			 * - if rev is a branch name, use that branch instead of oid in
+			 *   submodules.
+			 */
+			current_branch = resolve_refdup("HEAD", 0, NULL, &flags);
+			if (!(flags & REF_ISSYMREF))
+				current_branch = NULL;
+			if (dwim_ref(rev, strlen(rev), &discard, &new_ref) != 1 ||
+			    !starts_with(new_ref, "refs/heads/")) {
+				free(new_ref);
+				new_ref = NULL;
+			}
+
+			opts.move_head = unpack_trees_move_head;
+
+			memset(&mopts, 0, sizeof(mopts));
+			mopts.force = 1;
+			mopts.new_ref = new_ref;
+			mopts.target_ref = current_branch;
+			opts.unpack_data = &mopts;
+		}
 		/* fallthrough */
 	default:
 		opts.reset = 1;
@@ -101,6 +131,8 @@ static int reset_index(const struct object_id *oid, int reset_type, int quiet)
 out:
 	for (i = 0; i < nr; i++)
 		free((void *)desc[i].buffer);
+	free(current_branch);
+	free(new_ref);
 	return ret;
 }
 
@@ -317,6 +349,7 @@ int cmd_reset(int argc, const char **argv, const char *prefix)
 		oidcpy(&oid, the_hash_algo->empty_tree);
 	} else if (!pathspec.nr) {
 		struct commit *commit;
+
 		if (get_oid_committish(rev, &oid))
 			die(_("Failed to resolve '%s' as a valid revision."), rev);
 		commit = lookup_commit_reference(the_repository, &oid);
@@ -379,9 +412,9 @@ int cmd_reset(int argc, const char **argv, const char *prefix)
 				refresh_index(&the_index, flags, NULL, NULL,
 					      _("Unstaged changes after reset:"));
 		} else {
-			int err = reset_index(&oid, reset_type, quiet);
+			int err = reset_index(rev, &oid, reset_type, quiet);
 			if (reset_type == KEEP && !err)
-				err = reset_index(&oid, MIXED, quiet);
+				err = reset_index(rev, &oid, MIXED, quiet);
 			if (err)
 				die(_("Could not reset index file to revision '%s'."), rev);
 		}
diff --git a/entry.c b/entry.c
index 5d136c5d55..eee5bc498e 100644
--- a/entry.c
+++ b/entry.c
@@ -2,11 +2,23 @@
 #include "blob.h"
 #include "object-store.h"
 #include "dir.h"
+#include "unpack-trees.h"
 #include "streaming.h"
 #include "submodule.h"
 #include "progress.h"
 #include "fsmonitor.h"
 
+/* NEEDSWORK: share code with unpack-trees.c */
+static int move_head(const struct unpack_trees_options *o, const char *path, const char *old, const char *new, unsigned flags)
+{
+	if (behave_google_repo_like()) {
+		if (!o || !o->move_head)
+			return submodule_move_head(path, old, new, flags);
+		return o->move_head(o, path, old, new, flags);
+	} else
+		return submodule_move_head(path, old, new, flags);
+}
+
 static void create_directories(const char *path, int path_len,
 			       const struct checkout *state)
 {
@@ -251,7 +263,7 @@ int finish_delayed_checkout(struct checkout *state)
 	return errs;
 }
 
-static int write_entry(struct cache_entry *ce,
+static int write_entry(struct unpack_trees_options *o, struct cache_entry *ce,
 		       char *path, const struct checkout *state, int to_tempfile)
 {
 	unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT;
@@ -357,7 +369,7 @@ static int write_entry(struct cache_entry *ce,
 			return error("cannot create submodule directory %s", path);
 		sub = submodule_from_ce(ce);
 		if (sub)
-			return submodule_move_head(ce->name,
+			return move_head(o, ce->name,
 				NULL, oid_to_hex(&ce->oid),
 				state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
 		break;
@@ -435,14 +447,15 @@ static void mark_colliding_entries(const struct checkout *state,
  * its name is returned in topath[], which must be able to hold at
  * least TEMPORARY_FILENAME_LENGTH bytes long.
  */
-int checkout_entry(struct cache_entry *ce,
-		   const struct checkout *state, char *topath)
+int unpack_trees_checkout_entry(struct unpack_trees_options *o,
+				struct cache_entry *ce,
+				const struct checkout *state, char *topath)
 {
 	static struct strbuf path = STRBUF_INIT;
 	struct stat st;
 
 	if (topath)
-		return write_entry(ce, topath, state, 1);
+		return write_entry(o, ce, topath, state, 1);
 
 	strbuf_reset(&path);
 	strbuf_add(&path, state->base_dir, state->base_dir_len);
@@ -466,10 +479,10 @@ int checkout_entry(struct cache_entry *ce,
 				if (!(st.st_mode & S_IFDIR))
 					unlink_or_warn(ce->name);
 
-				return submodule_move_head(ce->name,
+				return move_head(o, ce->name,
 					NULL, oid_to_hex(&ce->oid), 0);
 			} else
-				return submodule_move_head(ce->name,
+				return move_head(o, ce->name,
 					"HEAD", oid_to_hex(&ce->oid),
 					state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
 		}
@@ -506,5 +519,19 @@ int checkout_entry(struct cache_entry *ce,
 		return 0;
 
 	create_directories(path.buf, path.len, state);
-	return write_entry(ce, path.buf, state, 0);
+	return write_entry(o, ce, path.buf, state, 0);
+}
+
+/*
+ * Write the contents from ce out to the working tree.
+ *
+ * When topath[] is not NULL, instead of writing to the working tree
+ * file named by ce, a temporary file is created by this function and
+ * its name is returned in topath[], which must be able to hold at
+ * least TEMPORARY_FILENAME_LENGTH bytes long.
+ */
+int checkout_entry(struct cache_entry *ce,
+		   const struct checkout *state, char *topath)
+{
+	return unpack_trees_checkout_entry(NULL, ce, state, topath);
 }
diff --git a/git-submodule.sh b/git-submodule.sh
index 1b568e29b9..5e6fcbe06e 100755
--- a/git-submodule.sh
+++ b/git-submodule.sh
@@ -424,6 +424,18 @@ fetch_in_submodule () (
 	esac
 )
 
+# usage: checkout_in_submoodule "$sm_path" "$command" "${branch:-}" "$rev"
+checkout_in_submodule () (
+	sanitize_submodule_env &&
+	cd "$1" &&
+	if test -n "$3"
+	then
+		$2 -B "$3" "$4"
+	else
+		$2 "$4"
+	fi
+)
+
 #
 # Update each submodule path to correct revision, using clone and checkout as needed
 #
@@ -476,6 +488,11 @@ cmd_update()
 		--recursive)
 			recursive=1
 			;;
+		--checkout-branch)
+			update="checkout"
+			checkout_dest=$2
+			shift
+			;;
 		--checkout)
 			update="checkout"
 			;;
@@ -592,6 +609,11 @@ cmd_update()
 				die "$(eval_gettext "Fetched in submodule path '\$displaypath', but it did not contain \$sha1. Direct fetching of that commit failed.")"
 			fi
 
+			if test -n "$checkout_dest" && test "$update_module" != checkout
+			then
+				die "Cannot use --checkout-branch with update mode '$update_module'"
+			fi
+
 			must_die_on_failure=
 			case "$update_module" in
 			checkout)
@@ -621,7 +643,7 @@ cmd_update()
 				die "$(eval_gettext "Invalid update mode '$update_module' for submodule path '$path'")"
 			esac
 
-			if (sanitize_submodule_env; cd "$sm_path" && $command "$sha1")
+			if checkout_in_submodule "$sm_path" "$command" "$checkout_dest" "$sha1"
 			then
 				say "$say_msg"
 			elif test -n "$must_die_on_failure"
diff --git a/git.c b/git.c
index a6f4b44af5..3b9c112a36 100644
--- a/git.c
+++ b/git.c
@@ -447,6 +447,7 @@ static struct cmd_struct commands[] = {
 	{ "bisect--helper", cmd_bisect__helper, RUN_SETUP },
 	{ "blame", cmd_blame, RUN_SETUP },
 	{ "branch", cmd_branch, RUN_SETUP | DELAY_PAGER_CONFIG },
+	{ "branch--helper", cmd_branch_helper, RUN_SETUP },
 	{ "bundle", cmd_bundle, RUN_SETUP_GENTLY | NO_PARSEOPT },
 	{ "cat-file", cmd_cat_file, RUN_SETUP },
 	{ "check-attr", cmd_check_attr, RUN_SETUP },
diff --git a/submodule-move-head.c b/submodule-move-head.c
new file mode 100644
index 0000000000..4c8f49066f
--- /dev/null
+++ b/submodule-move-head.c
@@ -0,0 +1,81 @@
+#include "cache.h"
+#include "submodule-move-head.h"
+#include "repository.h"
+#include "submodule.h"
+#include "refs.h"
+#include "unpack-trees.h"
+#include "run-command.h"
+
+void create_symref_in_submodule(const char *path, const char *symref, const char *target, const char *logmsg)
+{
+	struct child_process cp = CHILD_PROCESS_INIT;
+
+	/* NEEDSWORK: What about sub-submodules? */
+	prepare_submodule_repo_env(&cp.env_array);
+	cp.git_cmd = 1;
+	cp.dir = path;
+	argv_array_pushl(&cp.args, "symbolic-ref", "-m", logmsg, symref, target, NULL);
+
+	if (run_command(&cp))
+		die("process for submodule '%s' failed", path);
+}
+
+static void create_ref_in_submodule(const char *path, const char *ref, const char *value)
+{
+	struct child_process cp = CHILD_PROCESS_INIT;
+
+	/* NEEDSWORK: set a reasonable reflog message. */
+	prepare_submodule_repo_env(&cp.env_array);
+	cp.git_cmd = 1;
+	cp.dir = path;
+	argv_array_pushl(&cp.args, "update-ref", ref, value, sha1_to_hex(null_sha1), NULL);
+
+	if (run_command(&cp))
+		die("process for submodule '%s' failed", path);
+}
+
+static int ref_exists_in_submodule(const char *submodule_path, const char *refname)
+{
+	struct ref_store *refs = get_submodule_ref_store(submodule_path);
+	if (!refs)
+		return 0;
+	return refs_resolve_ref_unsafe(refs, refname, RESOLVE_REF_READING, NULL, NULL) != NULL;
+}
+
+int unpack_trees_move_head(const struct unpack_trees_options *opt, const char *path, const char *old, const char *new, unsigned flags)
+{
+	struct submodule_move_head_options *o = opt->unpack_data;
+	const char *new_ref = o->new_ref;
+	const char *target_ref = o->target_ref;
+	const char *old_commit = old;
+	const char *new_commit = new;
+
+	/*
+	 * NEEDSWORK:
+	 * - set log message
+	 * - what about sub-submodules?
+	 */
+
+	if (!is_submodule_active(the_repository, path))
+		return 0;
+
+	if (old) {
+		if (o->force)
+			old_commit = "HEAD";
+		else if (o->old_ref && ref_exists_in_submodule(path, o->old_ref))
+			old_commit = o->old_ref;
+	}
+	if (new_ref && new && ref_exists_in_submodule(path, new_ref))
+		new_commit = new_ref;
+
+	if (target_ref)
+		flags |= SUBMODULE_MOVE_HEAD_SKIP_REF_UPDATE;
+	if (submodule_move_head(path, old_commit, new_commit, flags) < 0)
+		return -1;
+	if (new && target_ref && !(flags & SUBMODULE_MOVE_HEAD_DRY_RUN)) {
+		if (!ref_exists_in_submodule(path, target_ref))
+			create_ref_in_submodule(path, target_ref, new);
+		create_symref_in_submodule(path, "HEAD", target_ref, "msg");
+	}
+	return 0;
+}
diff --git a/submodule-move-head.h b/submodule-move-head.h
new file mode 100644
index 0000000000..80e2679038
--- /dev/null
+++ b/submodule-move-head.h
@@ -0,0 +1,22 @@
+#ifndef SUBMODULE_MOVE_HEAD_H
+#define SUBMODULE_MOVE_HEAD_H
+
+struct unpack_trees_options;
+
+/* NEEDSWORK: document */
+struct submodule_move_head_options {
+	int force;
+	const char *old_ref;
+	const char *new_ref;
+	const char *target_ref;
+};
+
+/*
+ * For use as unpack_trees_options.move_head. Parameters should be a
+ * struct submodule_move_head_options * in unpack_trees_options.unpack_data.
+ */
+extern int unpack_trees_move_head(const struct unpack_trees_options *opt, const char *path, const char *old, const char *new, unsigned flags);
+
+extern void create_symref_in_submodule(const char *path, const char *symref, const char *target, const char *logmsg);
+
+#endif /* SUBMODULE_MOVE_HEAD_H */
diff --git a/submodule.c b/submodule.c
index b53cb6e9c4..81eace805f 100644
--- a/submodule.c
+++ b/submodule.c
@@ -30,6 +30,17 @@ static int initialized_fetch_ref_tips;
 static struct oid_array ref_tips_before_fetch;
 static struct oid_array ref_tips_after_fetch;
 
+int behave_google_repo_like(void)
+{
+	static int google_repo_like = -1;
+
+	if (google_repo_like == -1)
+		git_config_get_bool("submodule.repolike", &google_repo_like);
+
+	return google_repo_like;
+}
+
+
 /*
  * Check if the .gitmodules file is unmerged. Parsing of the .gitmodules file
  * will be disabled because we can't guess what might be configured in
@@ -1676,6 +1687,9 @@ int submodule_move_head(const char *path,
 
 	if (!(flags & SUBMODULE_MOVE_HEAD_DRY_RUN)) {
 		if (new_head) {
+			if (flags & SUBMODULE_MOVE_HEAD_SKIP_REF_UPDATE)
+				goto out;
+
 			child_process_init(&cp);
 			/* also set the HEAD accordingly */
 			cp.git_cmd = 1;
diff --git a/submodule.h b/submodule.h
index e452919aa4..3346b1a12e 100644
--- a/submodule.h
+++ b/submodule.h
@@ -120,8 +120,11 @@ int push_unpushed_submodules(struct oid_array *commits,
  */
 int submodule_to_gitdir(struct strbuf *buf, const char *submodule);
 
-#define SUBMODULE_MOVE_HEAD_DRY_RUN (1<<0)
-#define SUBMODULE_MOVE_HEAD_FORCE   (1<<1)
+#define SUBMODULE_MOVE_HEAD_DRY_RUN             (1<<0)
+#define SUBMODULE_MOVE_HEAD_FORCE               (1<<1)
+#define SUBMODULE_MOVE_HEAD_SKIP_REF_UPDATE     (1<<2)
+
+/* NULL values for old or new represent the "missing" state. */
 int submodule_move_head(const char *path,
 			const char *old,
 			const char *new_head,
@@ -146,4 +149,6 @@ void absorb_git_dir_into_superproject(const char *prefix,
  */
 const char *get_superproject_working_tree(void);
 
+int behave_google_repo_like(void);
+
 #endif
diff --git a/t/lib-submodule-update.sh b/t/lib-submodule-update.sh
index 016391723c..71d5b506de 100755
--- a/t/lib-submodule-update.sh
+++ b/t/lib-submodule-update.sh
@@ -657,7 +657,54 @@ test_submodule_recursing_with_args_common() {
 			test_submodule_content sub1 origin/add_sub1
 		)
 	'
-	test_expect_success "$command: submodule branch is not changed, detach HEAD instead" '
+
+	if test "$KNOWN_FAILURE_SUBMODULE_REFS_NOT_UPDATED" = 1
+	then
+		RESULT=failure
+	else
+		RESULT=success
+	fi
+	if test "$KNOWN_DIFFERENCE_SUBMODULE_REFS_NOT_UPDATED" = "read-tree"
+	then
+	test_expect_$RESULT "$command: submodule branch is not changed, detach HEAD" '
+		prolog &&
+		reset_work_tree_to_interested add_sub1 &&
+		(
+			cd submodule_update &&
+			git -C sub1 checkout -b keep_branch &&
+			git -C sub1 rev-parse HEAD >expect &&
+			git branch -t modify_sub1 origin/modify_sub1 &&
+			$command modify_sub1 &&
+			test_superproject_content origin/modify_sub1 &&
+			test_submodule_content sub1 origin/modify_sub1 &&
+			git -C sub1 rev-parse keep_branch >actual &&
+			test_cmp expect actual &&
+			test_must_fail git -C sub1 symbolic-ref HEAD >actual
+		)
+	'
+	elif test "$KNOWN_DIFFERENCE_SUBMODULE_REFS_NOT_UPDATED" = "checkout -B current"
+	then
+	test_expect_$RESULT "$command: submodule branch is changed to 'current'" '
+		prolog &&
+		reset_work_tree_to_interested add_sub1 &&
+		(
+			cd submodule_update &&
+			git -C sub1 checkout -b keep_branch &&
+			git -C sub1 rev-parse HEAD >expect &&
+			git branch -t modify_sub1 origin/modify_sub1 &&
+			$command modify_sub1 &&
+			test_superproject_content origin/modify_sub1 &&
+			test_submodule_content sub1 origin/modify_sub1 &&
+			git -C sub1 rev-parse keep_branch >actual &&
+			test_cmp expect actual &&
+			echo refs/heads/current >expect &&
+			git -C sub1 symbolic-ref HEAD >actual &&
+			test_cmp expect actual
+		)
+	'
+	elif test "$KNOWN_DIFFERENCE_SUBMODULE_REFS_NOT_UPDATED" = "reset"
+	then
+	test_expect_$RESULT "$command: submodule branch is changed to superproject, resetting to target" '
 		prolog &&
 		reset_work_tree_to_interested add_sub1 &&
 		(
@@ -665,14 +712,49 @@ test_submodule_recursing_with_args_common() {
 			git -C sub1 checkout -b keep_branch &&
 			git -C sub1 rev-parse HEAD >expect &&
 			git branch -t modify_sub1 origin/modify_sub1 &&
+			git checkout -b newbranch &&
 			$command modify_sub1 &&
+
+			# we modified the wt
 			test_superproject_content origin/modify_sub1 &&
 			test_submodule_content sub1 origin/modify_sub1 &&
+
+			# keep_branch does not change
 			git -C sub1 rev-parse keep_branch >actual &&
 			test_cmp expect actual &&
-			test_must_fail git -C sub1 symbolic-ref HEAD
+
+			# the submodule is attached to the same branch as the superproject
+			git -C sub1 symbolic-ref HEAD >actual &&
+			echo refs/heads/newbranch >expect &&
+			test_cmp expect actual
 		)
 	'
+	else
+	test_expect_$RESULT "$command: submodule branch is changed" '
+		prolog &&
+		reset_work_tree_to_interested add_sub1 &&
+		(
+			cd submodule_update &&
+			git -C sub1 checkout -b keep_branch &&
+			git -C sub1 rev-parse HEAD >expect &&
+			git branch -t modify_sub1 origin/modify_sub1 &&
+			$command modify_sub1 &&
+
+			# modified wt
+			test_superproject_content origin/modify_sub1 &&
+			test_submodule_content sub1 origin/modify_sub1 &&
+
+			# unrelated keep_branch is fine
+			git -C sub1 rev-parse keep_branch >actual &&
+			test_cmp expect actual &&
+
+			# submodule ref is checked out
+			git -C sub1 symbolic-ref HEAD >actual &&
+			echo refs/heads/modify_sub1 >expect &&
+			test_cmp expect actual
+		)
+	'
+	fi
 
 	# Replacing a tracked file with a submodule produces a checked out submodule
 	test_expect_success "$command: replace tracked file with submodule checks out submodule" '
diff --git a/t/t1013-read-tree-submodule.sh b/t/t1013-read-tree-submodule.sh
index 91a6fafcb4..de59ebd121 100755
--- a/t/t1013-read-tree-submodule.sh
+++ b/t/t1013-read-tree-submodule.sh
@@ -7,6 +7,7 @@ test_description='read-tree can handle submodules'
 
 KNOWN_FAILURE_DIRECTORY_SUBMODULE_CONFLICTS=1
 KNOWN_FAILURE_SUBMODULE_OVERWRITE_IGNORED_UNTRACKED=1
+KNOWN_DIFFERENCE_SUBMODULE_REFS_NOT_UPDATED="read-tree"
 
 test_submodule_switch_recursing_with_args "read-tree -u -m"
 
diff --git a/t/t2013-checkout-submodule.sh b/t/t2013-checkout-submodule.sh
index 8f86b5f4b2..940f0fca20 100755
--- a/t/t2013-checkout-submodule.sh
+++ b/t/t2013-checkout-submodule.sh
@@ -66,6 +66,10 @@ test_expect_success '"checkout <submodule>" honors submodule.*.ignore from .git/
 KNOWN_FAILURE_DIRECTORY_SUBMODULE_CONFLICTS=1
 test_submodule_switch_recursing_with_args "checkout"
 
+KNOWN_DIFFERENCE_SUBMODULE_REFS_NOT_UPDATED="checkout -B current"
+test_submodule_switch_recursing_with_args "checkout -B current"
+unset KNOWN_DIFFERENCE_SUBMODULE_REFS_NOT_UPDATED
+
 test_submodule_forced_switch_recursing_with_args "checkout -f"
 
 test_submodule_switch "git checkout"
diff --git a/t/t7112-reset-submodule.sh b/t/t7112-reset-submodule.sh
index a1cb9ff858..c31acc6578 100755
--- a/t/t7112-reset-submodule.sh
+++ b/t/t7112-reset-submodule.sh
@@ -8,13 +8,18 @@ test_description='reset can handle submodules'
 KNOWN_FAILURE_SUBMODULE_RECURSIVE_NESTED=1
 KNOWN_FAILURE_DIRECTORY_SUBMODULE_CONFLICTS=1
 KNOWN_FAILURE_SUBMODULE_OVERWRITE_IGNORED_UNTRACKED=1
+KNOWN_DIFFERENCE_SUBMODULE_REFS_NOT_UPDATED="reset"
 
+KNOWN_FAILURE_SUBMODULE_REFS_NOT_UPDATED=1
 test_submodule_switch_recursing_with_args "reset --keep"
 
+unset KNOWN_FAILURE_SUBMODULE_REFS_NOT_UPDATED
 test_submodule_forced_switch_recursing_with_args "reset --hard"
 
+KNOWN_FAILURE_SUBMODULE_REFS_NOT_UPDATED=1
 test_submodule_switch "git reset --keep"
 
+unset KNOWN_FAILURE_SUBMODULE_REFS_NOT_UPDATED
 test_submodule_switch "git reset --merge"
 
 test_submodule_forced_switch "git reset --hard"
diff --git a/unpack-trees.c b/unpack-trees.c
index 51bfac6aa0..4d4a85226e 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -254,6 +254,16 @@ static void display_error_msgs(struct unpack_trees_options *o)
 		fprintf(stderr, _("Aborting\n"));
 }
 
+static int move_head(const struct unpack_trees_options *o, const char *path, const char *old, const char *new, unsigned flags)
+{
+	if (behave_google_repo_like()) {
+		if (!o->move_head)
+			return submodule_move_head(path, old, new, flags);
+		return o->move_head(o, path, old, new, flags);
+	} else
+		return submodule_move_head(path, old, new, flags);
+}
+
 static int check_submodule_move_head(const struct cache_entry *ce,
 				     const char *old_id,
 				     const char *new_id,
@@ -268,7 +278,7 @@ static int check_submodule_move_head(const struct cache_entry *ce,
 	if (o->reset)
 		flags |= SUBMODULE_MOVE_HEAD_FORCE;
 
-	if (submodule_move_head(ce->name, old_id, new_id, flags))
+	if (move_head(o, ce->name, old_id, new_id, flags))
 		return o->gently ? -1 :
 				   add_rejected_path(o, ERROR_WOULD_LOSE_SUBMODULE, ce->name);
 	return 0;
@@ -304,12 +314,12 @@ static void load_gitmodules_file(struct index_state *index,
  * Unlink the last component and schedule the leading directories for
  * removal, such that empty directories get removed.
  */
-static void unlink_entry(const struct cache_entry *ce)
+static void unlink_entry(const struct unpack_trees_options *o, const struct cache_entry *ce)
 {
 	const struct submodule *sub = submodule_from_ce(ce);
 	if (sub) {
 		/* state.force is set at the caller. */
-		submodule_move_head(ce->name, "HEAD", NULL,
+		move_head(o, ce->name, "HEAD", NULL,
 				    SUBMODULE_MOVE_HEAD_FORCE);
 	}
 	if (!check_leading_path(ce->name, ce_namelen(ce)))
@@ -408,7 +418,7 @@ static int check_updates(struct unpack_trees_options *o)
 		if (ce->ce_flags & CE_WT_REMOVE) {
 			display_progress(progress, ++cnt);
 			if (o->update && !o->dry_run)
-				unlink_entry(ce);
+				unlink_entry(o, ce);
 		}
 	}
 	remove_marked_cache_entries(index);
@@ -450,7 +460,7 @@ static int check_updates(struct unpack_trees_options *o)
 			display_progress(progress, ++cnt);
 			ce->ce_flags &= ~CE_UPDATE;
 			if (o->update && !o->dry_run) {
-				errs |= checkout_entry(ce, &state, NULL);
+				errs |= unpack_trees_checkout_entry(o, ce, &state, NULL);
 			}
 		}
 	}
diff --git a/unpack-trees.h b/unpack-trees.h
index 0135080a7b..3f1a205ad0 100644
--- a/unpack-trees.h
+++ b/unpack-trees.h
@@ -9,12 +9,15 @@
 #define MAX_UNPACK_TREES 8
 
 struct cache_entry;
+struct tree_desc;
 struct unpack_trees_options;
 struct exclude_list;
 
 typedef int (*merge_fn_t)(const struct cache_entry * const *src,
 		struct unpack_trees_options *options);
 
+typedef int (*submodule_move_head_fn)(const struct unpack_trees_options *o, const char *path, const char *old, const char *new, unsigned flags);
+
 enum unpack_trees_error_types {
 	ERROR_WOULD_OVERWRITE = 0,
 	ERROR_NOT_UPTODATE_FILE,
@@ -65,6 +68,7 @@ struct unpack_trees_options {
 	struct dir_struct *dir;
 	struct pathspec *pathspec;
 	merge_fn_t fn;
+	submodule_move_head_fn move_head;
 	const char *msgs[NB_UNPACK_TREES_ERROR_TYPES];
 	struct argv_array msgs_to_free;
 	/*
@@ -86,6 +90,12 @@ struct unpack_trees_options {
 	struct exclude_list *el; /* for internal use */
 };
 
+/* defined in entry.c, for internal use */
+int unpack_trees_checkout_entry(struct unpack_trees_options *o,
+				struct cache_entry *ce,
+				const struct checkout *state,
+				char *topath);
+
 int unpack_trees(unsigned n, struct tree_desc *t,
 		 struct unpack_trees_options *options);
 
-- 
2.19.0


^ permalink raw reply	[relevance 20%]

* Re: [PATCH] FYI / RFC: submodules: introduce repo-like workflow
  2018-09-27 22:16 [PATCH] FYI / RFC: submodules: introduce repo-like workflow Stefan Beller
@ 2018-09-27 22:27 ` Jonathan Nieder
  2018-09-28 18:08   ` Junio C Hamano
  2018-09-28 19:26 ` Ævar Arnfjörð Bjarmason
  1 sibling, 1 reply; 200+ results
From: Jonathan Nieder @ 2018-09-27 22:27 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

(dropping cc-s to my internal address that I don't use on this list
 and to git-core@google.com which bounces)
Hi,

Stefan Beller wrote:

> Internally we have rolled out this as an experiment for
> "submodules replacing the repo tool[1]". The repo tool is described as:
>
>     Repo unifies Git repositories when necessary, performs uploads to the
>     Gerrit revision control system, and automates parts of the Android
>     development workflow. Repo is not meant to replace Git, only to make
>     it easier to work with Git in the context of Android. The repo command
>     is an executable Python script that you can put anywhere in your path.
[...]
> Submodules can also be understood as unifying Git repositories, even more,
> in the superproject the submodules have relationships between their
> versions, which the repo tool only provides in releases.
>
> The repo tool does not provide these relationships between versions, but
> all the repositories (in case of Android think of ~1000 git repositories)
> are put in their place without depending on each other.
>
> This comes with a couple of advantages and disadvantages:

Thanks for describing this background.

[...]
> This changes the following commands in the superproject:
>
>   checkout -B/-b create branches in subs, too
>   checkout (-f): update branch in submodule (create if needed) and check
>                  it out; Pass the argument down literally if it is a branch
>                  name (e.g. "checkout -f master" will run a
>                             "checkout -f master" in the submodule as well)
>   clone: see checkout
>   reset --hard: see checkout

As you mentioned, I've been using this submodule.repoLike=true mode
for my own use for a while.  You did a nice job of explaining on how
it fits into a Gerrit-driven workflow; I'd like to add that I find it
pleasant in non-Gerrit-driven contexts as well.

The primary difference from repoLike=false is that this makes the
normal state to have branches checked out in submodules.  For example,
if I run

	git checkout --recurse-submodules -B master origin/master

then this will create and check out a "master" branch in all
submodules instead of only in the superproject.  This helps avoid some
issues in Git's submodule handling where submodule commits can be
pruned if they have not been checked out in a while because there is
no ref pointing to them.

Some next steps:

- now that we have a repository object, some of the implementation can
  be simplified and made more robust.  I expect that will also make
  these patches easier to review

- also in the direction of reviewability, at that point we may want to
  split this into multiple patches

- gitsubmodules.txt and config.txt should describe the new option, to
  help new users understand what this new repoLike workflow does

- there are some edge cases in the UX that get... messy that I should
  describe in another message

All that said, thanks for sending this out, and I'd be happy to hear
from any interested people --- feedback from anyone adventurous enough
to try this out would be very welcome.

Happy hacking,
Jonathan

^ permalink raw reply	[relevance 7%]

* Re: [PATCH] FYI / RFC: submodules: introduce repo-like workflow
  2018-09-27 22:27 ` Jonathan Nieder
@ 2018-09-28 18:08   ` Junio C Hamano
  0 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-09-28 18:08 UTC (permalink / raw)
  To: Jonathan Nieder; +Cc: Stefan Beller, git

Jonathan Nieder <jrnieder@gmail.com> writes:

> (dropping cc-s to my internal address that I don't use on this list
>  and to git-core@google.com which bounces)
> Hi,
>
> Stefan Beller wrote:
>
>> Internally we have rolled out this as an experiment for
>> "submodules replacing the repo tool[1]". The repo tool is described as:
>>
>>     Repo unifies Git repositories when necessary, performs uploads to the
>>     Gerrit revision control system, and automates parts of the Android
>>     development workflow. Repo is not meant to replace Git, only to make
>>     it easier to work with Git in the context of Android. The repo command
>>     is an executable Python script that you can put anywhere in your path.
> [...]
>> Submodules can also be understood as unifying Git repositories, even more,
>> in the superproject the submodules have relationships between their
>> versions, which the repo tool only provides in releases.
>>
>> The repo tool does not provide these relationships between versions, but
>> all the repositories (in case of Android think of ~1000 git repositories)
>> are put in their place without depending on each other.
>>
>> This comes with a couple of advantages and disadvantages:
>
> Thanks for describing this background.

Thanks for this.  I probably won't be reading this before other
topics, but I've often found that changes from google were lacking
the backstory to make sense of them as a coherent whole.  

For example, a patch that says "Instead of detaching at the commit
recorded in the superproject, check out the branch with the same
name, even if it points at a different commit.  Here is the write up
of what it does in the Documentation/ and code" is hard to judge
beyond checking if the code does what it claims to do and if the
docs describe what the code does---without backstory like this that
talks about how individual small pieces fit within the plan for the
whole thing, judging if that "what it claims to do" is even sensible
is impossible.

^ permalink raw reply	[relevance 2%]

* Re: [PATCH] FYI / RFC: submodules: introduce repo-like workflow
  2018-09-27 22:16 [PATCH] FYI / RFC: submodules: introduce repo-like workflow Stefan Beller
  2018-09-27 22:27 ` Jonathan Nieder
@ 2018-09-28 19:26 ` Ævar Arnfjörð Bjarmason
  2018-09-28 20:23   ` Stefan Beller
  1 sibling, 1 reply; 200+ results
From: Ævar Arnfjörð Bjarmason @ 2018-09-28 19:26 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git, jrnieder


On Thu, Sep 27 2018, Stefan Beller wrote:

> Internally we have rolled out this as an experiment for
> "submodules replacing the repo tool[1]". The repo tool is described as:
>
>     Repo unifies Git repositories when necessary, performs uploads to the
>     Gerrit revision control system, and automates parts of the Android
>     development workflow. Repo is not meant to replace Git, only to make
>     it easier to work with Git in the context of Android. The repo command
>     is an executable Python script that you can put anywhere in your path.
>
>     In working with the Android source files, you use Repo for
>     across-network operations. For example, with a single Repo command you
>     can download files from multiple repositories into your local working
>     directory.
>
>     In most situations, you can use Git instead of Repo, or mix Repo and
>     Git commands to form complex commands.
>
> [1] https://source.android.com/setup/develop/

Some questions just out of curiosity, not for this patch in particular:

Those docs seem to describe the situation without this patch, with this
patch is the repo tool fully replaced?

How are you planning to migrate from repo to this on a repository-data
basis, does gerrit also populate .gitmodules files appropriately, which
your clone --recurse-submodules will pick up, but repo will just ignore,
so you can use the two in parallel?

Now "repo init -u" takes a URL to a manifest of repositories to stitch
together, I've understood from past conversations (but am not sure) that
this is used e.g. by downstream Android vendors so they can use what
Google's using + whatever they have in-house, i.e. make the manifest the
set of open source repos plus some (e.g. drivers specific to their
device). How is that sort of workflow going to work where you
(presumably) have do that via .gitmodules + commit entries in trees?
They run their own Gerrit install with some magic to sync back & forth?

I assume that now the recursive "checkout" relies on all the origin/HEAD
symbolic refs pointing to "master", but how is this going to deal with
incorporating a repo whose main branch has a different name?
E.g. "trunk" or "blead"? Perhaps some interaction with
checkout.defaultRemote + submodule.<name>.branch=<NAME> could make "git
checkout :mainbranch" DWYM.

> * Fetching changes ("repo sync") needs to fetch all repositories, as there
>   is no central place that tracks what has changed. In a superproject
>   git fetch can determine which submodules need fetching.  In Androids
>   case the daily change is only in a few repositories (think 10s), so
>   migrating to a superproject would save an order of magnitude in fetch
>   traffic for daily updates of developers.

Interesting that in all this time with the reliance on a central server
repo wasn't already asking some custom API "what repos changed since xyz
time" to narrow that down, but hey, .gitmodules + commit entries in
trees will do it for you.

> * Sometimes when the dependencies are not on a clear repository boundary
>   one would like to have git-bisect available across the different
>   repositories, which repo cannot provide due to its design.

I assume that you're not upgrading independently to e.g. every single
linux commit, just stable releases, so does bisecting deal with knowing
that e.g. a breakage occurred when linux.git was updated from v4.10 to
v4.12, and then to go within the repo itself and bisect from there, or
is that done manually?

^ permalink raw reply	[relevance 6%]

* Re: [PATCH] FYI / RFC: submodules: introduce repo-like workflow
  2018-09-28 19:26 ` Ævar Arnfjörð Bjarmason
@ 2018-09-28 20:23   ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-09-28 20:23 UTC (permalink / raw)
  To: Ævar Arnfjörð Bjarmason; +Cc: git, Jonathan Nieder

On Fri, Sep 28, 2018 at 12:26 PM Ævar Arnfjörð Bjarmason
<avarab@gmail.com> wrote:
>
>
> On Thu, Sep 27 2018, Stefan Beller wrote:
>
> > Internally we have rolled out this as an experiment for
> > "submodules replacing the repo tool[1]". The repo tool is described as:
> >
> >     Repo unifies Git repositories when necessary, performs uploads to the
> >     Gerrit revision control system, and automates parts of the Android
> >     development workflow. Repo is not meant to replace Git, only to make
> >     it easier to work with Git in the context of Android. The repo command
> >     is an executable Python script that you can put anywhere in your path.
> >
> >     In working with the Android source files, you use Repo for
> >     across-network operations. For example, with a single Repo command you
> >     can download files from multiple repositories into your local working
> >     directory.
> >
> >     In most situations, you can use Git instead of Repo, or mix Repo and
> >     Git commands to form complex commands.
> >
> > [1] https://source.android.com/setup/develop/
>
> Some questions just out of curiosity, not for this patch in particular:
>
> Those docs seem to describe the situation without this patch, with this
> patch is the repo tool fully replaced?

Yeah I started by describing the status quo. Some points to add:

* The repo UX looks very similar to perforce, as that was used internally
   at Google at the time a lot. Git wasn't around for long in the days of
   early Android. Now that Git is well known, new grads joining are more likely
   to know Git than perforce.
* repo has no tests because it started as a small script
* maintenance/releases of repo are not ideal.

So yes, one of our long term goals is to replace repo with Git-only
(on the client side) workflow.

> How are you planning to migrate from repo to this on a repository-data
> basis

We don't plan to migrate the existing clients. You'd have to re-clone.

>, does gerrit also populate .gitmodules files appropriately, which
> your clone --recurse-submodules will pick up, but repo will just ignore,
> so you can use the two in parallel?

Gerrit has native support to update submodules in a superproject.
So if you submit code to a project that is a submodule of a superproject
the superproject updates its gitlink. (and if you use a topic based workflow
to submit to two projects, it will show up as an atomic update in the
superproject by having just one commit updating two gitlinks)

Also Gerrit has a plugin "supermanifest" [1], which tracks repo
manifests and mirrors changes from the manifest into the
superproject, e.g. adding a new project to the manifest will
add a new submodule to the superproject.

[1] https://gerrit.googlesource.com/plugins/supermanifest/

With both Gerrits internal superproject subscription and that plugin
it should be possible to use repo or git-submodule in parallel (on an
organisational level, i.e. you choose one of them, and your coworker
chooses the other one)

> Now "repo init -u" takes a URL to a manifest of repositories to stitch
> together, I've understood from past conversations (but am not sure) that
> this is used e.g. by downstream Android vendors so they can use what
> Google's using + whatever they have in-house, i.e. make the manifest the
> set of open source repos plus some (e.g. drivers specific to their
> device). How is that sort of workflow going to work where you
> (presumably) have do that via .gitmodules + commit entries in trees?

The manifest is tracked in its own manifest repo, so today they fork
that project, with the superproject you'd fork the superproject and modify
the .gitmodules file as needed.

> They run their own Gerrit install with some magic to sync back & forth?
>
> I assume that now the recursive "checkout" relies on all the origin/HEAD
> symbolic refs pointing to "master", but how is this going to deal with
> incorporating a repo whose main branch has a different name?

Change the name? Or pin it via sha1.

> E.g. "trunk" or "blead"? Perhaps some interaction with
> checkout.defaultRemote + submodule.<name>.branch=<NAME> could make "git
> checkout :mainbranch" DWYM.
>
> > * Fetching changes ("repo sync") needs to fetch all repositories, as there
> >   is no central place that tracks what has changed. In a superproject
> >   git fetch can determine which submodules need fetching.  In Androids
> >   case the daily change is only in a few repositories (think 10s), so
> >   migrating to a superproject would save an order of magnitude in fetch
> >   traffic for daily updates of developers.
>
> Interesting that in all this time with the reliance on a central server
> repo wasn't already asking some custom API "what repos changed since xyz
> time" to narrow that down, but hey, .gitmodules + commit entries in
> trees will do it for you.

It's complicated. Shawn really grew to dislike the repo tool and wanted
to have it gone as quickly as possible, so no hacks that extend its life. ;-)
I would prefer to keep that stance as any hack in there would grow into
a nightmare down the road.

> > * Sometimes when the dependencies are not on a clear repository boundary
> >   one would like to have git-bisect available across the different
> >   repositories, which repo cannot provide due to its design.
>
> I assume that you're not upgrading independently to e.g. every single
> linux commit, just stable releases, so does bisecting deal with knowing
> that e.g. a breakage occurred when linux.git was updated from v4.10 to
> v4.12, and then to go within the repo itself and bisect from there, or
> is that done manually?

We currently have no submodule support in bisect, but that is open to
future contribution. But with relations between submodules via the
superproject it is at least possible to bisect precisely, whereas in repo
the projects are unrelated to each other and it becomes a political
issue if A broke B or B broke A.

Thanks for taking time to read and think through the concept here,

Stefan

^ permalink raw reply	[relevance 7%]

* Re: [PATCH v5 9/9] submodule: support reading .gitmodules when it's not in the working tree
  2018-09-27 18:00           ` Stefan Beller
@ 2018-10-01 15:45             ` Antonio Ospite
  2018-10-01 19:42               ` Stefan Beller
  0 siblings, 1 reply; 200+ results
From: Antonio Ospite @ 2018-10-01 15:45 UTC (permalink / raw)
  To: Stefan Beller
  Cc: SZEDER Gábor, Junio C Hamano, git, Brandon Williams,
	Daniel Graña, Jonathan Nieder, Richard Hartmann,
	Ævar Arnfjörð Bjarmason

On Thu, 27 Sep 2018 11:00:52 -0700
Stefan Beller <sbeller@google.com> wrote:

> On Thu, Sep 27, 2018 at 7:44 AM Antonio Ospite <ao2@ao2.it> wrote:
[...]
> > OK, so the plan for v6 is:
> >
> >   - avoid the corruption issues spotted by Gábor by removing the call
> >     to repo_read_gitmodules in builtin/grep.c (this still does not fix
> >     the potential problem with nested submodules).
> >

Actually that is not enough to fix the inconsistent access to the
object store: the functions is_submodule_active() and
repo_submodule_init() too end up calling config_from_gitmodules() and
need protecting as well, so I am going to put them under the git read
lock and leave repo_read_gitmodules() in place for now.

Removing unneeded code can go in a possible stand-alone patch.

> >   - add a new test-tool which better exercises the new
> >     config_from_gitmodules code,
> 
> Sounds good.
> 
> >
> >   - add also a test_expect_failure test to document the use case that
> >     cannot be supported yet: nested submodules without .gitmodules in
> >     their working tree.
> 
> Personally I would want to live in a world where we don't *have* to nor
> *want* to support submodules without .gitmodules in the respective
> superproject.
>

Just to double check: are you referring to *nested* submodules in the
sentence above?

I am asking because the whole point of this patchset is to *enable* the
use of submodules without .gitmodules in the working tree of the
superproject. :)

It's just that current limitations in git do not allow to support this
for *nested* submodules yet.

> We did support some use cases historically that I would make sure to
> continue to support, but I am not sure how much effort we want to spend
> on supporting further use cases of incomplete submodules.
>
> Feel free to do so, as such tests help to document the boundaries.
> 

Let's see how v6 turns out.

Thanks,
   Antonio

-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 7%]

* Re: [BUG] Segfault in "git submodule"
      [irrelevant]     ` <CAGDaZ_oS4bjzd67T5atJrJ6_x2Cfr8JgnoGWePzA53mb2ObdNg@mail.gmail.com>
@ 2018-10-01 19:19       ` Stefan Beller
  2018-10-01 21:31         ` Raymond Jennings
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-01 19:19 UTC (permalink / raw)
  To: shentino; +Cc: Duy Nguyen, Ævar Arnfjörð Bjarmason, git

On Sat, Sep 29, 2018 at 9:43 AM Raymond Jennings <shentino@gmail.com> wrote:
>
> I have a repo, but it appears to be specific to staging area state.
> It only segfaults when I have a certain file deleted.
>
> Where do you want me to upload it?
> On Sat, Sep 29, 2018 at 8:34 AM Duy Nguyen <pclouds@gmail.com> wrote:
> >
> > On Sat, Sep 29, 2018 at 5:31 PM Ævar Arnfjörð Bjarmason
> > <avarab@gmail.com> wrote:
> > > > #1  refs_resolve_ref_unsafe (refs=0x0,
> > > > refname=refname@entry=0x55e863062253 "HEAD",
> > > > resolve_flags=resolve_flags@entry=1, oid=oid@entry=0x7ffdc834b1c0,
> > > > flags=flags@entry=0x7ffdc834b1bc) at refs.c:1493
> >
> > refs is NULL. It looks like somebody fails to get the right submodule
> > ref store (or tries to get it but fails to check if it may return
> > NULL)

This is spot on.

Raymond, are you on Git v2.16.0 by any chance?
(and if now which version are you on).

I suspect 2.16, as that is a version of Git, in which there happens to be
a call into the refs subsystem in submodule--helper.c in line 624.

Is it possible to upgrade Git (to v2.18.0 or later) or cherry-pick
74b6bda32f (submodule: check for NULL return of get_submodule_ref_store(),
2018-03-28) into your copy of Git?

Thanks,
Stefan

^ permalink raw reply	[relevance 8%]

* Re: [PATCH] grep: provide a noop --recursive option
      [irrelevant] ` <20180929145527.23444-1-avarab@gmail.com>
@ 2018-10-01 19:23   ` Stefan Beller
  2018-10-05  8:15     ` Christoph Berg
  2018-10-05  8:19     ` Junio C Hamano
  0 siblings, 2 replies; 200+ results
From: Stefan Beller @ 2018-10-01 19:23 UTC (permalink / raw)
  To: Ævar Arnfjörð Bjarmason; +Cc: git, Junio C Hamano, myon

On Sat, Sep 29, 2018 at 7:55 AM Ævar Arnfjörð Bjarmason
<avarab@gmail.com> wrote:
>
> This --recursive (-r) option does nothing, and is purely here to
> appease people who have "grep -r ..." burned into their muscle memory.
>
> Requested-by: Christoph Berg <myon@debian.org>
> Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
> ---
>
> On Sat, Sep 29, 2018 at 4:10 PM Christoph Berg <myon@debian.org> wrote:
> >
> > I often use "grep -r $pattern" to recursively grep a source tree. If
> > that takes too long, I hit ^C and tag "git" in front of the command
> > line and re-run it. git then complains "error: unknown switch `r'"
> > because "git grep" is naturally recursive.
> >
> > Could we have "git grep -r" accept the argument for compatibility?
> > Other important grep switches like "-i" are compatible, adding -r
> > would improve usability.
>
> I don't have an opinion on this either way, it doesn't scratch my
> itch, but hey, why not. Here's a patch to implement it.
>
>  Documentation/git-grep.txt | 6 ++++++
>  builtin/grep.c             | 3 +++
>  t/t7810-grep.sh            | 8 ++++++++
>  3 files changed, 17 insertions(+)
>
> diff --git a/Documentation/git-grep.txt b/Documentation/git-grep.txt
> index a3049af1a3..a1aea8be4e 100644
> --- a/Documentation/git-grep.txt
> +++ b/Documentation/git-grep.txt
> @@ -290,6 +290,12 @@ providing this option will cause it to die.
>         Do not output matched lines; instead, exit with status 0 when
>         there is a match and with non-zero status when there isn't.
>
> +-r::
> +--recursive::
> +       This option does nothing. git-grep is always recursive. This
> +       noop option is provided for compatibility with the muscle
> +       memory of people used to grep(1).

git-grep is always file/tree recursive, but there is --recurse-submodules
which is off by default. Instead of providing a short alias to a noop,
we could use -r for submodules. (And if you happen to have no
submodules, this is a noop for you)

^ permalink raw reply	[relevance 5%]

* Re: [PATCH v5 9/9] submodule: support reading .gitmodules when it's not in the working tree
  2018-10-01 15:45             ` Antonio Ospite
@ 2018-10-01 19:42               ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-01 19:42 UTC (permalink / raw)
  To: Antonio Ospite
  Cc: SZEDER Gábor, Junio C Hamano, git, Brandon Williams,
	Daniel Graña, Jonathan Nieder, Richard Hartmann,
	Ævar Arnfjörð Bjarmason

On Mon, Oct 1, 2018 at 8:45 AM Antonio Ospite <ao2@ao2.it> wrote:
>
> On Thu, 27 Sep 2018 11:00:52 -0700
> Stefan Beller <sbeller@google.com> wrote:
>
> > On Thu, Sep 27, 2018 at 7:44 AM Antonio Ospite <ao2@ao2.it> wrote:
> [...]
> > > OK, so the plan for v6 is:
> > >
> > >   - avoid the corruption issues spotted by Gábor by removing the call
> > >     to repo_read_gitmodules in builtin/grep.c (this still does not fix
> > >     the potential problem with nested submodules).
> > >
>
> Actually that is not enough to fix the inconsistent access to the
> object store: the functions is_submodule_active() and
> repo_submodule_init() too end up calling config_from_gitmodules() and
> need protecting as well, so I am going to put them under the git read
> lock and leave repo_read_gitmodules() in place for now.
>

>
> I am asking because the whole point of this patchset is to *enable* the
> use of submodules without .gitmodules in the working tree of the
> superproject. :)

I was imprecise and meant to
s/.gitmodules/mechanism to configure the name <-> path mapping/

In this series, the .gitmodules may not be present in the working tree,
but it is still there in the repo. Later we may want to rename that file
or put it into a magic branch, and I'd still find it a good idea.
What I find a bad idea is to have only a gitlink and a repo put
into that path and expect that it magically works as then it is not
a submodule, but some "halfway there thing". We need to have
an explicit "yes this is a submodule" statement, (which currently
comes from the .gitmodules file in the working tree), and I am not
attached to where it comes from, but that it exists.

^ permalink raw reply	[relevance 7%]

* Re: [BUG] Segfault in "git submodule"
  2018-10-01 19:19       ` [BUG] Segfault in "git submodule" Stefan Beller
@ 2018-10-01 21:31         ` Raymond Jennings
  2018-10-02  4:10           ` Raymond Jennings
  0 siblings, 1 reply; 200+ results
From: Raymond Jennings @ 2018-10-01 21:31 UTC (permalink / raw)
  To: sbeller; +Cc: pclouds, avarab, git

Yes, git 2.16.4 to be exact.

I upgraded to 2.19 after ~arch keywording the package on gentoo and
that fixed it.
On Mon, Oct 1, 2018 at 12:19 PM Stefan Beller <sbeller@google.com> wrote:
>
> On Sat, Sep 29, 2018 at 9:43 AM Raymond Jennings <shentino@gmail.com> wrote:
> >
> > I have a repo, but it appears to be specific to staging area state.
> > It only segfaults when I have a certain file deleted.
> >
> > Where do you want me to upload it?
> > On Sat, Sep 29, 2018 at 8:34 AM Duy Nguyen <pclouds@gmail.com> wrote:
> > >
> > > On Sat, Sep 29, 2018 at 5:31 PM Ævar Arnfjörð Bjarmason
> > > <avarab@gmail.com> wrote:
> > > > > #1  refs_resolve_ref_unsafe (refs=0x0,
> > > > > refname=refname@entry=0x55e863062253 "HEAD",
> > > > > resolve_flags=resolve_flags@entry=1, oid=oid@entry=0x7ffdc834b1c0,
> > > > > flags=flags@entry=0x7ffdc834b1bc) at refs.c:1493
> > >
> > > refs is NULL. It looks like somebody fails to get the right submodule
> > > ref store (or tries to get it but fails to check if it may return
> > > NULL)
>
> This is spot on.
>
> Raymond, are you on Git v2.16.0 by any chance?
> (and if now which version are you on).
>
> I suspect 2.16, as that is a version of Git, in which there happens to be
> a call into the refs subsystem in submodule--helper.c in line 624.
>
> Is it possible to upgrade Git (to v2.18.0 or later) or cherry-pick
> 74b6bda32f (submodule: check for NULL return of get_submodule_ref_store(),
> 2018-03-28) into your copy of Git?
>
> Thanks,
> Stefan

^ permalink raw reply	[relevance 5%]

* [PATCH] submodule update --remote: introduce pinning
      [irrelevant] <20180906225423.GB81412@aiede.svl.corp.google.com>
@ 2018-10-02  0:17 ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-02  0:17 UTC (permalink / raw)
  To: jrnieder; +Cc: git, sbeller

gitmodules(5) sayeth:

   submodule.<name>.branch
       A remote branch name for tracking updates in the upstream
       submodule. If the option is not specified, it defaults to master.

This doesn't allow having a "pinned" submodule that should not be updated
from upstream. We should change this to have no default --- if branch is
not specified, don't update that submodule, just like in Gerrit's
corresponding feature[1].

[1] https://gerrit-review.googlesource.com/Documentation/user-submodules.html#_defining_the_submodule_branch


However changing defaults is difficult as it may surprise the user,
Jonathan came up with a 4 step plan:
Step 0: introduce

	# current behavior:
        git submodule update --remote --remote-default-to-master

	# new behavior:
        git submodule update --remote --remote-pinned

and treat plain "git submodule update --remote" as --default-to-master.

Step 1: when neither --default-to-master nor --no-default-to-master
has been passed, warn when encountering a submodule with no branch
and treat it as "master".

Step 2: when neither --default-to-master nor --no-default-to-master
has been passed, error out when encountering a submodule with no
branch.

Step 3: when neither --default-to-master nor --no-default-to-master
has been passed, warn when encountering a submodule with no branch
and treat it as pinned.

Step 4: eliminate the warning.

This implements step 0 and 1.

Signed-off-by: Stefan Beller <sbeller@google.com>
---

Sorry that this took so long, this is a rough patch for steps 0 & 1,
I'd still need to update docs.

Stefan


 Documentation/gitmodules.txt | 11 ++++++-----
 builtin/submodule--helper.c  | 36 +++++++++++++++++++++++++++++-------
 git-submodule.sh             | 34 +++++++++++++++++++++++++---------
 t/t7406-submodule-update.sh  | 29 +++++++++++++++++++++++++++++
 4 files changed, 89 insertions(+), 21 deletions(-)

diff --git a/Documentation/gitmodules.txt b/Documentation/gitmodules.txt
index 4d63def206..fe42dbdb3e 100644
--- a/Documentation/gitmodules.txt
+++ b/Documentation/gitmodules.txt
@@ -50,11 +50,12 @@ submodule.<name>.update::
 
 submodule.<name>.branch::
 	A remote branch name for tracking updates in the upstream submodule.
-	If the option is not specified, it defaults to 'master'.  A special
-	value of `.` is used to indicate that the name of the branch in the
-	submodule should be the same name as the current branch in the
-	current repository.  See the `--remote` documentation in
-	linkgit:git-submodule[1] for details.
+	A special value of `.` is used to indicate that the name of the
+	branch in the submodule should be the same name as the current
+	branch in the current repository.  See the `--remote` documentation
+	in linkgit:git-submodule[1] for details.
+	If not set, the default for `git submodule update --remote` is
+	to update the submodule to the superproject's recorded object id.
 
 submodule.<name>.fetchRecurseSubmodules::
 	This option can be used to control recursive fetching of this
diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 40844870cf..6413f2b410 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -1889,7 +1889,7 @@ static int resolve_relative_path(int argc, const char **argv, const char *prefix
 	return 0;
 }
 
-static const char *remote_submodule_branch(const char *path)
+static const char *remote_submodule_branch(const char *path, int default_pinned)
 {
 	const struct submodule *sub;
 	const char *branch = NULL;
@@ -1904,8 +1904,12 @@ static const char *remote_submodule_branch(const char *path)
 		branch = sub->branch;
 	free(key);
 
-	if (!branch)
-		return "master";
+	if (!branch) {
+		if (default_pinned)
+			return "";
+		else
+			return "master";
+	}
 
 	if (!strcmp(branch, ".")) {
 		const char *refname = resolve_ref_unsafe("HEAD", 0, NULL, NULL);
@@ -1932,12 +1936,30 @@ static int resolve_remote_submodule_branch(int argc, const char **argv,
 {
 	const char *ret;
 	struct strbuf sb = STRBUF_INIT;
-	if (argc != 2)
-		die("submodule--helper remote-branch takes exactly one arguments, got %d", argc);
+	int default_pinned = 0;
+
+	struct option remote_options[] = {
+		OPT_SET_INT(0, "default-master", &default_pinned,
+				N_("unconfigured submodules default to master branch"), 0),
+		OPT_SET_INT(0, "default-pinned", &default_pinned,
+				N_("unconfigured submodules default to superproject object id"), 1),
+		OPT_END()
+	};
+
+	const char *const git_submodule_helper_usage[] = {
+		N_("git submodule--helper remote-branch [--default-{master, pinned}]  -- <path>"),
+		NULL
+	};
+
+	argc = parse_options(argc, argv, prefix, remote_options,
+			     git_submodule_helper_usage, 0);
+
+	if (argc != 1)
+		die("submodule--helper remote-branch takes exactly one path, got %d", argc);
 
-	ret = remote_submodule_branch(argv[1]);
+	ret = remote_submodule_branch(argv[0], default_pinned);
 	if (!ret)
-		die("submodule %s doesn't exist", argv[1]);
+		die("submodule %s doesn't exist", argv[0]);
 
 	printf("%s", ret);
 	strbuf_release(&sb);
diff --git a/git-submodule.sh b/git-submodule.sh
index 1b568e29b9..829b90ea97 100755
--- a/git-submodule.sh
+++ b/git-submodule.sh
@@ -431,6 +431,7 @@ fetch_in_submodule () (
 #
 cmd_update()
 {
+	remote_default_option=
 	# parse $args after "submodule ... update".
 	while test $# -ne 0
 	do
@@ -450,6 +451,12 @@ cmd_update()
 		--remote)
 			remote=1
 			;;
+		--remote-default-to-master)
+			remote_default_option=--default-master
+			;;
+		--remote-pinned)
+			remote_default_option=--default-pinned
+			;;
 		-N|--no-fetch)
 			nofetch=1
 			;;
@@ -555,17 +562,26 @@ cmd_update()
 
 		if test -n "$remote"
 		then
-			branch=$(git submodule--helper remote-branch "$sm_path")
-			if test -z "$nofetch"
+			if test -z $remote_default_option
 			then
-				# Fetch remote before determining tracking $sha1
-				fetch_in_submodule "$sm_path" $depth ||
-				die "$(eval_gettext "Unable to fetch in submodule path '\$sm_path'")"
+				say "--remote needs clarification: --remote-pinned or --remote-default-to-master?"
+				say "assuming --remote-default-to-master for now"
+				remote_default_option=--default-master
+			fi
+			branch=$(git submodule--helper remote-branch ${remote_default_option} -- "$sm_path")
+			if test -n "$branch"
+			then
+				if test -z "$nofetch"
+				then
+					# Fetch remote before determining tracking $sha1
+					fetch_in_submodule "$sm_path" $depth ||
+					die "$(eval_gettext "Unable to fetch in submodule path '\$sm_path'")"
+				fi
+				remote_name=$(sanitize_submodule_env; cd "$sm_path" && get_default_remote)
+				sha1=$(sanitize_submodule_env; cd "$sm_path" &&
+					git rev-parse --verify "${remote_name}/${branch}") ||
+				die "$(eval_gettext "Unable to find current \${remote_name}/\${branch} revision in submodule path '\$sm_path'")"
 			fi
-			remote_name=$(sanitize_submodule_env; cd "$sm_path" && get_default_remote)
-			sha1=$(sanitize_submodule_env; cd "$sm_path" &&
-				git rev-parse --verify "${remote_name}/${branch}") ||
-			die "$(eval_gettext "Unable to find current \${remote_name}/\${branch} revision in submodule path '\$sm_path'")"
 		fi
 
 		if test "$subsha1" != "$sha1" || test -n "$force"
diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh
index 10dc91620a..3019978211 100755
--- a/t/t7406-submodule-update.sh
+++ b/t/t7406-submodule-update.sh
@@ -260,6 +260,35 @@ test_expect_success 'submodule update --remote should fetch upstream changes wit
 	)
 '
 
+test_expect_success 'submodule update --remote should not fetch upstream when no branch is set' '
+	(
+		cd super &&
+		test_might_fail git config --unset -f .gitmodules submodule.submodule.branch &&
+		git add .gitmodules &&
+		git commit --allow-empty -m "submodules: pin in superproject branch" &&
+		git -C ../submodule checkout -f master
+	) &&
+	(
+		cd submodule &&
+		echo line4b >>file &&
+		git add file &&
+		test_tick &&
+		git commit -m "upstream line4b"
+	) &&
+	(
+		cd super &&
+
+		git submodule update --remote-pinned --remote --force submodule &&
+		git status --porcelain --untracked=no --ignore-submodules=none >actual &&
+		test_must_be_empty actual &&
+
+		git submodule update --remote-default-to-master --remote --force submodule &&
+		git -C submodule log -1 --oneline >actual &&
+		git -C ../submodule log -1 --oneline >expect &&
+		test_cmp expect actual
+	)
+'
+
 test_expect_success 'local config should override .gitmodules branch' '
 	(cd submodule &&
 	 git checkout test-branch &&
-- 
2.19.0


^ permalink raw reply	[relevance 27%]

* Re: [BUG] Segfault in "git submodule"
  2018-10-01 21:31         ` Raymond Jennings
@ 2018-10-02  4:10           ` Raymond Jennings
  0 siblings, 0 replies; 200+ results
From: Raymond Jennings @ 2018-10-02  4:10 UTC (permalink / raw)
  To: sbeller; +Cc: pclouds, avarab, git

I instructed downstream to update their repository.
On Mon, Oct 1, 2018 at 2:31 PM Raymond Jennings <shentino@gmail.com> wrote:
>
> Yes, git 2.16.4 to be exact.
>
> I upgraded to 2.19 after ~arch keywording the package on gentoo and
> that fixed it.
> On Mon, Oct 1, 2018 at 12:19 PM Stefan Beller <sbeller@google.com> wrote:
> >
> > On Sat, Sep 29, 2018 at 9:43 AM Raymond Jennings <shentino@gmail.com> wrote:
> > >
> > > I have a repo, but it appears to be specific to staging area state.
> > > It only segfaults when I have a certain file deleted.
> > >
> > > Where do you want me to upload it?
> > > On Sat, Sep 29, 2018 at 8:34 AM Duy Nguyen <pclouds@gmail.com> wrote:
> > > >
> > > > On Sat, Sep 29, 2018 at 5:31 PM Ævar Arnfjörð Bjarmason
> > > > <avarab@gmail.com> wrote:
> > > > > > #1  refs_resolve_ref_unsafe (refs=0x0,
> > > > > > refname=refname@entry=0x55e863062253 "HEAD",
> > > > > > resolve_flags=resolve_flags@entry=1, oid=oid@entry=0x7ffdc834b1c0,
> > > > > > flags=flags@entry=0x7ffdc834b1bc) at refs.c:1493
> > > >
> > > > refs is NULL. It looks like somebody fails to get the right submodule
> > > > ref store (or tries to get it but fails to check if it may return
> > > > NULL)
> >
> > This is spot on.
> >
> > Raymond, are you on Git v2.16.0 by any chance?
> > (and if now which version are you on).
> >
> > I suspect 2.16, as that is a version of Git, in which there happens to be
> > a call into the refs subsystem in submodule--helper.c in line 624.
> >
> > Is it possible to upgrade Git (to v2.18.0 or later) or cherry-pick
> > 74b6bda32f (submodule: check for NULL return of get_submodule_ref_store(),
> > 2018-03-28) into your copy of Git?
> >
> > Thanks,
> > Stefan

^ permalink raw reply	[relevance 2%]

* Re: [PATCH 1/1] protocol: limit max protocol version per service
      [irrelevant]     ` <20181003213349.GA32105@google.com>
@ 2018-10-03 22:47       ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-03 22:47 UTC (permalink / raw)
  To: Josh Steadmon; +Cc: git, Jonathan Nieder, Junio C Hamano

On Wed, Oct 3, 2018 at 2:34 PM Josh Steadmon <steadmon@google.com> wrote:

>
> I believe that git-upload-archive can still speak version 1 without any
> trouble, but it at least doesn't break anything in the test suite to
> limit this to v0 either.

ok, let me just play around with archive to follow along:

Running the excerpt that I found in one of the tests in t5000

    GIT_TRACE_PACKET=1 git archive --remote=. HEAD >b5.tar

(whoooa ... that spews a lot of text into my terminal, which makes
sense as transported tar files unlike packets starting with PACK are
not cut short; so onwards:)

    $ git init test && cd test
    $ GIT_TRACE_PACKET=1 git archive --remote=. HEAD >b5.tar
    ...
    git< \2fatal: no such ref: HEAD
    ...
    fatal: sent error to the client: git upload-archive: archiver died
with error
    remote: git upload-archive: archiver died with error

This sounds similar to a bug that I have on my todo list for
clone recursing into submodules. Maybe we need to talk
about HEAD-less repositories and how to solve handling them
in general. Usually it doesn't happen except for corner cases like
now, so:

    $ git commit --allow-empty -m "commit"
    [master (root-commit) 24d7967] commit
    $ GIT_TRACE_PACKET=1 git archive --remote=. HEAD >b5.tar
15:28:00.762615 pkt-line.c:80           packet:          git> argument HEAD
15:28:00.762704 pkt-line.c:80           packet:          git> 0000
15:28:00.766336 pkt-line.c:80           packet:          git> ACK
15:28:00.766428 pkt-line.c:80           packet:          git> 0000
15:28:00.766483 pkt-line.c:80           packet:          git< ACK
15:28:00.766508 pkt-line.c:80           packet:          git< 0000
15:28:00.767694 pkt-line.c:80           packet:          git< \2
15:28:00.767524 pkt-line.c:80           packet:          git< argument HEAD
15:28:00.767583 pkt-line.c:80           packet:          git< 0000
remote: 15:28:00.767524 pkt-line.c:80           packet:          git<
argument HEAD
remote: 15:28:00.767583 pkt-line.c:80           packet:          git< 0000
15:28:00.768758 pkt-line.c:80           packet:          git> 0000
15:28:00.770475 pkt-line.c:80           packet:          git<
\1pax_global_header
    ... \0\0\0\0\0\0\0\0\0\0\0\0\ ...
 # not too bad but the tar file contains a lot of zeros
    $

Ah I forgot the crucial part, so

    $ GIT_TRACE_PACKET=1 git -c protocol.version=1 archive --remote=.
HEAD >b5.tar
15:33:10.030508 pkt-line.c:80           packet:          git> argument HEAD
...

This pretty much looks like v0 as v1 would send a "version 1", c.f.

    $ GIT_TRACE_PACKET=1 git -c protocol.version=1 fetch .
15:34:26.111013 pkt-line.c:80           packet:  upload-pack> version 1
15:34:26.111140 pkt-line.c:80           packet:        fetch< version 1
....


>
> Is there a method or design for advertising multiple acceptable versions
> from the client?

I think the client can send multiple versions, looking through protocol.c
(and not the documentation as I should for this:)

  /*
   * Determine which protocol version the client has requested.  Since
   * multiple 'version' keys can be sent by the client, indicating that
   * the client is okay to speak any of them, select the greatest version
   * that the client has requested.  This is due to the assumption that
   * the most recent protocol version will be the most state-of-the-art.
   */
    ...
    const char *git_protocol = getenv(GIT_PROTOCOL_ENVIRONMENT);
    string_list_split(&list, git_protocol, ':', -1);
    ...
    for_each_string_list_item(item, &list) {
        ...
        if (skip_prefix(item->string, "version=", &value))
            ...

in determine_protocol_version_server which already had the client
speak to it, so I think at least the server can deal with multiple versions.

But given that we transport the version in env variables, we'd
need to be extra careful if we just did not see the version
in the --remote=. above?

> From my understanding, we can only add a single
> version=X field in the advertisement, but IIUC we can extend this fairly
> easily? Perhaps we can have "version=X" to mean the preferred version,
> and then a repeatable "acceptable_version=Y" field or similar?

Just re-use "version X", separated by colons as above.

> > From a maintenance perspective, do we want to keep
> > this part of the code central, as it ties protocol (as proxied
> > by service name) to the max version number?
> > I would think that we'd rather have the decision local to the
> > code, i.e. builtin/fetch would need to tell protocol.c that it
> > can do (0,1,2) and builtin/push can do (0,1), and then the
> > networking layers of code would figure out by the input
> > from the caller and the input from the user (configured
> > protocol.version) what is the best to go forward from
> > then on.
>
> I like having it centralized, because enforcing this in git_connect()
> and discover_refs() catches all the outgoing version advertisements, but
> there's lots of code paths that lead to those two functions that would
> all have to have the acceptable version numbers plumbed through.

Makes sense.

> I suppose we could also have a registry of services to version numbers,
> but I tend to dislike non-local sources of data. But if the list likes
> that approach better, I'll be happy to implement it.

> > But I guess having the central place here is not to
> > bad either. How will it cope with the desire of protocol v2
> > to have only one end point (c.f. serve.{c,h} via builtin/serve
> > as "git serve") ?
>
> I'm not sure about this. In my series to add a v2 archive command, I
> added support for a new endpoint for proto v2 and I don't recall seeing
> any complaints, but that is still open for review.

Ah I guess new end points would imply that you can speak at least
a given version N.

> I suppose if we are strict about serving from a single endpoint, the
> version registry makes more sense, and individual operations can declare
> acceptable version numbers before calling any network code?

Ah yeah, that makes sense!

Thanks for your explanations and prodding,
Stefan

^ permalink raw reply	[relevance 3%]

* Re: [PATCH] grep: provide a noop --recursive option
  2018-10-01 19:23   ` [PATCH] grep: provide a noop --recursive option Stefan Beller
@ 2018-10-05  8:15     ` Christoph Berg
  2018-10-05  8:19     ` Junio C Hamano
  1 sibling, 0 replies; 200+ results
From: Christoph Berg @ 2018-10-05  8:15 UTC (permalink / raw)
  To: Stefan Beller; +Cc: Ævar Arnfjörð Bjarmason, git, Junio C Hamano

Re: Stefan Beller 2018-10-01 <CAGZ79kbw96x2Dow7d-sUfOHXiVN8j9KgqEObo+TrVd5zWKbaEA@mail.gmail.com>
> git-grep is always file/tree recursive, but there is --recurse-submodules
> which is off by default. Instead of providing a short alias to a noop,
> we could use -r for submodules. (And if you happen to have no
> submodules, this is a noop for you)

That would be fine for me.

Thanks,
Christoph

^ permalink raw reply	[relevance 2%]

* Re: [PATCH] grep: provide a noop --recursive option
  2018-10-01 19:23   ` [PATCH] grep: provide a noop --recursive option Stefan Beller
  2018-10-05  8:15     ` Christoph Berg
@ 2018-10-05  8:19     ` Junio C Hamano
  2018-10-05 13:05       ` Mischa POSLAWSKY
  1 sibling, 1 reply; 200+ results
From: Junio C Hamano @ 2018-10-05  8:19 UTC (permalink / raw)
  To: Stefan Beller; +Cc: Ævar Arnfjörð Bjarmason, git, myon

Stefan Beller <sbeller@google.com> writes:

> git-grep is always file/tree recursive, but there is --recurse-submodules
> which is off by default. Instead of providing a short alias to a noop,
> we could use -r for submodules. (And if you happen to have no
> submodules, this is a noop for you)

I am not sure if it is an overall win for those who do have and use
submodules to easily be able to go recursive with a short-and-sweet
'r', or even they want to work inside one project at a time most of
the time.  If the latter, then using 'r' for recurse-submodules is
going to be a mistake (besides, other commands that have 'recursive'
typically use 'r' for its shorthand,and 'r' does not stand for
'recurse-submodules' for them).

^ permalink raw reply	[relevance 5%]

* Re: [PATCH] grep: provide a noop --recursive option
  2018-10-05  8:19     ` Junio C Hamano
@ 2018-10-05 13:05       ` Mischa POSLAWSKY
  2018-10-05 19:17         ` Stefan Beller
  0 siblings, 1 reply; 200+ results
From: Mischa POSLAWSKY @ 2018-10-05 13:05 UTC (permalink / raw)
  To: Junio C Hamano
  Cc: Stefan Beller, Ævar Arnfjörð Bjarmason, git, myon

Junio C Hamano wrote 2018-10-05 1:19 (-0700):
> Stefan Beller <sbeller@google.com> writes:
> 
> > git-grep is always file/tree recursive, but there is --recurse-submodules
> > which is off by default. Instead of providing a short alias to a noop,
> > we could use -r for submodules. (And if you happen to have no
> > submodules, this is a noop for you)
> 
> I am not sure if it is an overall win for those who do have and use
> submodules to easily be able to go recursive with a short-and-sweet
> 'r', or even they want to work inside one project at a time most of
> the time.  If the latter, then using 'r' for recurse-submodules is
> going to be a mistake (besides, other commands that have 'recursive'
> typically use 'r' for its shorthand,and 'r' does not stand for
> 'recurse-submodules' for them).

Personally I would welcome a shorthand for --recurse-submodules,
especially if --r^I no longer completes to this.

It is also closer to the behaviour provided by grep -r as that recurses
into submodules as well.

Regards,
-- 
Mischa

^ permalink raw reply	[relevance 2%]

* [PATCH v6 00/10] Make submodules work if .gitmodules is not checked out
@ 2018-10-05 13:05 Antonio Ospite
      [irrelevant] ` <20181005130601.15879-9-ao2@ao2.it>
      [irrelevant] ` <20181005130601.15879-10-ao2@ao2.it>
  0 siblings, 2 replies; 200+ results
From: Antonio Ospite @ 2018-10-05 13:05 UTC (permalink / raw)
  To: gitster
  Cc: git, Brandon Williams, Jonathan Nieder, Stefan Beller, Jeff King,
	SZEDER Gábor, Antonio Ospite

Hi,

this series teaches git to try and read the .gitmodules file from the
index (:.gitmodules) or from the current branch (HEAD:.gitmodules) when
the file is not readily available in the working tree.

This can be used, together with sparse checkouts, to enable submodule
usage with programs like vcsh[1] which manage multiple repositories with
their working trees sharing the same path.

[1] https://github.com/RichiH/vcsh


Thanks to SZEDER Gábor we found out that the changes in patch 9 could
allow to access the object store in an inconsistent way when using
multi-threading in "git grep --recurse-submodules", this has been dealt
with in this revision.

BTW, with Stefan Beller we also identified some unneeded code which
could have been removed to alleviate the issue, but that would not have
solved it completely; so, I am not removing the unnecessary call to
repo_read_gitmodules() builtin/grep.c in this series, possibly this can
become a stand-alone change.

The problems from above also made me investigate what implications the
current use of a global object store had on my new addition, and now
I know that there is one case which I cannot support just yet: nested
submodules without .gitmodules in their working tree.

This case has been documented with a warning and test_expect_failure
items in tests, and hitting the unsupported case does not alter the
current behavior of git.

Apart form patch 9 and 10 there are no major changes to the previous
iteration.

IMHO we are in a place where the problem has been analyzed with enough
depth, the limitations have been identified and dealt with in a way that
should not affect current users nor diminish the usefulness of the new
code.

v5 of the series is here:
https://public-inbox.org/git/20180917140940.3839-1-ao2@ao2.it/

v4 of the series is here:
https://public-inbox.org/git/20180824132951.8000-1-ao2@ao2.it/

v3 of the series is here:
https://public-inbox.org/git/20180814110525.17801-1-ao2@ao2.it/

v2 of the series is here:
https://public-inbox.org/git/20180802134634.10300-1-ao2@ao2.it/

v1 of the series, with some background info, is here:
https://public-inbox.org/git/20180514105823.8378-1-ao2@ao2.it/

Changes since v5:

  * print_config_from_gitmodules() in patch 1 now accepts a struct
    repository argument.

  * submodule--helper config in patch 5 has been adjusted to use the new
    signature of print_config_from_gitmodules().

  * In patch 9 the grep read lock in builtin/grep.c now covers all code
    paths involving config_from_gitmodules().
    
    FTR git-grep is the only place where config_from_gitmodules() is
    called from multi-threaded code.

  * Patch 9 also documents the rare case that cannot be supported just
    yet, and adds a warning to the user.

  * In patch 9, config_from_gitmodules() now does not read any config
    when the config source is not specified.(I added a catch-all "else"
    block) This match more closely the behavior of the old code using
    git_config_from_file.

  * Added a new test tool in patch 10 to exercise config_read_config()
    in a more direct way, passing an arbitrary repository.
    
    Admittedly, patch 10 performs a similar test to the one added to
    t7814 in patch 9, so I'd be OK with dropping patch 10 if it is too
    specific.

Thank you,
   Antonio

Antonio Ospite (10):
  submodule: add a print_config_from_gitmodules() helper
  submodule: factor out a config_set_in_gitmodules_file_gently function
  t7411: merge tests 5 and 6
  t7411: be nicer to future tests and really clean things up
  submodule--helper: add a new 'config' subcommand
  submodule: use the 'submodule--helper config' command
  t7506: clean up .gitmodules properly before setting up new scenario
  submodule: add a helper to check if it is safe to write to .gitmodules
  submodule: support reading .gitmodules when it's not in the working
    tree
  t/helper: add test-submodule-nested-repo-config

 Makefile                                     |   1 +
 builtin/grep.c                               |  17 ++-
 builtin/submodule--helper.c                  |  40 ++++++
 cache.h                                      |   2 +
 git-submodule.sh                             |  13 +-
 submodule-config.c                           |  68 ++++++++-
 submodule-config.h                           |   2 +
 submodule.c                                  |  28 +++-
 submodule.h                                  |   1 +
 t/helper/test-submodule-nested-repo-config.c |  30 ++++
 t/helper/test-tool.c                         |   1 +
 t/helper/test-tool.h                         |   1 +
 t/t7411-submodule-config.sh                  | 141 +++++++++++++++++--
 t/t7416-submodule-sparse-gitmodules.sh       |  78 ++++++++++
 t/t7506-status-submodule.sh                  |   3 +-
 t/t7814-grep-recurse-submodules.sh           |  16 +++
 16 files changed, 410 insertions(+), 32 deletions(-)
 create mode 100644 t/helper/test-submodule-nested-repo-config.c
 create mode 100755 t/t7416-submodule-sparse-gitmodules.sh

-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 6%]

* Re: [PATCH] grep: provide a noop --recursive option
  2018-10-05 13:05       ` Mischa POSLAWSKY
@ 2018-10-05 19:17         ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-05 19:17 UTC (permalink / raw)
  To: git; +Cc: Junio C Hamano, Ævar Arnfjörð Bjarmason, git, myon

On Fri, Oct 5, 2018 at 6:05 AM Mischa POSLAWSKY <git@shiar.nl> wrote:
>
> Junio C Hamano wrote 2018-10-05 1:19 (-0700):
> > Stefan Beller <sbeller@google.com> writes:
> >
> > > git-grep is always file/tree recursive, but there is --recurse-submodules
> > > which is off by default. Instead of providing a short alias to a noop,
> > > we could use -r for submodules. (And if you happen to have no
> > > submodules, this is a noop for you)
> >
> > I am not sure if it is an overall win for those who do have and use
> > submodules to easily be able to go recursive with a short-and-sweet
> > 'r', or even they want to work inside one project at a time most of
> > the time.  If the latter, then using 'r' for recurse-submodules is
> > going to be a mistake (besides, other commands that have 'recursive'
> > typically use 'r' for its shorthand,and 'r' does not stand for
> > 'recurse-submodules' for them).
>
> Personally I would welcome a shorthand for --recurse-submodules,
> especially if --r^I no longer completes to this.

The new switch differs by one dash, so I'd think the double dashed
version would still autocomplete.

Unrelated to this, but more to submodules:
There is submodule.recurse which you may want to set.
Would you be interested in a more specific config option there?
(i.e. grep.recurseSubmodules to only apply to grep recursing into
submodules, just like fetch.recurseSubmodules only applies to fetch)

> It is also closer to the behaviour provided by grep -r as that recurses
> into submodules as well.

That sort of makes for the grep case, but not for other commands.
See the related discussion at
https://public-inbox.org/git/20180907064026.GB172953@aiede.svl.corp.google.com/

^ permalink raw reply	[relevance 5%]

* [PATCH] builtin/grep.c: remote superflous submodule code
@ 2018-10-05 22:45 Stefan Beller
  2018-10-06  8:59 ` Antonio Ospite
                   ` (2 more replies)
  0 siblings, 3 replies; 200+ results
From: Stefan Beller @ 2018-10-05 22:45 UTC (permalink / raw)
  To: git; +Cc: ao2, Stefan Beller

In f9ee2fcdfa (grep: recurse in-process using 'struct repository',
2017-08-02), we introduced a call to repo_read_gitmodules in builtin/grep
to simplify the submodule handling.

After ff6f1f564c4 (submodule-config: lazy-load a repository's .gitmodules
file, 2017-08-03) this is no longer necessary, but that commit did not
cleanup the whole tree, but just show cased the new way how to deal with
submodules in ls-files.

Cleanup the only remaining caller to repo_read_gitmodules outside of
submodule.c

Signed-off-by: Stefan Beller <sbeller@google.com>
---

Antonio Ospite writes:
> BTW, with Stefan Beller we also identified some unneeded code which
> could have been removed to alleviate the issue, but that would not have
> solved it completely; so, I am not removing the unnecessary call to
> repo_read_gitmodules() builtin/grep.c in this series, possibly this can
> become a stand-alone change.

Here is the stand-alone change.

The patch [1] contains the lines as deleted below in the context lines
but they would not conflict as there is one empty line between the changes
in this patch in [1].

[1] https://public-inbox.org/git/20181005130601.15879-10-ao2@ao2.it/


 builtin/grep.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/builtin/grep.c b/builtin/grep.c
index 601f801158..a6272b9c2f 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -427,8 +427,6 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 	if (repo_submodule_init(&submodule, superproject, path))
 		return 0;
 
-	repo_read_gitmodules(&submodule);
-
 	/*
 	 * NEEDSWORK: This adds the submodule's object directory to the list of
 	 * alternates for the single in-memory object store.  This has some bad
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* Re: [PATCH v6 08/10] submodule: add a helper to check if it is safe to write to .gitmodules
      [irrelevant] ` <20181005130601.15879-9-ao2@ao2.it>
@ 2018-10-05 23:50   ` Stefan Beller
  2018-10-06  9:19     ` Antonio Ospite
  2018-10-06 23:44     ` Junio C Hamano
  0 siblings, 2 replies; 200+ results
From: Stefan Beller @ 2018-10-05 23:50 UTC (permalink / raw)
  To: Antonio Ospite
  Cc: Junio C Hamano, git, Brandon Williams, Jonathan Nieder,
	Jeff King, SZEDER Gábor

>  static int module_config(int argc, const char **argv, const char *prefix)
>  {
> +       enum {
> +               CHECK_WRITEABLE = 1
> +       } command = 0;

Can we have the default named? Then we would only use states
from within the enum?

^ permalink raw reply	[relevance 5%]

* Re: [PATCH] builtin/grep.c: remote superflous submodule code
  2018-10-05 22:45 [PATCH] builtin/grep.c: remote superflous submodule code Stefan Beller
@ 2018-10-06  8:59 ` Antonio Ospite
  2018-10-07  0:29 ` Junio C Hamano
  2018-10-07  0:33 ` Junio C Hamano
  2 siblings, 0 replies; 200+ results
From: Antonio Ospite @ 2018-10-06  8:59 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

On Fri,  5 Oct 2018 15:45:57 -0700
Stefan Beller <sbeller@google.com> wrote:

> In f9ee2fcdfa (grep: recurse in-process using 'struct repository',
> 2017-08-02), we introduced a call to repo_read_gitmodules in builtin/grep
> to simplify the submodule handling.
> 
> After ff6f1f564c4 (submodule-config: lazy-load a repository's .gitmodules
> file, 2017-08-03) this is no longer necessary, but that commit did not
> cleanup the whole tree, but just show cased the new way how to deal with
> submodules in ls-files.
> 
> Cleanup the only remaining caller to repo_read_gitmodules outside of
> submodule.c
> 
> Signed-off-by: Stefan Beller <sbeller@google.com>

Not sure if I am entitled to formally ack it, but:

Acked-by: Antonio Ospite <ao2@ao2.it>

> ---
> 
> Antonio Ospite writes:
> > BTW, with Stefan Beller we also identified some unneeded code which
> > could have been removed to alleviate the issue, but that would not have
> > solved it completely; so, I am not removing the unnecessary call to
> > repo_read_gitmodules() builtin/grep.c in this series, possibly this can
> > become a stand-alone change.
> 
> Here is the stand-alone change.
>

Thank you for sending it.

Ciao,
   Antonio

-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 5%]

* Re: [PATCH v6 08/10] submodule: add a helper to check if it is safe to write to .gitmodules
  2018-10-05 23:50   ` [PATCH v6 08/10] submodule: add a helper to check if it is safe to write to .gitmodules Stefan Beller
@ 2018-10-06  9:19     ` Antonio Ospite
  2018-10-06 23:44     ` Junio C Hamano
  1 sibling, 0 replies; 200+ results
From: Antonio Ospite @ 2018-10-06  9:19 UTC (permalink / raw)
  To: Stefan Beller
  Cc: Junio C Hamano, git, Brandon Williams, Jonathan Nieder,
	Jeff King, SZEDER Gábor

On Fri, 5 Oct 2018 16:50:10 -0700
Stefan Beller <sbeller@google.com> wrote:

> >  static int module_config(int argc, const char **argv, const char *prefix)
> >  {
> > +       enum {
> > +               CHECK_WRITEABLE = 1
> > +       } command = 0;
> 
> Can we have the default named? Then we would only use states
> from within the enum?

The default would mean:

  "no command passed as a CLI *option*"

I copied this style from builtin/bisect--helper.c::cmd_bisect__helper()
and it's also used in builtin/rebase--helper.c

I can add a name for the default enum value but I am not sure what it
should be: NO_COMMAND_OPTION, COMMAND_DEFAULT, MODE_DEFAULT?

Ciao,
   Antonio

-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 4%]

* Re: [PATCH v6 08/10] submodule: add a helper to check if it is safe to write to .gitmodules
  2018-10-05 23:50   ` [PATCH v6 08/10] submodule: add a helper to check if it is safe to write to .gitmodules Stefan Beller
  2018-10-06  9:19     ` Antonio Ospite
@ 2018-10-06 23:44     ` Junio C Hamano
  2018-10-08 12:37       ` Antonio Ospite
  1 sibling, 1 reply; 200+ results
From: Junio C Hamano @ 2018-10-06 23:44 UTC (permalink / raw)
  To: Stefan Beller
  Cc: Antonio Ospite, git, Brandon Williams, Jonathan Nieder,
	Jeff King, SZEDER Gábor

Stefan Beller <sbeller@google.com> writes:

>>  static int module_config(int argc, const char **argv, const char *prefix)
>>  {
>> +       enum {
>> +               CHECK_WRITEABLE = 1
>> +       } command = 0;
>
> Can we have the default named? Then we would only use states
> from within the enum?

Why?  Do we use a half-intelligent "switch () { case ...: ... }"
checker that would otherwise complain if we handled "case 0" in such
a switch statement, or something like that?

Are we going to gain a lot more enum members, by the way?  At this
point, this looks more like a

	unsigned check_writable = 0; /* default is not to check */


to me.

^ permalink raw reply	[relevance 5%]

* Re: [PATCH] builtin/grep.c: remote superflous submodule code
  2018-10-05 22:45 [PATCH] builtin/grep.c: remote superflous submodule code Stefan Beller
  2018-10-06  8:59 ` Antonio Ospite
@ 2018-10-07  0:29 ` Junio C Hamano
  2018-10-07  0:33 ` Junio C Hamano
  2 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-10-07  0:29 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git, ao2

Stefan Beller <sbeller@google.com> writes:

> In f9ee2fcdfa (grep: recurse in-process using 'struct repository',
> 2017-08-02), we introduced a call to repo_read_gitmodules in builtin/grep
> to simplify the submodule handling.
>
> After ff6f1f564c4 (submodule-config: lazy-load a repository's .gitmodules
> file, 2017-08-03) this is no longer necessary, but that commit did not
> cleanup the whole tree, but just show cased the new way how to deal with
> submodules in ls-files.
>
> Cleanup the only remaining caller to repo_read_gitmodules outside of
> submodule.c

Well, submodule-config.c has its implementation and another caller,
which technically is outside submodule.c ;-)  repo_read_gitmodules
has two more callers in unpack-trees.c these days, so perhaps we can
do without this last paragraph.


^ permalink raw reply	[relevance 8%]

* Re: [PATCH] builtin/grep.c: remote superflous submodule code
  2018-10-05 22:45 [PATCH] builtin/grep.c: remote superflous submodule code Stefan Beller
  2018-10-06  8:59 ` Antonio Ospite
  2018-10-07  0:29 ` Junio C Hamano
@ 2018-10-07  0:33 ` Junio C Hamano
  2018-10-09  0:14   ` Stefan Beller
  2 siblings, 1 reply; 200+ results
From: Junio C Hamano @ 2018-10-07  0:33 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git, ao2

Stefan Beller <sbeller@google.com> writes:

> After ff6f1f564c4 (submodule-config: lazy-load a repository's .gitmodules
> file, 2017-08-03) this is no longer necessary, but that commit did not
> cleanup the whole tree, but just show cased the new way how to deal with
> submodules in ls-files.

The log message of the above one singles out "grep" as a special
case and explalins why it did not touch, by the way.  You probably
need to explain the reason why "this is no longer necessary" a bit
better than the above---as it stands, it is "ff6f1f564c4 said it
still is necessary, I say it is not".

^ permalink raw reply	[relevance 5%]

* Re: [PATCH v6 08/10] submodule: add a helper to check if it is safe to write to .gitmodules
  2018-10-06 23:44     ` Junio C Hamano
@ 2018-10-08 12:37       ` Antonio Ospite
  0 siblings, 0 replies; 200+ results
From: Antonio Ospite @ 2018-10-08 12:37 UTC (permalink / raw)
  To: Junio C Hamano
  Cc: Stefan Beller, git, Brandon Williams, Jonathan Nieder, Jeff King,
	SZEDER Gábor

On Sun, 07 Oct 2018 08:44:20 +0900
Junio C Hamano <gitster@pobox.com> wrote:

> Stefan Beller <sbeller@google.com> writes:
> 
> >>  static int module_config(int argc, const char **argv, const char *prefix)
> >>  {
> >> +       enum {
> >> +               CHECK_WRITEABLE = 1
> >> +       } command = 0;
> >
> > Can we have the default named? Then we would only use states
> > from within the enum?
> 
> Why?  Do we use a half-intelligent "switch () { case ...: ... }"
> checker that would otherwise complain if we handled "case 0" in such
> a switch statement, or something like that?
> 
> Are we going to gain a lot more enum members, by the way?  At this
> point, this looks more like a
> 
> 	unsigned check_writable = 0; /* default is not to check */
> 
> to me.

Hi,

the CHECK_WRITEABLE operation is alternative to the get/set ones, not
an addition, so I can see the rationale behind Stefan's suggestion:
either have named enums members for all command "modes" or for none of
them; however other users of enum+OPT_CMDMODE seems to think like the
enum is for commands passed as *options* and the unnamed default is for
actions derived from *arguments*. I don't have a strong opinion on this
matter, tho, so just tell me what you prefer and I'll do it for v7.

Using an enum was to have a more explicit syntax in case other commands
were going to be added in the future (I imagine "--stage" or
"--list-all" as possible additions), and does not affect the generated
code, so I though it was worth it.

Anyways, these are really details, let's concentrate on patches 9 and
10 which deserve much more attention. :)

Thanks you,
   Antonio
-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 2%]

* Re: [PATCH v6 09/10] submodule: support reading .gitmodules when it's not in the working tree
      [irrelevant] ` <20181005130601.15879-10-ao2@ao2.it>
@ 2018-10-08 22:19   ` Stefan Beller
  2018-10-10 18:56     ` Antonio Ospite
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-08 22:19 UTC (permalink / raw)
  To: Antonio Ospite
  Cc: Junio C Hamano, git, Brandon Williams, Jonathan Nieder,
	Jeff King, SZEDER Gábor

> +test_expect_success 'not writing gitmodules config file when it is not checked out' '
> +        test_must_fail git -C super submodule--helper config submodule.submodule.url newurl

This only checks the exit code, do we also want to check for

    test_path_is_missing .gitmodules ?

> +test_expect_success 'initialising submodule when the gitmodules config is not checked out' '
> +       git -C super submodule init
> +'
> +
> +test_expect_success 'showing submodule summary when the gitmodules config is not checked out' '
> +       git -C super submodule summary
> +'

Same for these, is the exit code enough, or do we want to look at
specific things?

> +
> +test_expect_success 'updating submodule when the gitmodules config is not checked out' '
> +       (cd submodule &&
> +               echo file2 >file2 &&
> +               git add file2 &&
> +               git commit -m "add file2 to submodule"
> +       ) &&
> +       git -C super submodule update

git status would want to be clean afterwards?

^ permalink raw reply	[relevance 5%]

* Re: [PATCH 13/14] submodule: make zero-oid comparison hash function agnostic
      [irrelevant] ` <20181008215701.779099-14-sandals@crustytoothpaste.net>
@ 2018-10-08 23:10   ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-08 23:10 UTC (permalink / raw)
  To: brian m. carlson; +Cc: git, Jeff King, Eric Sunshine, Duy Nguyen

On Mon, Oct 8, 2018 at 2:57 PM brian m. carlson
<sandals@crustytoothpaste.net> wrote:
>
> With SHA-256, the length of the all-zeros object ID is longer.  Add a
> function to git-submodule.sh to check if a full hex object ID is the
> all-zeros value, and use it to check the output we're parsing from git
> diff-files or diff-index.
>
> Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net>
> ---

Nice!

^ permalink raw reply	[relevance 5%]

* Re: [PATCH] builtin/grep.c: remote superflous submodule code
  2018-10-07  0:33 ` Junio C Hamano
@ 2018-10-09  0:14   ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-09  0:14 UTC (permalink / raw)
  To: Junio C Hamano; +Cc: git, Antonio Ospite

> Well, submodule-config.c has its implementation and another caller,
> which technically is outside submodule.c ;-)

i.e. there is a typo in my commit message.
I meant to say submodule-config.c

>  repo_read_gitmodules
> has two more callers in unpack-trees.c these days, so perhaps we can
> do without this last paragraph.

Gah, looking at that code, did we have any reason to rush that series?
c.f. https://public-inbox.org/git/20170811171811.GC1472@book.hvoigt.net/


On Sat, Oct 6, 2018 at 5:33 PM Junio C Hamano <gitster@pobox.com> wrote:
>
> Stefan Beller <sbeller@google.com> writes:
>
> > After ff6f1f564c4 (submodule-config: lazy-load a repository's .gitmodules
> > file, 2017-08-03) this is no longer necessary, but that commit did not
> > cleanup the whole tree, but just show cased the new way how to deal with
> > submodules in ls-files.
>
> The log message of the above one singles out "grep" as a special
> case and explalins why it did not touch, by the way.  You probably
> need to explain the reason why "this is no longer necessary" a bit
> better than the above---as it stands, it is "ff6f1f564c4 said it
> still is necessary, I say it is not".

That is true.

For grep, the reason seems to be, that we check is_submodule_active
based off the index, i.e. using
   module = submodule_from_path(repo, &null_oid, path);
as the deciding factor, which falls in line with lazyloading.

However the use of the specialized gitmodules_config_oid
in grep is also guarded by the same commit ff6f1f564c4.

Going back to the use case of unpack-trees.c,
I think that we need to keep it there as alternatives
seem to be more complicated.

So I guess I'll just resend with a better commit message.

Thanks,
Stefan

^ permalink raw reply	[relevance 7%]

* Re: [PATCH 2/3] midx: close multi-pack-index on repack
      [irrelevant]     ` <8b5dbe3d-b382-bf48-b524-d9e8a074ac4d@gmail.com>
@ 2018-10-09 18:15       ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-09 18:15 UTC (permalink / raw)
  To: Derrick Stolee; +Cc: Junio C Hamano, gitgitgadget, git, Derrick Stolee

On Tue, Oct 9, 2018 at 7:11 AM Derrick Stolee <stolee@gmail.com> wrote:
>
> CC Stefan: Is there a plan to make get_object_directory() take a
> repository parameter?

Not an immediate plan. Regarding the large refactorings I am mostly
waiting for nd/the-index to stabilize (which may be stable enough by
now) before proceeding.  Specifically 2abf350385 (revision.c: remove
implicit dependency on the_index, 2018-09-21) is a missing piece that
I want to build on.

My next step is to look into making use of the refactorings that we did
over the last year, so I'd rather look into more submodule code again.

To come back to your question:
    git grep get_object_directory |wc -l
    30
    git grep "objects->objectdir" |wc -l
    11
2 out of the 11 matches are from the implementation of get_object_directory

So either we'd actually teach get_object_directory to take a repository
and inspect the 11 occurrences of direct access to objects->objectdir
or we'd view get_object_directory() as a fancy wrapper, that we might want
to drop eventually.

^ permalink raw reply	[relevance 4%]

* [PATCH] builtin/grep.c: remote superflous submodule code
@ 2018-10-09 18:35 Stefan Beller
      [irrelevant] ` <20181010001037.74709-1-jonathantanmy@google.com>
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-09 18:35 UTC (permalink / raw)
  To: gitster, ao2; +Cc: git, Stefan Beller

In f9ee2fcdfa (grep: recurse in-process using 'struct repository',
2017-08-02), we introduced a call to repo_read_gitmodules in builtin/grep
to simplify the submodule handling.

After ff6f1f564c4 (submodule-config: lazy-load a repository's .gitmodules
file, 2017-08-03) this is no longer necessary, but that commit did not
cleanup the whole tree, but just show cased the new way how to deal with
submodules in ls-files.

It claimed that grep would still need some explicit handling, but that is
not the call to repo_read_gitmodules (applying this patch on top of
ff6f1f564c4 still keep the test suite happy, specifically
t7814-grep-recurse-submodules, which contains a test
"grep history with moved submoules")

The special handling is the call to gitmodules_config_oid which was added
already in 74ed43711f (grep: enable recurse-submodules to work on
<tree> objects, 2016-12-16), but then was still named
gitmodules_config_sha1.

Signed-off-by: Stefan Beller <sbeller@google.com>
Acked-by: Antonio Ospite <ao2@ao2.it>
---

This is a resend of origin/sb/grep-submodule-cleanup,
and I think picking ff6f1f564c4 as the base for the series would
also be appropriate.

Stefan


 builtin/grep.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/builtin/grep.c b/builtin/grep.c
index 601f801158..a6272b9c2f 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -427,8 +427,6 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 	if (repo_submodule_init(&submodule, superproject, path))
 		return 0;
 
-	repo_read_gitmodules(&submodule);
-
 	/*
 	 * NEEDSWORK: This adds the submodule's object directory to the list of
 	 * alternates for the single in-memory object store.  This has some bad
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* Re: What's cooking in git.git (Oct 2018, #01; Wed, 10)
      [irrelevant] <xmqq8t36mk4t.fsf@gitster-ct.c.googlers.com>
@ 2018-10-10 18:55 ` Stefan Beller
  2018-10-11  2:00   ` Junio C Hamano
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-10 18:55 UTC (permalink / raw)
  To: Junio C Hamano; +Cc: git

> * pw/diff-color-moved-ws-fix (2018-10-04) 5 commits
>  - diff --color-moved: fix a memory leak
>  - diff --color-moved-ws: fix another memory leak
>  - diff --color-moved-ws: fix a memory leak
>  - diff --color-moved-ws: fix out of bounds string access
>  - diff --color-moved-ws: fix double free crash
>
>  Various fixes to "diff --color-moved-ws".
>
>  What's the status of this topic?

Per [1] ("The whole series is
Reviewed-by: Stefan Beller <sbeller@google.com>"),
I would suggest merging to 'next'.

[1] https://public-inbox.org/git/CAGZ79kbamUK=d+-ejy9vopDiVZF7OVOngz1Zx9y04VR3HnmoXg@mail.gmail.com/

> * sb/strbuf-h-update (2018-09-29) 1 commit
>  - strbuf.h: format according to coding guidelines
>
>  Code clean-up to serve as a BCP example.
>
>  What's the status of this one after the discussion thread stopped here?
>  cf. <CAGZ79kbV6QjsFKcD2uG_P9j1AvzSNQSi-_jXGQ9w0YU9fjhEGg@mail.gmail.com>

I was waiting for more discussion and stricter guidelines,
which never happened.

The only controversial issue about this patch is whether we want
to name all parameters or only when we feel like it.

Peff did not seem to care about this particular detail
https://public-inbox.org/git/20180929073827.GD2174@sigill.intra.peff.net/

You suggested to embrace it further and use caps for the parameter
names in the docs comment.
https://public-inbox.org/git/xmqq8t3lb8uu.fsf@gitster-ct.c.googlers.com/

The patch as-is just adds names everywhere.
I'd be happy to resend with either
(a) not enforcing names everywhere, but only as needed or
(b) having names everywhere, capitalizing them NAMES in
    the doc comment.

I am tempted to ask for
(c) take as-is, defer the rewording of doc strings for a follow up patch.

> * sb/grep-submodule-cleanup (2018-10-10) 1 commit
>  - builtin/grep.c: remove superfluous submodule code
>
>  Code clean-up.
>
>  cf. <20181010001037.74709-1-jonathantanmy@google.com>

Will resend.


> * bw/submodule-name-to-dir (2018-08-10) 2 commits
>  - submodule: munge paths to submodule git directories
>  - submodule: create helper to build paths to submodule gitdirs
>
>  In modern repository layout, the real body of a cloned submodule
>  repository is held in .git/modules/ of the superproject, indexed by
>  the submodule name.  URLencode the submodule name before computing
>  the name of the directory to make sure they form a flat namespace.
>
>  Kicked back to 'pu', expecting further work on the topic.
>  cf. <CAGZ79kYnbjaPoWdda0SM_-_X77mVyYC7JO61OV8nm2yj3Q1OvQ@mail.gmail.com>

Thanks.

>
> * sb/submodule-move-head-with-corruption (2018-08-28) 2 commits
>  - submodule.c: warn about missing submodule git directories
>  - t2013: add test for missing but active submodule
>
>  Will discard and wait for a cleaned-up rewrite.
>  cf. <20180907195349.GA103699@aiede.svl.corp.google.com>

Yeah I think discarding this is the right move.

> * sb/submodule-recursive-fetch-gets-the-tip (2018-09-12) 9 commits
>  - builtin/fetch: check for submodule updates for non branch fetches
>  - fetch: retry fetching submodules if sha1 were not fetched
>  - submodule: fetch in submodules git directory instead of in worktree
>  - submodule.c: do not copy around submodule list
>  - submodule: move global changed_submodule_names into fetch submodule struct
>  - submodule.c: sort changed_submodule_names before searching it
>  - submodule.c: fix indentation
>  - sha1-array: provide oid_array_filter
>  - string-list: add string_list_{pop, last} functions
>
>  "git fetch --recurse-submodules" may not fetch the necessary commit
>  that is bound to the superproject, which is getting corrected.
>
>  Expecting a reroll.
>  cf. <b16af8c0-0435-0de4-ed6c-53888d6190af@ramsayjones.plus.com>

is fixed in
https://public-inbox.org/git/20180917213559.126404-7-sbeller@google.com/

>  cf. <CAGZ79kbavjVbTqXsmtjW6=jhkq47_p3mc6=92xOp4_mfhqDtvw@mail.gmail.com>

That is fixed locally

>  cf. <CAGZ79kZKKf9N8yx9EuCRZhrZS_mA2218PouEG7aHDhK2bJGEdA@mail.gmail.com>

That has been addressed via
https://public-inbox.org/git/20180925194755.105578-1-sbeller@google.com/

Will resend after a local review.

> * pk/rebase-in-c-6-final (2018-10-09) 1 commit
>  - rebase: default to using the builtin rebase
>  (this branch uses ag/rebase-i-in-c, js/rebase-in-c-5.5-work-with-rebase-i-in-c, pk/rebase-in-c, pk/rebase-in-c-2-basic, pk/rebase-in-c-3-acts, pk/rebase-in-c-4-opts and pk/rebase-in-c-5-test; is tangled with ag/sequencer-reduce-rewriting-todo, jc/rebase-in-c-5-test-typofix and js/rebase-i-break.)
>
>  The final step of rewriting "rebase -i" in C.
>
>  Undecided.
>  I've been using this (i.e. the whole "rebase -i" and "rebase"
>  rewritten in C) in my personal build, and I also know users on
>  Windows port have been using it with the last feature release.  I
>  am tempted to merge the whole thing to 'next' soonish.
>
>  Opinions?  It's the last chance to remove any existing and avoid
>  any future "oops, that was wrong, and here is a fix-up"
>  embarrassment in these topics.

Yes, please merge to next.

Stefan

^ permalink raw reply	[relevance 4%]

* Re: [PATCH v6 09/10] submodule: support reading .gitmodules when it's not in the working tree
  2018-10-08 22:19   ` [PATCH v6 09/10] submodule: support reading .gitmodules when it's not in the working tree Stefan Beller
@ 2018-10-10 18:56     ` Antonio Ospite
  2018-10-10 22:55       ` Stefan Beller
  0 siblings, 1 reply; 200+ results
From: Antonio Ospite @ 2018-10-10 18:56 UTC (permalink / raw)
  To: Stefan Beller
  Cc: Junio C Hamano, git, Jonathan Nieder, Jeff King, SZEDER Gábor

On Mon, 8 Oct 2018 15:19:00 -0700
Stefan Beller <sbeller@google.com> wrote:

> > +test_expect_success 'not writing gitmodules config file when it is not checked out' '
> > +        test_must_fail git -C super submodule--helper config submodule.submodule.url newurl
> 
> This only checks the exit code, do we also want to check for
> 
>     test_path_is_missing .gitmodules ?
>

OK, I agree, let's re-check also *after* we tried and failed to set
a config value, just to be sure that the code does not get accidentally
changed in the future to create the file. I'll add the check.

> > +test_expect_success 'initialising submodule when the gitmodules config is not checked out' '
> > +       git -C super submodule init
> > +'
> > +
> > +test_expect_success 'showing submodule summary when the gitmodules config is not checked out' '
> > +       git -C super submodule summary
> > +'
> 
> Same for these, is the exit code enough, or do we want to look at
> specific things?
>

Except for the "summary" test which was not even exercising the
config_from_gitmodule path,  checking exist status should be sufficient
to verify that "submodule--helper config" does not fail, but we can
surely do better.

I will add checks to confirm that not only the commands exited without
errors but they also achieved the desired effect, to validate the actual
high-level use case advertised by the test file. This should be more
future-proof.

And I think I'll merge the summary and the update tests.

> > +
> > +test_expect_success 'updating submodule when the gitmodules config is not checked out' '
> > +       (cd submodule &&
> > +               echo file2 >file2 &&
> > +               git add file2 &&
> > +               git commit -m "add file2 to submodule"
> > +       ) &&
> > +       git -C super submodule update
> 
> git status would want to be clean afterwards?

Mmh, this should have been "submodule update --remote" in the first
place to have any effect, I'll take the chance and rewrite this test in
a different way and also check the effect of the update operation, and
the repository status.

I'll be something like this:

ORIG_SUBMODULE=$(git -C submodule rev-parse HEAD)
ORIG_UPSTREAM=$(git -C upstream rev-parse HEAD)
ORIG_SUPER=$(git -C super rev-parse HEAD)

test_expect_success 're-updating submodule when the gitmodules config is not checked out' '
	test_when_finished "git -C submodule reset --hard $ORIG_SUBMODULE;
	                    git -C upstream reset --hard $ORIG_UPSTREAM;
	                    git -C super reset --hard $ORIG_SUPER;
	                    git -C upstream submodule update --remote;
	                    git -C super pull;
	                    git -C super submodule update --remote" &&
	(cd submodule &&
		echo file2 >file2 &&
		git add file2 &&
		test_tick &&
		git commit -m "add file2 to submodule"
	) &&
	(cd upstream &&
		git submodule update --remote &&
		git add submodule &&
		test_tick &&
		git commit -m "Update submodule"
	) &&
	git -C super pull &&
	# The --for-status options reads the gitmdoules config
	git -C super submodule summary --for-status >actual &&
	cat >expect <<-\EOF &&
	* submodule 951c301...a939200 (1):
	  < add file2 to submodule
	
	EOF
	test_cmp expect actual &&
	# Test that the update actually succeeds
	test_path_is_missing super/submodule/file2 &&
	git -C super submodule update &&
	test_cmp submodule/file2 super/submodule/file2 &&
	git -C super status --short >output &&
	test_must_be_empty output
'

Maybe a little overkill?

The "upstream" repo will be added in test 1 to better clarify the roles
of the involved repositories.

The commit ids should be stable because of test_tick, shouldn't they?

Thanks for the comments, they helped improving the quality of the tests
once again.

I'll wait a few days before sending a v7, hopefully someone will find
time to take another look at patch 9 and comment also on patch 10, and
give an opinion on the "mergeability" status of the whole patchset.

Ciao ciao,
   Antonio

-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 7%]

* [PATCH v5 0/9] fetch: make sure submodule oids are fetched
@ 2018-10-10 21:49 Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 2/9] submodule.c: fix indentation Stefan Beller
                   ` (7 more replies)
  0 siblings, 8 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:49 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

This is nearly the same as sent in [1], with one commit message fixed.
[1] https://public-inbox.org/git/20180925194755.105578-1-sbeller@google.com/
and replaces sb/submodule-recursive-fetch-gets-the-tip.

Thanks,
Stefan

Stefan Beller (9):
  sha1-array: provide oid_array_filter
  submodule.c: fix indentation
  submodule.c: sort changed_submodule_names before searching it
  submodule.c: move global changed_submodule_names into fetch submodule
    struct
  submodule.c: do not copy around submodule list
  repository: repo_submodule_init to take a submodule struct
  submodule: fetch in submodules git directory instead of in worktree
  fetch: retry fetching submodules if needed objects were not fetched
  builtin/fetch: check for submodule updates for non branch fetches

 Documentation/technical/api-oid-array.txt |   5 +
 builtin/fetch.c                           |  14 +-
 builtin/grep.c                            |  17 +-
 builtin/ls-files.c                        |  12 +-
 builtin/submodule--helper.c               |   2 +-
 repository.c                              |  27 +--
 repository.h                              |  11 +-
 sha1-array.c                              |  17 ++
 sha1-array.h                              |   3 +
 submodule.c                               | 275 +++++++++++++++++-----
 t/t5526-fetch-submodules.sh               |  23 +-
 11 files changed, 311 insertions(+), 95 deletions(-)

-- 
2.19.0


^ permalink raw reply	[relevance 11%]

* [PATCH v5 2/9] submodule.c: fix indentation
  2018-10-10 21:49 [PATCH v5 0/9] fetch: make sure submodule oids are fetched Stefan Beller
@ 2018-10-10 21:50 ` Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 3/9] submodule.c: sort changed_submodule_names before searching it Stefan Beller
                   ` (6 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:50 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

The submodule subsystem is really bad at staying within 80 characters.
Fix it while we are here.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/submodule.c b/submodule.c
index b53cb6e9c4..0de9e2800a 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1244,7 +1244,8 @@ static int get_next_submodule(struct child_process *cp,
 		if (!submodule) {
 			const char *name = default_name_or_path(ce->name);
 			if (name) {
-				default_submodule.path = default_submodule.name = name;
+				default_submodule.path = name;
+				default_submodule.name = name;
 				submodule = &default_submodule;
 			}
 		}
@@ -1254,8 +1255,10 @@ static int get_next_submodule(struct child_process *cp,
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names,
-							 submodule->name))
+			if (!submodule ||
+			    !unsorted_string_list_lookup(
+					&changed_submodule_names,
+					submodule->name))
 				continue;
 			default_argv = "on-demand";
 			break;
-- 
2.19.0


^ permalink raw reply	[relevance 26%]

* [PATCH v5 3/9] submodule.c: sort changed_submodule_names before searching it
  2018-10-10 21:49 [PATCH v5 0/9] fetch: make sure submodule oids are fetched Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 2/9] submodule.c: fix indentation Stefan Beller
@ 2018-10-10 21:50 ` Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 4/9] submodule.c: move global changed_submodule_names into fetch submodule struct Stefan Beller
                   ` (5 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:50 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

We can string_list_insert() to maintain sorted-ness of the
list as we find new items, or we can string_list_append() to
build an unsorted list and sort it at the end just once.

As we do not rely on the sortedness while building the
list, we pick the "append and sort at the end" as it
has better worst case execution times.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/submodule.c b/submodule.c
index 0de9e2800a..22c64bd855 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1256,7 +1256,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
-			    !unsorted_string_list_lookup(
+			    !string_list_lookup(
 					&changed_submodule_names,
 					submodule->name))
 				continue;
@@ -1350,6 +1350,7 @@ int fetch_populated_submodules(struct repository *r,
 	/* default value, "--submodule-prefix" and its value are added later */
 
 	calculate_changed_submodule_paths();
+	string_list_sort(&changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* [PATCH v5 4/9] submodule.c: move global changed_submodule_names into fetch submodule struct
  2018-10-10 21:49 [PATCH v5 0/9] fetch: make sure submodule oids are fetched Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 2/9] submodule.c: fix indentation Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 3/9] submodule.c: sort changed_submodule_names before searching it Stefan Beller
@ 2018-10-10 21:50 ` Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 5/9] submodule.c: do not copy around submodule list Stefan Beller
                   ` (4 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:50 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

The `changed_submodule_names` are only used for fetching, so let's make it
part of the struct that is passed around for fetching submodules.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/submodule.c b/submodule.c
index 22c64bd855..17103379ba 100644
--- a/submodule.c
+++ b/submodule.c
@@ -25,7 +25,7 @@
 #include "commit-reach.h"
 
 static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
-static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
+
 static int initialized_fetch_ref_tips;
 static struct oid_array ref_tips_before_fetch;
 static struct oid_array ref_tips_after_fetch;
@@ -1110,7 +1110,22 @@ void check_for_new_submodule_commits(struct object_id *oid)
 	oid_array_append(&ref_tips_after_fetch, oid);
 }
 
-static void calculate_changed_submodule_paths(void)
+struct submodule_parallel_fetch {
+	int count;
+	struct argv_array args;
+	struct repository *r;
+	const char *prefix;
+	int command_line_option;
+	int default_option;
+	int quiet;
+	int result;
+
+	struct string_list changed_submodule_names;
+};
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+
+static void calculate_changed_submodule_paths(
+	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
 	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
@@ -1148,7 +1163,8 @@ static void calculate_changed_submodule_paths(void)
 			continue;
 
 		if (!submodule_has_commits(path, commits))
-			string_list_append(&changed_submodule_names, name->string);
+			string_list_append(&spf->changed_submodule_names,
+					   name->string);
 	}
 
 	free_submodules_oids(&changed_submodules);
@@ -1185,18 +1201,6 @@ int submodule_touches_in_range(struct object_id *excl_oid,
 	return ret;
 }
 
-struct submodule_parallel_fetch {
-	int count;
-	struct argv_array args;
-	struct repository *r;
-	const char *prefix;
-	int command_line_option;
-	int default_option;
-	int quiet;
-	int result;
-};
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
-
 static int get_fetch_recurse_config(const struct submodule *submodule,
 				    struct submodule_parallel_fetch *spf)
 {
@@ -1257,7 +1261,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
 			    !string_list_lookup(
-					&changed_submodule_names,
+					&spf->changed_submodule_names,
 					submodule->name))
 				continue;
 			default_argv = "on-demand";
@@ -1349,8 +1353,8 @@ int fetch_populated_submodules(struct repository *r,
 	argv_array_push(&spf.args, "--recurse-submodules-default");
 	/* default value, "--submodule-prefix" and its value are added later */
 
-	calculate_changed_submodule_paths();
-	string_list_sort(&changed_submodule_names);
+	calculate_changed_submodule_paths(&spf);
+	string_list_sort(&spf.changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
@@ -1359,7 +1363,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&changed_submodule_names, 1);
+	string_list_clear(&spf.changed_submodule_names, 1);
 	return spf.result;
 }
 
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* [PATCH v5 5/9] submodule.c: do not copy around submodule list
  2018-10-10 21:49 [PATCH v5 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (2 preceding siblings ...)
  2018-10-10 21:50 ` [PATCH v5 4/9] submodule.c: move global changed_submodule_names into fetch submodule struct Stefan Beller
@ 2018-10-10 21:50 ` Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 6/9] repository: repo_submodule_init to take a submodule struct Stefan Beller
                   ` (3 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:50 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

'calculate_changed_submodule_paths' uses a local list to compute the
changed submodules, and then produces the result by copying appropriate
items into the result list.

Instead use the result list directly and prune items afterwards
using string_list_remove_empty_items.

By doing so we'll have access to the util pointer for longer that
contains the commits that we need to fetch, which will be
useful in a later patch.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/submodule.c b/submodule.c
index 17103379ba..dd478ed70b 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1128,8 +1128,7 @@ static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
-	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
-	const struct string_list_item *name;
+	struct string_list_item *name;
 
 	/* No need to check if there are no submodules configured */
 	if (!submodule_from_path(the_repository, NULL, NULL))
@@ -1146,9 +1145,9 @@ static void calculate_changed_submodule_paths(
 	 * Collect all submodules (whether checked out or not) for which new
 	 * commits have been recorded upstream in "changed_submodule_names".
 	 */
-	collect_changed_submodules(&changed_submodules, &argv);
+	collect_changed_submodules(&spf->changed_submodule_names, &argv);
 
-	for_each_string_list_item(name, &changed_submodules) {
+	for_each_string_list_item(name, &spf->changed_submodule_names) {
 		struct oid_array *commits = name->util;
 		const struct submodule *submodule;
 		const char *path = NULL;
@@ -1162,12 +1161,14 @@ static void calculate_changed_submodule_paths(
 		if (!path)
 			continue;
 
-		if (!submodule_has_commits(path, commits))
-			string_list_append(&spf->changed_submodule_names,
-					   name->string);
+		if (submodule_has_commits(path, commits)) {
+			oid_array_clear(commits);
+			*name->string = '\0';
+		}
 	}
 
-	free_submodules_oids(&changed_submodules);
+	string_list_remove_empty_items(&spf->changed_submodule_names, 1);
+
 	argv_array_clear(&argv);
 	oid_array_clear(&ref_tips_before_fetch);
 	oid_array_clear(&ref_tips_after_fetch);
@@ -1363,7 +1364,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&spf.changed_submodule_names, 1);
+	free_submodules_oids(&spf.changed_submodule_names);
 	return spf.result;
 }
 
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* [PATCH v5 6/9] repository: repo_submodule_init to take a submodule struct
  2018-10-10 21:49 [PATCH v5 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (3 preceding siblings ...)
  2018-10-10 21:50 ` [PATCH v5 5/9] submodule.c: do not copy around submodule list Stefan Beller
@ 2018-10-10 21:50 ` Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 7/9] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
                   ` (2 subsequent siblings)
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:50 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

When constructing a struct repository for a submodule for some revision
of the superproject where the submodule is not contained in the index,
it may not be present in the working tree currently either. In that
siutation giving a 'path' argument is not useful. Upgrade the
repo_submodule_init function to take a struct submodule instead.

While we are at it, overhaul the repo_submodule_init function by renaming
the submodule repository struct, which is to be initialized, to a name
that is not confused with the struct submodule as easily.

Also move its documentation into the header file.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/grep.c              | 17 ++++++++++-------
 builtin/ls-files.c          | 12 +++++++-----
 builtin/submodule--helper.c |  2 +-
 repository.c                | 27 ++++++++++-----------------
 repository.h                | 11 +++++++++--
 5 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/builtin/grep.c b/builtin/grep.c
index 601f801158..81c53c862b 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -418,16 +418,19 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 			  const struct object_id *oid,
 			  const char *filename, const char *path)
 {
-	struct repository submodule;
+	struct repository subrepo;
+	const struct submodule *sub = submodule_from_path(superproject,
+							  &null_oid, path);
+
 	int hit;
 
 	if (!is_submodule_active(superproject, path))
 		return 0;
 
-	if (repo_submodule_init(&submodule, superproject, path))
+	if (repo_submodule_init(&subrepo, superproject, sub))
 		return 0;
 
-	repo_read_gitmodules(&submodule);
+	repo_read_gitmodules(&subrepo);
 
 	/*
 	 * NEEDSWORK: This adds the submodule's object directory to the list of
@@ -440,7 +443,7 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 	 * object.
 	 */
 	grep_read_lock();
-	add_to_alternates_memory(submodule.objects->objectdir);
+	add_to_alternates_memory(subrepo.objects->objectdir);
 	grep_read_unlock();
 
 	if (oid) {
@@ -465,14 +468,14 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 
 		init_tree_desc(&tree, data, size);
 		hit = grep_tree(opt, pathspec, &tree, &base, base.len,
-				object->type == OBJ_COMMIT, &submodule);
+				object->type == OBJ_COMMIT, &subrepo);
 		strbuf_release(&base);
 		free(data);
 	} else {
-		hit = grep_cache(opt, &submodule, pathspec, 1);
+		hit = grep_cache(opt, &subrepo, pathspec, 1);
 	}
 
-	repo_clear(&submodule);
+	repo_clear(&subrepo);
 	return hit;
 }
 
diff --git a/builtin/ls-files.c b/builtin/ls-files.c
index 7f9919a362..4d1649c1b3 100644
--- a/builtin/ls-files.c
+++ b/builtin/ls-files.c
@@ -206,17 +206,19 @@ static void show_files(struct repository *repo, struct dir_struct *dir);
 static void show_submodule(struct repository *superproject,
 			   struct dir_struct *dir, const char *path)
 {
-	struct repository submodule;
+	struct repository subrepo;
+	const struct submodule *sub = submodule_from_path(superproject,
+							  &null_oid, path);
 
-	if (repo_submodule_init(&submodule, superproject, path))
+	if (repo_submodule_init(&subrepo, superproject, sub))
 		return;
 
-	if (repo_read_index(&submodule) < 0)
+	if (repo_read_index(&subrepo) < 0)
 		die("index file corrupt");
 
-	show_files(&submodule, dir);
+	show_files(&subrepo, dir);
 
-	repo_clear(&submodule);
+	repo_clear(&subrepo);
 }
 
 static void show_ce(struct repository *repo, struct dir_struct *dir,
diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 247881189f..8214e77688 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -2038,7 +2038,7 @@ static int ensure_core_worktree(int argc, const char **argv, const char *prefix)
 	if (!sub)
 		BUG("We could get the submodule handle before?");
 
-	if (repo_submodule_init(&subrepo, the_repository, path))
+	if (repo_submodule_init(&subrepo, the_repository, sub))
 		die(_("could not get a repository handle for submodule '%s'"), path);
 
 	if (!repo_config_get_string(&subrepo, "core.worktree", &cw)) {
diff --git a/repository.c b/repository.c
index 5dd1486718..aabe64ee5d 100644
--- a/repository.c
+++ b/repository.c
@@ -166,30 +166,23 @@ int repo_init(struct repository *repo,
 	return -1;
 }
 
-/*
- * Initialize 'submodule' as the submodule given by 'path' in parent repository
- * 'superproject'.
- * Return 0 upon success and a non-zero value upon failure.
- */
-int repo_submodule_init(struct repository *submodule,
+int repo_submodule_init(struct repository *subrepo,
 			struct repository *superproject,
-			const char *path)
+			const struct submodule *sub)
 {
-	const struct submodule *sub;
 	struct strbuf gitdir = STRBUF_INIT;
 	struct strbuf worktree = STRBUF_INIT;
 	int ret = 0;
 
-	sub = submodule_from_path(superproject, &null_oid, path);
 	if (!sub) {
 		ret = -1;
 		goto out;
 	}
 
-	strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", path);
-	strbuf_repo_worktree_path(&worktree, superproject, "%s", path);
+	strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", sub->path);
+	strbuf_repo_worktree_path(&worktree, superproject, "%s", sub->path);
 
-	if (repo_init(submodule, gitdir.buf, worktree.buf)) {
+	if (repo_init(subrepo, gitdir.buf, worktree.buf)) {
 		/*
 		 * If initilization fails then it may be due to the submodule
 		 * not being populated in the superproject's worktree.  Instead
@@ -201,16 +194,16 @@ int repo_submodule_init(struct repository *submodule,
 		strbuf_repo_git_path(&gitdir, superproject,
 				     "modules/%s", sub->name);
 
-		if (repo_init(submodule, gitdir.buf, NULL)) {
+		if (repo_init(subrepo, gitdir.buf, NULL)) {
 			ret = -1;
 			goto out;
 		}
 	}
 
-	submodule->submodule_prefix = xstrfmt("%s%s/",
-					      superproject->submodule_prefix ?
-					      superproject->submodule_prefix :
-					      "", path);
+	subrepo->submodule_prefix = xstrfmt("%s%s/",
+					    superproject->submodule_prefix ?
+					    superproject->submodule_prefix :
+					    "", sub->path);
 
 out:
 	strbuf_release(&gitdir);
diff --git a/repository.h b/repository.h
index 9f16c42c1e..a9c7a5baa5 100644
--- a/repository.h
+++ b/repository.h
@@ -116,9 +116,16 @@ void repo_set_worktree(struct repository *repo, const char *path);
 void repo_set_hash_algo(struct repository *repo, int algo);
 void initialize_the_repository(void);
 int repo_init(struct repository *r, const char *gitdir, const char *worktree);
-int repo_submodule_init(struct repository *submodule,
+
+/*
+ * Initialize the repository 'subrepo' as the submodule given by the
+ * struct submodule 'sub' in parent repository 'superproject'.
+ * Return 0 upon success and a non-zero value upon failure.
+ */
+struct submodule;
+int repo_submodule_init(struct repository *subrepo,
 			struct repository *superproject,
-			const char *path);
+			const struct submodule *sub);
 void repo_clear(struct repository *repo);
 
 /*
-- 
2.19.0


^ permalink raw reply	[relevance 26%]

* [PATCH v5 7/9] submodule: fetch in submodules git directory instead of in worktree
  2018-10-10 21:49 [PATCH v5 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (4 preceding siblings ...)
  2018-10-10 21:50 ` [PATCH v5 6/9] repository: repo_submodule_init to take a submodule struct Stefan Beller
@ 2018-10-10 21:50 ` Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 8/9] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 9/9] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:50 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

This patch started as a refactoring to make 'get_next_submodule' more
readable, but upon doing so, I realized that "git fetch" of the submodule
actually doesn't need to be run in the submodules worktree. So let's run
it in its git dir instead.

That should pave the way towards fetching submodules that are currently
not checked out.

This patch leaks the cp->dir in get_next_submodule, as any further
callback in run_processes_parallel doesn't have access to the child
process any more. In an early iteration of this patch, the function
get_submodule_repo_for directly returned the string containing the
git directory, which would be a better design choice for this patch.

However the next patch both fixes the memory leak of cp->dir and also has
a use case for using the full repository handle of the submodule, so
it makes sense to introduce the get_submodule_repo_for here already.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c                 | 51 +++++++++++++++++++++++++++----------
 t/t5526-fetch-submodules.sh |  7 ++++-
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/submodule.c b/submodule.c
index dd478ed70b..3f791f2277 100644
--- a/submodule.c
+++ b/submodule.c
@@ -481,6 +481,12 @@ void prepare_submodule_repo_env(struct argv_array *out)
 			 DEFAULT_GIT_DIR_ENVIRONMENT);
 }
 
+static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out)
+{
+	prepare_submodule_repo_env_no_git_dir(out);
+	argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
+}
+
 /* Helper function to display the submodule header line prior to the full
  * summary output. If it can locate the submodule objects directory it will
  * attempt to lookup both the left and right commits and put them into the
@@ -1227,6 +1233,29 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+static struct repository *get_submodule_repo_for(struct repository *r,
+						 const struct submodule *sub)
+{
+	struct repository *ret = xmalloc(sizeof(*ret));
+
+	if (repo_submodule_init(ret, r, sub)) {
+		/*
+		 * No entry in .gitmodules? Technically not a submodule,
+		 * but historically we supported repositories that happen to be
+		 * in-place where a gitlink is. Keep supporting them.
+		 */
+		struct strbuf gitdir = STRBUF_INIT;
+		strbuf_repo_worktree_path(&gitdir, r, "%s/.git", sub->path);
+		if (repo_init(ret, gitdir.buf, NULL)) {
+			strbuf_release(&gitdir);
+			return NULL;
+		}
+		strbuf_release(&gitdir);
+	}
+
+	return ret;
+}
+
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
@@ -1234,12 +1263,11 @@ static int get_next_submodule(struct child_process *cp,
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_path = STRBUF_INIT;
-		struct strbuf submodule_git_dir = STRBUF_INIT;
 		struct strbuf submodule_prefix = STRBUF_INIT;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
-		const char *git_dir, *default_argv;
+		const char *default_argv;
 		const struct submodule *submodule;
+		struct repository *repo;
 		struct submodule default_submodule = SUBMODULE_INIT;
 
 		if (!S_ISGITLINK(ce->ce_mode))
@@ -1274,16 +1302,12 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_repo_worktree_path(&submodule_path, spf->r, "%s", ce->name);
-		strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
 		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		git_dir = read_gitfile(submodule_git_dir.buf);
-		if (!git_dir)
-			git_dir = submodule_git_dir.buf;
-		if (is_directory(git_dir)) {
+		repo = get_submodule_repo_for(spf->r, submodule);
+		if (repo) {
 			child_process_init(cp);
-			cp->dir = strbuf_detach(&submodule_path, NULL);
-			prepare_submodule_repo_env(&cp->env_array);
+			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+			cp->dir = xstrdup(repo->gitdir);
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1293,10 +1317,11 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
 			argv_array_push(&cp->args, submodule_prefix.buf);
+
+			repo_clear(repo);
+			free(repo);
 			ret = 1;
 		}
-		strbuf_release(&submodule_path);
-		strbuf_release(&submodule_git_dir);
 		strbuf_release(&submodule_prefix);
 		if (ret) {
 			spf->count++;
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 6c2f9b2ba2..42692219a1 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -566,7 +566,12 @@ test_expect_success 'fetching submodule into a broken repository' '
 
 	test_must_fail git -C dst status &&
 	test_must_fail git -C dst diff &&
-	test_must_fail git -C dst fetch --recurse-submodules
+
+	# git-fetch cannot find the git directory of the submodule,
+	# so it will do nothing, successfully, as it cannot distinguish between
+	# this broken submodule and a submodule that was just set active but
+	# not cloned yet
+	git -C dst fetch --recurse-submodules
 '
 
 test_expect_success "fetch new commits when submodule got renamed" '
-- 
2.19.0


^ permalink raw reply	[relevance 28%]

* [PATCH v5 8/9] fetch: retry fetching submodules if needed objects were not fetched
  2018-10-10 21:49 [PATCH v5 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (5 preceding siblings ...)
  2018-10-10 21:50 ` [PATCH v5 7/9] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
@ 2018-10-10 21:50 ` Stefan Beller
  2018-10-10 21:50 ` [PATCH v5 9/9] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:50 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

Currently when git-fetch is asked to recurse into submodules, it dispatches
a plain "git-fetch -C <submodule-dir>" (with some submodule related options
such as prefix and recusing strategy, but) without any information of the
remote or the tip that should be fetched.

This works surprisingly well in some workflows (such as using submodules
as a third party library), while not so well in other scenarios, such
as in a Gerrit topic-based workflow, that can tie together changes
(potentially across repositories) on the server side. One of the parts
of such a Gerrit workflow is to download a change when wanting to examine
it, and you'd want to have its submodule changes that are in the same
topic downloaded as well. However these submodule changes reside in their
own repository in their own ref (refs/changes/<int>).

Retry fetching a submodule by object name if the object id that the
superproject points to, cannot be found.

This retrying does not happen when the "git fetch" done at the
superproject is not storing the fetched results in remote
tracking branches (i.e. instead just recording them to
FETCH_HEAD) in this step. A later patch will fix this.

builtin/fetch used to only inspect submodules when they were fetched
"on-demand", as in either on/off case it was clear whether the submodule
needs to be fetched. However to know whether we need to try fetching the
object ids, we need to identify the object names, which is done in this
function check_for_new_submodule_commits(), so we'll also run that code
in case the submodule recursion is set to "on".

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c             |   9 +-
 submodule.c                 | 185 ++++++++++++++++++++++++++++++------
 t/t5526-fetch-submodules.sh |  16 ++++
 3 files changed, 177 insertions(+), 33 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 0696abfc2a..e3b03ad3bd 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -707,8 +707,7 @@ static int update_local_ref(struct ref *ref,
 			what = _("[new ref]");
 		}
 
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref(msg, ref, 0);
 		format_display(display, r ? '!' : '*', what,
@@ -723,8 +722,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "..");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("fast-forward", ref, 1);
 		format_display(display, r ? '!' : ' ', quickref.buf,
@@ -738,8 +736,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "...");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("forced-update", ref, 1);
 		format_display(display, r ? '!' : '+', quickref.buf,
diff --git a/submodule.c b/submodule.c
index 3f791f2277..05799362e0 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1127,8 +1127,12 @@ struct submodule_parallel_fetch {
 	int result;
 
 	struct string_list changed_submodule_names;
+	struct get_next_submodule_task **retry;
+	int retry_nr, retry_alloc;
 };
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \
+		  STRING_LIST_INIT_DUP, \
+		  NULL, 0, 0}
 
 static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
@@ -1233,6 +1237,56 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+struct get_next_submodule_task {
+	struct repository *repo;
+	const struct submodule *sub;
+	unsigned free_sub : 1; /* Do we need to free the submodule? */
+	struct oid_array *commits;
+};
+
+static const struct submodule *get_default_submodule(const char *path)
+{
+	struct submodule *ret = NULL;
+	const char *name = default_name_or_path(path);
+
+	if (!name)
+		return NULL;
+
+	ret = xmalloc(sizeof(*ret));
+	memset(ret, 0, sizeof(*ret));
+	ret->path = name;
+	ret->name = name;
+
+	return (const struct submodule *) ret;
+}
+
+static struct get_next_submodule_task *get_next_submodule_task_create(
+	struct repository *r, const char *path)
+{
+	struct get_next_submodule_task *task = xmalloc(sizeof(*task));
+	memset(task, 0, sizeof(*task));
+
+	task->sub = submodule_from_path(r, &null_oid, path);
+	if (!task->sub) {
+		task->sub = get_default_submodule(path);
+		task->free_sub = 1;
+	}
+
+	return task;
+}
+
+static void get_next_submodule_task_release(struct get_next_submodule_task *p)
+{
+	if (p->free_sub)
+		free((void*)p->sub);
+	p->free_sub = 0;
+	p->sub = NULL;
+
+	if (p->repo)
+		repo_clear(p->repo);
+	FREE_AND_NULL(p->repo);
+}
+
 static struct repository *get_submodule_repo_for(struct repository *r,
 						 const struct submodule *sub)
 {
@@ -1259,39 +1313,35 @@ static struct repository *get_submodule_repo_for(struct repository *r,
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
-	int ret = 0;
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_prefix = STRBUF_INIT;
+		int recurse_config;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
 		const char *default_argv;
-		const struct submodule *submodule;
-		struct repository *repo;
-		struct submodule default_submodule = SUBMODULE_INIT;
+		struct get_next_submodule_task *task;
 
 		if (!S_ISGITLINK(ce->ce_mode))
 			continue;
 
-		submodule = submodule_from_path(spf->r, &null_oid, ce->name);
-		if (!submodule) {
-			const char *name = default_name_or_path(ce->name);
-			if (name) {
-				default_submodule.path = name;
-				default_submodule.name = name;
-				submodule = &default_submodule;
-			}
+		task = get_next_submodule_task_create(spf->r, ce->name);
+
+		if (!task->sub) {
+			free(task);
+			continue;
 		}
 
-		switch (get_fetch_recurse_config(submodule, spf))
+		recurse_config = get_fetch_recurse_config(task->sub, spf);
+
+		switch (recurse_config)
 		{
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule ||
+			if (!task->sub ||
 			    !string_list_lookup(
 					&spf->changed_submodule_names,
-					submodule->name))
+					task->sub->name))
 				continue;
 			default_argv = "on-demand";
 			break;
@@ -1302,12 +1352,12 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		repo = get_submodule_repo_for(spf->r, submodule);
-		if (repo) {
+		task->repo = get_submodule_repo_for(spf->r, task->sub);
+		if (task->repo) {
+			struct strbuf submodule_prefix = STRBUF_INIT;
 			child_process_init(cp);
 			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
-			cp->dir = xstrdup(repo->gitdir);
+			cp->dir = task->repo->gitdir;
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1316,18 +1366,51 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_pushv(&cp->args, spf->args.argv);
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
+
+			strbuf_addf(&submodule_prefix, "%s%s/",
+						       spf->prefix,
+						       task->sub->path);
 			argv_array_push(&cp->args, submodule_prefix.buf);
 
-			repo_clear(repo);
-			free(repo);
-			ret = 1;
-		}
-		strbuf_release(&submodule_prefix);
-		if (ret) {
 			spf->count++;
+			*task_cb = task;
+
+			strbuf_release(&submodule_prefix);
 			return 1;
+		} else {
+			get_next_submodule_task_release(task);
+			free(task);
 		}
 	}
+
+	if (spf->retry_nr) {
+		struct get_next_submodule_task *task = spf->retry[spf->retry_nr - 1];
+		struct strbuf submodule_prefix = STRBUF_INIT;
+		spf->retry_nr--;
+
+		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, task->sub->path);
+
+		child_process_init(cp);
+		prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+		cp->git_cmd = 1;
+		cp->dir = task->repo->gitdir;
+
+		argv_array_init(&cp->args);
+		argv_array_pushv(&cp->args, spf->args.argv);
+		argv_array_push(&cp->args, "on-demand");
+		argv_array_push(&cp->args, "--submodule-prefix");
+		argv_array_push(&cp->args, submodule_prefix.buf);
+
+		/* NEEDSWORK: have get_default_remote from s--h */
+		argv_array_push(&cp->args, "origin");
+		oid_array_for_each_unique(task->commits,
+					  append_oid_to_argv, &cp->args);
+
+		*task_cb = task;
+		strbuf_release(&submodule_prefix);
+		return 1;
+	}
+
 	return 0;
 }
 
@@ -1335,20 +1418,68 @@ static int fetch_start_failure(struct strbuf *err,
 			       void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct get_next_submodule_task *task = task_cb;
 
 	spf->result = 1;
 
+	get_next_submodule_task_release(task);
 	return 0;
 }
 
+static int commit_exists_in_sub(const struct object_id *oid, void *data)
+{
+	struct repository *subrepo = data;
+
+	enum object_type type = oid_object_info(subrepo, oid, NULL);
+
+	return type != OBJ_COMMIT;
+}
+
 static int fetch_finish(int retvalue, struct strbuf *err,
 			void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct get_next_submodule_task *task = task_cb;
+	const struct submodule *sub;
+
+	struct string_list_item *it;
+	struct oid_array *commits;
 
 	if (retvalue)
 		spf->result = 1;
 
+	if (!task)
+		return 0;
+
+	sub = task->sub;
+	if (!sub)
+		goto out;
+
+	it = string_list_lookup(&spf->changed_submodule_names, sub->name);
+	if (!it)
+		goto out;
+
+	commits = it->util;
+	oid_array_filter(commits,
+			 commit_exists_in_sub,
+			 task->repo);
+
+	/* Are there commits that do not exist? */
+	if (commits->nr) {
+		/* We already tried fetching them, do not try again. */
+		if (task->commits)
+			return 0;
+
+		task->commits = commits;
+		ALLOC_GROW(spf->retry, spf->retry_nr + 1, spf->retry_alloc);
+		spf->retry[spf->retry_nr] = task;
+		spf->retry_nr++;
+		return 0;
+	}
+
+out:
+	get_next_submodule_task_release(task);
+
 	return 0;
 }
 
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 42692219a1..af12c50e7d 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -605,4 +605,20 @@ test_expect_success "fetch new commits when submodule got renamed" '
 	test_cmp expect actual
 '
 
+test_expect_success "fetch new commits on-demand when they are not reachable" '
+	git checkout --detach &&
+	C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
+	git -C submodule update-ref refs/changes/1 $C &&
+	git update-index --cacheinfo 160000 $C submodule &&
+	git commit -m "updated submodule outside of refs/heads" &&
+	D=$(git rev-parse HEAD) &&
+	git update-ref refs/changes/2 $D &&
+	(
+		cd downstream &&
+		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git -C submodule cat-file -t $C &&
+		git checkout --recurse-submodules FETCH_HEAD
+	)
+'
+
 test_done
-- 
2.19.0


^ permalink raw reply	[relevance 25%]

* [PATCH v5 9/9] builtin/fetch: check for submodule updates for non branch fetches
  2018-10-10 21:49 [PATCH v5 0/9] fetch: make sure submodule oids are fetched Stefan Beller
                   ` (6 preceding siblings ...)
  2018-10-10 21:50 ` [PATCH v5 8/9] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
@ 2018-10-10 21:50 ` Stefan Beller
  7 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 21:50 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

Gerrit, the code review tool, has a different workflow than our mailing
list based approach. Usually users upload changes to a Gerrit server and
continuous integration and testing happens by bots. Sometimes however a
user wants to checkout a change locally and look at it locally. For this
use case, Gerrit offers a command line snippet to copy and paste to your
terminal, which looks like

  git fetch https://<host>/gerrit refs/changes/<id> &&
  git checkout FETCH_HEAD

For Gerrit changes that contain changing submodule gitlinks, it would be
easy to extend both the fetch and checkout with the '--recurse-submodules'
flag, such that this command line snippet would produce the state of a
change locally.

However the functionality added in the previous patch, which would
ensure that we fetch the objects in the submodule that the gitlink pointed
at, only works for remote tracking branches so far, not for FETCH_HEAD.

Make sure that fetching a superproject to its FETCH_HEAD, also respects
the existence checks for objects in the submodule recursion.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/fetch.c             | 5 ++++-
 t/t5526-fetch-submodules.sh | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index e3b03ad3bd..f2d9e548bf 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -894,11 +894,14 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
 				rc |= update_local_ref(ref, what, rm, &note,
 						       summary_width);
 				free(ref);
-			} else
+			} else {
+				check_for_new_submodule_commits(&rm->old_oid);
 				format_display(&note, '*',
 					       *kind ? kind : "branch", NULL,
 					       *what ? what : "HEAD",
 					       "FETCH_HEAD", summary_width);
+			}
+
 			if (note.len) {
 				if (verbosity >= 0 && !shown_url) {
 					fprintf(stderr, _("From %.*s\n"),
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index af12c50e7d..a509eabb04 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -615,7 +615,7 @@ test_expect_success "fetch new commits on-demand when they are not reachable" '
 	git update-ref refs/changes/2 $D &&
 	(
 		cd downstream &&
-		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git fetch --recurse-submodules origin refs/changes/2 &&
 		git -C submodule cat-file -t $C &&
 		git checkout --recurse-submodules FETCH_HEAD
 	)
-- 
2.19.0


^ permalink raw reply	[relevance 22%]

* Re: [PATCH] builtin/grep.c: remote superflous submodule code
      [irrelevant] ` <20181010001037.74709-1-jonathantanmy@google.com>
@ 2018-10-10 22:49   ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 22:49 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: Junio C Hamano, Antonio Ospite, git

On Tue, Oct 9, 2018 at 5:10 PM Jonathan Tan <jonathantanmy@google.com> wrote:
>
> > It claimed that grep would still need some explicit handling, but that is
> > not the call to repo_read_gitmodules (applying this patch on top of
> > ff6f1f564c4 still keep the test suite happy, specifically
> > t7814-grep-recurse-submodules, which contains a test
> > "grep history with moved submoules")
>
> Firstly, spelling of "remove" and "superfluous" in the commit title.
>
> I don't think the "grep history with moved submodules" test exercises
> much. That test only tests the superproject > submodule case, but we
> need a superproject > submodule > sub-submodule case, because what is
> being removed is a call to repo_read_gitmodules() on a repository
> ("struct repository submodule") that has a superproject ("struct
> repository *superproject"). In other words, we need a submodule that has
> its own gitmodules.

Right; we do have a test 'grep and nested submodules', which still passes.
I added another test, that would grep through nested submodules in
the history (not checked out), but that would not work on nested submodules
with or without this patch applied. (As the nested submodule is not checked
out, is_submodule_active(repo, path) would return false and we'd not dive
into the nested submodule.

I looked into ao/submodule-wo-gitmodules-checked-out, as that touches
this area of code as well and promises to allow working with submodules
when .gitmodules is not checked out, it doesn't help this use case, either.

That is (as Antonio diagnosed), due to get_oid not working with a repository
handle, yet.

> > The special handling is the call to gitmodules_config_oid which was added
> > already in 74ed43711f (grep: enable recurse-submodules to work on
> > <tree> objects, 2016-12-16), but then was still named
> > gitmodules_config_sha1.
>
> If you're stating that gitmodules_config_oid() is where the .gitmodules
> file is lazily loaded, it doesn't seem to be that way, because that
> function works only on the_repository (at least on 'master' and 'next').

yes, that is why nested submodules do not work currently when they
are not in the working tree.

>
> > This is a resend of origin/sb/grep-submodule-cleanup,
> > and I think picking ff6f1f564c4 as the base for the series would
> > also be appropriate.
>
> Any particular reason why you suggest that commit (which is more than a
> year old)? It seems that basing this on 'master' is fine.

After more analysis, I think we'd want to wait for Antonios series to land
and then build on top of that, while also getting get_oid converted.

Regarding this patch, let's retract it for now and revisit it once we have
more submodule infrastructure working.

Thanks,
Stefan

^ permalink raw reply	[relevance 8%]

* Re: [PATCH v6 09/10] submodule: support reading .gitmodules when it's not in the working tree
  2018-10-10 18:56     ` Antonio Ospite
@ 2018-10-10 22:55       ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-10 22:55 UTC (permalink / raw)
  To: Antonio Ospite
  Cc: Junio C Hamano, git, Jonathan Nieder, Jeff King, SZEDER Gábor

On Wed, Oct 10, 2018 at 11:56 AM Antonio Ospite <ao2@ao2.it> wrote:
>
> On Mon, 8 Oct 2018 15:19:00 -0700
> Stefan Beller <sbeller@google.com> wrote:
>
> > > +test_expect_success 'not writing gitmodules config file when it is not checked out' '
> > > +        test_must_fail git -C super submodule--helper config submodule.submodule.url newurl
> >
> > This only checks the exit code, do we also want to check for
> >
> >     test_path_is_missing .gitmodules ?
> >
>
> OK, I agree, let's re-check also *after* we tried and failed to set
> a config value, just to be sure that the code does not get accidentally
> changed in the future to create the file. I'll add the check.
>
> > > +test_expect_success 'initialising submodule when the gitmodules config is not checked out' '
> > > +       git -C super submodule init
> > > +'
> > > +
> > > +test_expect_success 'showing submodule summary when the gitmodules config is not checked out' '
> > > +       git -C super submodule summary
> > > +'
> >
> > Same for these, is the exit code enough, or do we want to look at
> > specific things?
> >
>
> Except for the "summary" test which was not even exercising the
> config_from_gitmodule path,  checking exist status should be sufficient
> to verify that "submodule--helper config" does not fail, but we can
> surely do better.
>
> I will add checks to confirm that not only the commands exited without
> errors but they also achieved the desired effect, to validate the actual
> high-level use case advertised by the test file. This should be more
> future-proof.
>
> And I think I'll merge the summary and the update tests.
>
> > > +
> > > +test_expect_success 'updating submodule when the gitmodules config is not checked out' '
> > > +       (cd submodule &&
> > > +               echo file2 >file2 &&
> > > +               git add file2 &&
> > > +               git commit -m "add file2 to submodule"
> > > +       ) &&
> > > +       git -C super submodule update
> >
> > git status would want to be clean afterwards?
>
> Mmh, this should have been "submodule update --remote" in the first
> place to have any effect, I'll take the chance and rewrite this test in
> a different way and also check the effect of the update operation, and
> the repository status.
>
> I'll be something like this:
>
> ORIG_SUBMODULE=$(git -C submodule rev-parse HEAD)
> ORIG_UPSTREAM=$(git -C upstream rev-parse HEAD)
> ORIG_SUPER=$(git -C super rev-parse HEAD)
>
> test_expect_success 're-updating submodule when the gitmodules config is not checked out' '
>         test_when_finished "git -C submodule reset --hard $ORIG_SUBMODULE;
>                             git -C upstream reset --hard $ORIG_UPSTREAM;
>                             git -C super reset --hard $ORIG_SUPER;
>                             git -C upstream submodule update --remote;
>                             git -C super pull;
>                             git -C super submodule update --remote" &&
>         (cd submodule &&
>                 echo file2 >file2 &&
>                 git add file2 &&
>                 test_tick &&
>                 git commit -m "add file2 to submodule"
>         ) &&
>         (cd upstream &&
>                 git submodule update --remote &&
>                 git add submodule &&
>                 test_tick &&
>                 git commit -m "Update submodule"
>         ) &&
>         git -C super pull &&
>         # The --for-status options reads the gitmdoules config

gitmodules

>         git -C super submodule summary --for-status >actual &&
>         cat >expect <<-\EOF &&
>         * submodule 951c301...a939200 (1):

hardcoding hash values burdens the plan to migrate to another
hash function,

    rev1=$(git -C submodule rev-parse --short HEAD^)
    rev2=$(git -C submodule rev-parse --short HEAD)

and then use ${rev1}..${rev2} ?


>           < add file2 to submodule
>
>         EOF
>         test_cmp expect actual &&
>         # Test that the update actually succeeds
>         test_path_is_missing super/submodule/file2 &&
>         git -C super submodule update &&
>         test_cmp submodule/file2 super/submodule/file2 &&
>         git -C super status --short >output &&
>         test_must_be_empty output
> '
>
> Maybe a little overkill?

Wow, very thorough! You might call it overkill, but now that you have it...

> The "upstream" repo will be added in test 1 to better clarify the roles
> of the involved repositories.
>
> The commit ids should be stable because of test_tick, shouldn't they?

Yes, but see
Documentation/technical/hash-function-transition.txt
that a couple people are working on. Let's be nice to them. :-)

Stefan

^ permalink raw reply	[relevance 8%]

* Re: What's cooking in git.git (Oct 2018, #01; Wed, 10)
  2018-10-10 18:55 ` What's cooking in git.git (Oct 2018, #01; Wed, 10) Stefan Beller
@ 2018-10-11  2:00   ` Junio C Hamano
  0 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-10-11  2:00 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

Stefan Beller <sbeller@google.com> writes:

>> * pw/diff-color-moved-ws-fix (2018-10-04) 5 commits
> I would suggest merging to 'next'.

OK.

>> * sb/strbuf-h-update (2018-09-29) 1 commit
> The patch as-is just adds names everywhere.
> I'd be happy to resend with either
> (a) not enforcing names everywhere, but only as needed or
> (b) having names everywhere, capitalizing them NAMES in
>     the doc comment.
>
> I am tempted to ask for
> (c) take as-is, defer the rewording of doc strings for a follow up patch.

As long as the planned update eventually comes before all of us
forget, (c) is fine by me.  I'll mark it to be merged to 'next' for
now, and follow through that plan, unless somebody else stops me
before it happens.

>> * sb/submodule-recursive-fetch-gets-the-tip (2018-09-12) 9 commits
> Will resend after a local review.

OK.

Thanks for helping me in updating the status for various topics.

^ permalink raw reply	[relevance 2%]

* [RFC PATCH 00/19] Bring more repository handles into our code base
@ 2018-10-11 21:17 Stefan Beller
  2018-10-11 21:17 ` [PATCH 17/19] submodule: use submodule repos for object lookup Stefan Beller
                   ` (3 more replies)
  0 siblings, 4 replies; 200+ results
From: Stefan Beller @ 2018-10-11 21:17 UTC (permalink / raw)
  To: git; +Cc: jonathantanmy, Stefan Beller

This applies on nd/the-index (b3c7eef9b05) and is the logical continuation of
the object store series, which I sent over the last year.

The previous series did take a very slow and pedantic approach,
using a #define trick, see cfc62fc98c for details, but it turns out,
that it doesn't work:
   When changing the signature of widely used functions, it burdens the
   maintainer in resolving the semantic conflicts.
   
   In the orginal approach this was called a feature, as then we can ensure
   that not bugs creep into the code base during the merge window (while such
   a refactoring series wanders from pu to master). It turns out this
   was not well received and was just burdensome.
   
   The #define trick doesn't buy us much to begin with when dealing with
   non-merge-conflicts.  For example, see deref_tag at tag.c:68, which got
   the repository argument in 286d258d4f (tag.c: allow deref_tag to handle
   arbitrary repositories, 2018-06-28) but lost its property of working on any
   repository while 8c4cc32689 (tag: don't warn if target is missing but
   promised, 2018-07-12) was in flight simultaneously.
   
   Another example of failure of this approach is seen in patch 5, which
   shows that the pedantry was missed.
        
This series takes another approach as it doesn't change the signature of
functions, but introduces new functions that can deal with arbitrary 
repositories, keeping the old function signature around using a shallow wrapper.

Additionally each patch adds a semantic patch, that would port from the old to
the new function. These semantic patches are all applied in the very last patch,
but we could omit applying the last patch if it causes too many merge conflicts
and trickl in the semantic patches over time when there are no merge conflicts.


The original goal of all these refactoring series was to remove add_submodule_odb 
in submodule.c, which was partially reached with this series. I'll investigate the
remaining calls in another series, but it shows we're close to be done with these
large refactorings as far as I am concerned.

Thanks,
Stefan

Stefan Beller (19):
  sha1_file: allow read_object to read objects in arbitrary repositories
  packfile: allow has_packed_and_bad to handle arbitrary repositories
  object-store: allow read_object_file_extended to read from arbitrary
    repositories
  object-store: prepare read_object_file to deal with arbitrary
    repositories
  object: parse_object to honor its repository argument
  commit: allow parse_commit* to handle arbitrary repositories
  commit.c: allow paint_down_to_common to handle arbitrary repositories
  commit.c: allow merge_bases_many to handle arbitrary repositories
  commit.c: allow remove_redundant to handle arbitrary repositories
  commit: allow get_merge_bases_many_0 to handle arbitrary repositories
  commit: prepare get_merge_bases to handle arbitrary repositories
  commit: prepare get_commit_buffer to handle arbitrary repositories
  commit: prepare in_merge_bases[_many] to handle arbitrary repositories
  commit: prepare repo_unuse_commit_buffer to handle arbitrary
    repositories
  commit: prepare logmsg_reencode to handle arbitrary repositories
  pretty: prepare format_commit_message to handle arbitrary repositories
  submodule: use submodule repos for object lookup
  submodule: don't add submodule as odb for push
  Apply semantic patches from previous patches

 apply.c                                 |   6 +-
 archive.c                               |   5 +-
 bisect.c                                |   5 +-
 blame.c                                 |  15 +--
 builtin/am.c                            |   2 +-
 builtin/blame.c                         |   4 +-
 builtin/cat-file.c                      |  21 +++--
 builtin/checkout.c                      |   4 +-
 builtin/commit.c                        |  13 ++-
 builtin/describe.c                      |   4 +-
 builtin/difftool.c                      |   3 +-
 builtin/fast-export.c                   |   7 +-
 builtin/fmt-merge-msg.c                 |   8 +-
 builtin/grep.c                          |   2 +-
 builtin/index-pack.c                    |   8 +-
 builtin/log.c                           |   4 +-
 builtin/merge-base.c                    |   2 +-
 builtin/merge-tree.c                    |   9 +-
 builtin/mktag.c                         |   3 +-
 builtin/name-rev.c                      |   2 +-
 builtin/notes.c                         |  12 ++-
 builtin/pack-objects.c                  |  22 +++--
 builtin/reflog.c                        |   5 +-
 builtin/replace.c                       |   2 +-
 builtin/shortlog.c                      |   5 +-
 builtin/show-branch.c                   |   4 +-
 builtin/tag.c                           |   4 +-
 builtin/unpack-file.c                   |   2 +-
 builtin/unpack-objects.c                |   3 +-
 builtin/verify-commit.c                 |   2 +-
 bundle.c                                |   2 +-
 combine-diff.c                          |   2 +-
 commit-graph.c                          |   8 +-
 commit.c                                | 120 ++++++++++++++----------
 commit.h                                |  67 ++++++++++---
 config.c                                |   2 +-
 contrib/coccinelle/the_repository.cocci | 114 ++++++++++++++++++++++
 diff.c                                  |   3 +-
 dir.c                                   |   2 +-
 entry.c                                 |   3 +-
 fast-import.c                           |   7 +-
 fsck.c                                  |   9 +-
 grep.c                                  |   3 +-
 http-push.c                             |   3 +-
 log-tree.c                              |   3 +-
 mailmap.c                               |   2 +-
 match-trees.c                           |   4 +-
 merge-blobs.c                           |   6 +-
 merge-recursive.c                       |  13 +--
 negotiator/default.c                    |   6 +-
 negotiator/skipping.c                   |   2 +-
 notes-cache.c                           |   5 +-
 notes-merge.c                           |   4 +-
 notes-utils.c                           |   2 +-
 notes.c                                 |  10 +-
 object-store.h                          |  13 ++-
 object.c                                |   2 +-
 packfile.c                              |   5 +-
 packfile.h                              |   2 +-
 pretty.c                                |  33 ++++---
 pretty.h                                |   7 +-
 read-cache.c                            |   5 +-
 remote-testsvn.c                        |   4 +-
 remote.c                                |   2 +-
 rerere.c                                |   5 +-
 revision.c                              |  12 +--
 sequencer.c                             |  55 ++++++-----
 sha1-file.c                             |  22 +++--
 sha1-name.c                             |   9 +-
 shallow.c                               |   4 +-
 streaming.c                             |   2 +-
 submodule-config.c                      |   3 +-
 submodule.c                             |  51 +++++++---
 t/helper/test-revision-walking.c        |   3 +-
 tag.c                                   |   5 +-
 tree-walk.c                             |   6 +-
 tree.c                                  |   5 +-
 walker.c                                |   2 +-
 xdiff-interface.c                       |   2 +-
 79 files changed, 571 insertions(+), 278 deletions(-)
 create mode 100644 contrib/coccinelle/the_repository.cocci

-- 
2.19.0


^ permalink raw reply	[relevance 6%]

* [PATCH 17/19] submodule: use submodule repos for object lookup
  2018-10-11 21:17 [RFC PATCH 00/19] Bring more repository handles into our code base Stefan Beller
@ 2018-10-11 21:17 ` Stefan Beller
      [irrelevant]   ` <20181011224052.191281-1-jonathantanmy@google.com>
  2018-10-11 21:17 ` [PATCH 18/19] submodule: don't add submodule as odb for push Stefan Beller
                   ` (2 subsequent siblings)
  3 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-11 21:17 UTC (permalink / raw)
  To: git; +Cc: jonathantanmy, Stefan Beller

This converts the 'show_submodule_header' function to use
the repository API properly, such that the submodule objects
are not added to the main object store.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 48 ++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/submodule.c b/submodule.c
index 442229bb49..5e1a6c0b7c 100644
--- a/submodule.c
+++ b/submodule.c
@@ -443,7 +443,7 @@ static int prepare_submodule_summary(struct rev_info *rev, const char *path,
 	return prepare_revision_walk(rev);
 }
 
-static void print_submodule_summary(struct rev_info *rev, struct diff_options *o)
+static void print_submodule_summary(struct repository *r, struct rev_info *rev, struct diff_options *o)
 {
 	static const char format[] = "  %m %s";
 	struct strbuf sb = STRBUF_INIT;
@@ -454,7 +454,8 @@ static void print_submodule_summary(struct rev_info *rev, struct diff_options *o
 		ctx.date_mode = rev->date_mode;
 		ctx.output_encoding = get_log_output_encoding();
 		strbuf_setlen(&sb, 0);
-		format_commit_message(commit, format, &sb, &ctx);
+		repo_format_commit_message(r, commit, format, &sb,
+				      &ctx);
 		strbuf_addch(&sb, '\n');
 		if (commit->object.flags & SYMMETRIC_LEFT)
 			diff_emit_submodule_del(o, sb.buf);
@@ -481,12 +482,37 @@ void prepare_submodule_repo_env(struct argv_array *out)
 			 DEFAULT_GIT_DIR_ENVIRONMENT);
 }
 
+/*
+ * Initialize 'out' based on the provided submodule path.
+ *
+ * Unlike repo_submodule_init, this tolerates submodules not present
+ * in .gitmodules. NEEDSWORK: The repo_submodule_init behavior is
+ * preferrable. This function exists only to preserve historical behavior.
+ *
+ * Returns 0 on success, -1 when the submodule is not present.
+ */
+static int open_submodule(struct repository *out, const char *path)
+{
+	struct strbuf sb = STRBUF_INIT;
+
+	if (submodule_to_gitdir(&sb, path) || repo_init(out, sb.buf, NULL)) {
+		strbuf_release(&sb);
+		return -1;
+	}
+
+	out->submodule_prefix = xstrdup(path);
+
+	strbuf_release(&sb);
+	return 0;
+}
+
 /* Helper function to display the submodule header line prior to the full
  * summary output. If it can locate the submodule objects directory it will
  * attempt to lookup both the left and right commits and put them into the
  * left and right pointers.
  */
-static void show_submodule_header(struct diff_options *o, const char *path,
+static void show_submodule_header(struct diff_options *o, struct repository *sub,
+		const char *path,
 		struct object_id *one, struct object_id *two,
 		unsigned dirty_submodule,
 		struct commit **left, struct commit **right,
@@ -507,7 +533,7 @@ static void show_submodule_header(struct diff_options *o, const char *path,
 	else if (is_null_oid(two))
 		message = "(submodule deleted)";
 
-	if (add_submodule_odb(path)) {
+	if (open_submodule(sub, path) < 0) {
 		if (!message)
 			message = "(commits not present)";
 		goto output_header;
@@ -517,8 +543,8 @@ static void show_submodule_header(struct diff_options *o, const char *path,
 	 * Attempt to lookup the commit references, and determine if this is
 	 * a fast forward or fast backwards update.
 	 */
-	*left = lookup_commit_reference(the_repository, one);
-	*right = lookup_commit_reference(the_repository, two);
+	*left = lookup_commit_reference(sub, one);
+	*right = lookup_commit_reference(sub, two);
 
 	/*
 	 * Warn about missing commits in the submodule project, but only if
@@ -528,7 +554,7 @@ static void show_submodule_header(struct diff_options *o, const char *path,
 	     (!is_null_oid(two) && !*right))
 		message = "(commits not present)";
 
-	*merge_bases = get_merge_bases(*left, *right);
+	*merge_bases = repo_get_merge_bases(sub, *left, *right);
 	if (*merge_bases) {
 		if ((*merge_bases)->item == *left)
 			fast_forward = 1;
@@ -562,8 +588,9 @@ void show_submodule_summary(struct diff_options *o, const char *path,
 	struct rev_info rev;
 	struct commit *left = NULL, *right = NULL;
 	struct commit_list *merge_bases = NULL;
+	struct repository sub;
 
-	show_submodule_header(o, path, one, two, dirty_submodule,
+	show_submodule_header(o, &sub, path, one, two, dirty_submodule,
 			      &left, &right, &merge_bases);
 
 	/*
@@ -580,7 +607,7 @@ void show_submodule_summary(struct diff_options *o, const char *path,
 		goto out;
 	}
 
-	print_submodule_summary(&rev, o);
+	print_submodule_summary(&sub, &rev, o);
 
 out:
 	if (merge_bases)
@@ -598,8 +625,9 @@ void show_submodule_inline_diff(struct diff_options *o, const char *path,
 	struct commit_list *merge_bases = NULL;
 	struct child_process cp = CHILD_PROCESS_INIT;
 	struct strbuf sb = STRBUF_INIT;
+	struct repository sub;
 
-	show_submodule_header(o, path, one, two, dirty_submodule,
+	show_submodule_header(o, &sub, path, one, two, dirty_submodule,
 			      &left, &right, &merge_bases);
 
 	/* We need a valid left and right commit to display a difference */
-- 
2.19.0


^ permalink raw reply	[relevance 18%]

* [PATCH 18/19] submodule: don't add submodule as odb for push
  2018-10-11 21:17 [RFC PATCH 00/19] Bring more repository handles into our code base Stefan Beller
  2018-10-11 21:17 ` [PATCH 17/19] submodule: use submodule repos for object lookup Stefan Beller
@ 2018-10-11 21:17 ` Stefan Beller
      [irrelevant]   ` <20181011230028.200488-1-jonathantanmy@google.com>
  2018-10-11 21:17 ` [PATCH 19/19] Apply semantic patches from previous patches Stefan Beller
  2018-10-11 23:31 ` [RFC PATCH 00/19] Bring more repository handles into our code base Junio C Hamano
  3 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-11 21:17 UTC (permalink / raw)
  To: git; +Cc: jonathantanmy, Stefan Beller

The submodule was added as an alternative in eb21c732d6 (push: teach
--recurse-submodules the on-demand option, 2012-03-29), but was
not explained, why.

In similar code, submodule_has_commits, the submodule is added as an
alternative to perform a quick check if we need to dive into the submodule.

However in push_submodule
(a) for_each_remote_ref_submodule will also provide the quick check and
(b) after that we don't need to have submodule objects around, as all
    further code is to spawn a separate process.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/submodule.c b/submodule.c
index 5e1a6c0b7c..f70d75ef45 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1006,9 +1006,6 @@ static int push_submodule(const char *path,
 			  const struct string_list *push_options,
 			  int dry_run)
 {
-	if (add_submodule_odb(path))
-		return 1;
-
 	if (for_each_remote_ref_submodule(path, has_remote, NULL) > 0) {
 		struct child_process cp = CHILD_PROCESS_INIT;
 		argv_array_push(&cp.args, "push");
-- 
2.19.0


^ permalink raw reply	[relevance 15%]

* [PATCH 19/19] Apply semantic patches from previous patches
  2018-10-11 21:17 [RFC PATCH 00/19] Bring more repository handles into our code base Stefan Beller
  2018-10-11 21:17 ` [PATCH 17/19] submodule: use submodule repos for object lookup Stefan Beller
  2018-10-11 21:17 ` [PATCH 18/19] submodule: don't add submodule as odb for push Stefan Beller
@ 2018-10-11 21:17 ` Stefan Beller
  2018-10-11 23:31 ` [RFC PATCH 00/19] Bring more repository handles into our code base Junio C Hamano
  3 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-11 21:17 UTC (permalink / raw)
  To: git; +Cc: jonathantanmy, Stefan Beller

Previous commits added some cocci rules, but did not patch the whole tree,
as to not dilute the focus for reviewing the previous patches.

This patch is generated by 'make coccicheck' and applying the resulting
diff, which was white space damaged (>8 spaces after a tab) in blame.c,
which has been fixed.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 apply.c                          |  6 ++--
 archive.c                        |  5 +--
 bisect.c                         |  5 +--
 blame.c                          | 15 +++++----
 builtin/am.c                     |  2 +-
 builtin/blame.c                  |  4 +--
 builtin/cat-file.c               | 21 +++++++-----
 builtin/checkout.c               |  4 +--
 builtin/commit.c                 | 13 +++++---
 builtin/describe.c               |  4 +--
 builtin/difftool.c               |  3 +-
 builtin/fast-export.c            |  7 ++--
 builtin/fmt-merge-msg.c          |  8 +++--
 builtin/grep.c                   |  2 +-
 builtin/index-pack.c             |  8 +++--
 builtin/log.c                    |  4 +--
 builtin/merge-base.c             |  2 +-
 builtin/merge-tree.c             |  9 ++++--
 builtin/mktag.c                  |  3 +-
 builtin/name-rev.c               |  2 +-
 builtin/notes.c                  | 12 ++++---
 builtin/pack-objects.c           | 22 +++++++++----
 builtin/reflog.c                 |  5 +--
 builtin/replace.c                |  2 +-
 builtin/shortlog.c               |  5 +--
 builtin/show-branch.c            |  4 +--
 builtin/tag.c                    |  4 +--
 builtin/unpack-file.c            |  2 +-
 builtin/unpack-objects.c         |  3 +-
 builtin/verify-commit.c          |  2 +-
 bundle.c                         |  2 +-
 combine-diff.c                   |  2 +-
 commit-graph.c                   |  8 ++---
 commit.c                         | 15 +++++----
 config.c                         |  2 +-
 diff.c                           |  3 +-
 dir.c                            |  2 +-
 entry.c                          |  3 +-
 fast-import.c                    |  7 ++--
 fsck.c                           |  9 +++---
 grep.c                           |  3 +-
 http-push.c                      |  3 +-
 log-tree.c                       |  3 +-
 mailmap.c                        |  2 +-
 match-trees.c                    |  4 +--
 merge-blobs.c                    |  6 ++--
 merge-recursive.c                | 13 ++++----
 negotiator/default.c             |  6 ++--
 negotiator/skipping.c            |  2 +-
 notes-cache.c                    |  5 +--
 notes-merge.c                    |  4 +--
 notes-utils.c                    |  2 +-
 notes.c                          | 10 +++---
 pretty.c                         |  5 +--
 read-cache.c                     |  5 +--
 remote-testsvn.c                 |  4 +--
 remote.c                         |  2 +-
 rerere.c                         |  5 +--
 revision.c                       | 12 +++----
 sequencer.c                      | 55 +++++++++++++++++++-------------
 sha1-file.c                      |  3 +-
 sha1-name.c                      |  9 +++---
 shallow.c                        |  4 +--
 submodule-config.c               |  3 +-
 t/helper/test-revision-walking.c |  3 +-
 tag.c                            |  5 +--
 tree-walk.c                      |  6 ++--
 tree.c                           |  5 +--
 walker.c                         |  2 +-
 xdiff-interface.c                |  2 +-
 70 files changed, 254 insertions(+), 180 deletions(-)

diff --git a/apply.c b/apply.c
index fdae1d423b..5ac284b7e8 100644
--- a/apply.c
+++ b/apply.c
@@ -3187,7 +3187,8 @@ static int apply_binary(struct apply_state *state,
 		unsigned long size;
 		char *result;
 
-		result = read_object_file(&oid, &type, &size);
+		result = repo_read_object_file(the_repository, &oid, &type,
+					       &size);
 		if (!result)
 			return error(_("the necessary postimage %s for "
 				       "'%s' cannot be read"),
@@ -3249,7 +3250,8 @@ static int read_blob_object(struct strbuf *buf, const struct object_id *oid, uns
 		unsigned long sz;
 		char *result;
 
-		result = read_object_file(oid, &type, &sz);
+		result = repo_read_object_file(the_repository, oid, &type,
+					       &sz);
 		if (!result)
 			return -1;
 		/* XXX read_sha1_file NUL-terminates */
diff --git a/archive.c b/archive.c
index 994495af05..70e5eed535 100644
--- a/archive.c
+++ b/archive.c
@@ -55,7 +55,8 @@ static void format_subst(const struct commit *commit,
 		strbuf_add(&fmt, b + 8, c - b - 8);
 
 		strbuf_add(buf, src, b - src);
-		format_commit_message(commit, fmt.buf, buf, &ctx);
+		repo_format_commit_message(the_repository, commit, fmt.buf,
+					   buf, &ctx);
 		len -= c + 1 - src;
 		src  = c + 1;
 	}
@@ -73,7 +74,7 @@ void *object_file_to_archive(const struct archiver_args *args,
 	const struct commit *commit = args->convert ? args->commit : NULL;
 
 	path += args->baselen;
-	buffer = read_object_file(oid, type, sizep);
+	buffer = repo_read_object_file(the_repository, oid, type, sizep);
 	if (buffer && S_ISREG(mode)) {
 		struct strbuf buf = STRBUF_INIT;
 		size_t size = 0;
diff --git a/bisect.c b/bisect.c
index 6ae5e5b49e..ae92367738 100644
--- a/bisect.c
+++ b/bisect.c
@@ -136,8 +136,9 @@ static void show_list(const char *debug, int counted, int nr,
 		unsigned flags = commit->object.flags;
 		enum object_type type;
 		unsigned long size;
-		char *buf = read_object_file(&commit->object.oid, &type,
-					     &size);
+		char *buf = repo_read_object_file(the_repository,
+						  &commit->object.oid, &type,
+						  &size);
 		const char *subject_start;
 		int subject_len;
 
diff --git a/blame.c b/blame.c
index c229a10c0e..fc2764d036 100644
--- a/blame.c
+++ b/blame.c
@@ -322,8 +322,9 @@ static void fill_origin_blob(struct diff_options *opt,
 				    &o->blob_oid, 1, &file->ptr, &file_size))
 			;
 		else
-			file->ptr = read_object_file(&o->blob_oid, &type,
-						     &file_size);
+			file->ptr = repo_read_object_file(the_repository,
+							  &o->blob_oid, &type,
+							  &file_size);
 		file->size = file_size;
 
 		if (!file->ptr)
@@ -1455,7 +1456,7 @@ static void pass_blame(struct blame_scoreboard *sb, struct blame_origin *origin,
 
 			if (sg_origin[i])
 				continue;
-			if (parse_commit(p))
+			if (repo_parse_commit(the_repository, p))
 				continue;
 			porigin = find(sb->repo, p, origin);
 			if (!porigin)
@@ -1594,7 +1595,7 @@ void assign_blame(struct blame_scoreboard *sb, int opt)
 		 * so hold onto it in the meantime.
 		 */
 		blame_origin_incref(suspect);
-		parse_commit(commit);
+		repo_parse_commit(the_repository, commit);
 		if (sb->reverse ||
 		    (!(commit->object.flags & UNINTERESTING) &&
 		     !(revs->max_age != -1 && commit->date < revs->max_age)))
@@ -1864,8 +1865,10 @@ void setup_scoreboard(struct blame_scoreboard *sb,
 				    &sb->final_buf_size))
 			;
 		else
-			sb->final_buf = read_object_file(&o->blob_oid, &type,
-							 &sb->final_buf_size);
+			sb->final_buf = repo_read_object_file(the_repository,
+							      &o->blob_oid,
+							      &type,
+							      &sb->final_buf_size);
 
 		if (!sb->final_buf)
 			die(_("cannot read blob %s for path %s"),
diff --git a/builtin/am.c b/builtin/am.c
index 601570dbef..3c992a93fb 100644
--- a/builtin/am.c
+++ b/builtin/am.c
@@ -1360,7 +1360,7 @@ static void get_commit_info(struct am_state *state, struct commit *commit)
 		die(_("unable to parse commit %s"), oid_to_hex(&commit->object.oid));
 	state->msg = xstrdup(msg + 2);
 	state->msg_len = strlen(state->msg);
-	unuse_commit_buffer(commit, buffer);
+	repo_unuse_commit_buffer(the_repository, commit, buffer);
 }
 
 /**
diff --git a/builtin/blame.c b/builtin/blame.c
index a443af9ee9..dcd300c0d3 100644
--- a/builtin/blame.c
+++ b/builtin/blame.c
@@ -197,7 +197,7 @@ static void get_commit_info(struct commit *commit,
 		    &ret->author_time, &ret->author_tz);
 
 	if (!detailed) {
-		unuse_commit_buffer(commit, message);
+		repo_unuse_commit_buffer(the_repository, commit, message);
 		return;
 	}
 
@@ -211,7 +211,7 @@ static void get_commit_info(struct commit *commit,
 	else
 		strbuf_addf(&ret->summary, "(%s)", oid_to_hex(&commit->object.oid));
 
-	unuse_commit_buffer(commit, message);
+	repo_unuse_commit_buffer(the_repository, commit, message);
 }
 
 /*
diff --git a/builtin/cat-file.c b/builtin/cat-file.c
index 8d97c84725..afad7b1c84 100644
--- a/builtin/cat-file.c
+++ b/builtin/cat-file.c
@@ -34,7 +34,7 @@ static int filter_object(const char *path, unsigned mode,
 {
 	enum object_type type;
 
-	*buf = read_object_file(oid, &type, size);
+	*buf = repo_read_object_file(the_repository, oid, &type, size);
 	if (!*buf)
 		return error(_("cannot read object %s '%s'"),
 			     oid_to_hex(oid), path);
@@ -133,7 +133,8 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name,
 
 		if (type == OBJ_BLOB)
 			return stream_blob_to_fd(1, &oid, NULL, 0);
-		buf = read_object_file(&oid, &type, &size);
+		buf = repo_read_object_file(the_repository, &oid, &type,
+					    &size);
 		if (!buf)
 			die("Cannot read object %s", obj_name);
 
@@ -144,8 +145,10 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name,
 		if (type_from_string(exp_type) == OBJ_BLOB) {
 			struct object_id blob_oid;
 			if (oid_object_info(the_repository, &oid, NULL) == OBJ_TAG) {
-				char *buffer = read_object_file(&oid, &type,
-								&size);
+				char *buffer = repo_read_object_file(the_repository,
+								     &oid,
+								     &type,
+								     &size);
 				const char *target;
 				if (!skip_prefix(buffer, "object ", &target) ||
 				    get_oid_hex(target, &blob_oid))
@@ -309,9 +312,10 @@ static void print_object_or_die(struct batch_options *opt, struct expand_data *d
 				if (!textconv_object(the_repository,
 						     data->rest, 0100644, oid,
 						     1, &contents, &size))
-					contents = read_object_file(oid,
-								    &type,
-								    &size);
+					contents = repo_read_object_file(the_repository,
+									 oid,
+									 &type,
+									 &size);
 				if (!contents)
 					die("could not convert '%s' %s",
 					    oid_to_hex(oid), data->rest);
@@ -327,7 +331,8 @@ static void print_object_or_die(struct batch_options *opt, struct expand_data *d
 		unsigned long size;
 		void *contents;
 
-		contents = read_object_file(oid, &type, &size);
+		contents = repo_read_object_file(the_repository, oid, &type,
+						 &size);
 		if (!contents)
 			die("object %s disappeared", oid_to_hex(oid));
 		if (type != data->type)
diff --git a/builtin/checkout.c b/builtin/checkout.c
index ae28478ff8..0be05e0665 100644
--- a/builtin/checkout.c
+++ b/builtin/checkout.c
@@ -404,7 +404,7 @@ static void describe_detached_head(const char *msg, struct commit *commit)
 {
 	struct strbuf sb = STRBUF_INIT;
 
-	if (!parse_commit(commit))
+	if (!repo_parse_commit(the_repository, commit))
 		pp_commit_easy(CMIT_FMT_ONELINE, commit, &sb);
 	if (print_sha1_ellipsis()) {
 		fprintf(stderr, "%s %s... %s\n", msg,
@@ -732,7 +732,7 @@ static void describe_one_orphan(struct strbuf *sb, struct commit *commit)
 	strbuf_addstr(sb, "  ");
 	strbuf_add_unique_abbrev(sb, &commit->object.oid, DEFAULT_ABBREV);
 	strbuf_addch(sb, ' ');
-	if (!parse_commit(commit))
+	if (!repo_parse_commit(the_repository, commit))
 		pp_commit_easy(CMIT_FMT_ONELINE, commit, sb);
 	strbuf_addch(sb, '\n');
 }
diff --git a/builtin/commit.c b/builtin/commit.c
index 9d8ce6cb3b..4c07dc5681 100644
--- a/builtin/commit.c
+++ b/builtin/commit.c
@@ -680,8 +680,9 @@ static int prepare_to_commit(const char *index_file, const char *prefix,
 			if (!c)
 				die(_("could not lookup commit %s"), squash_message);
 			ctx.output_encoding = get_commit_output_encoding();
-			format_commit_message(c, "squash! %s\n\n", &sb,
-					      &ctx);
+			repo_format_commit_message(the_repository, c,
+						   "squash! %s\n\n", &sb,
+						   &ctx);
 		}
 	}
 
@@ -713,8 +714,9 @@ static int prepare_to_commit(const char *index_file, const char *prefix,
 		if (!commit)
 			die(_("could not lookup commit %s"), fixup_message);
 		ctx.output_encoding = get_commit_output_encoding();
-		format_commit_message(commit, "fixup! %s\n\n",
-				      &sb, &ctx);
+		repo_format_commit_message(the_repository, commit,
+					   "fixup! %s\n\n",
+					   &sb, &ctx);
 		if (have_option_m)
 			strbuf_addbuf(&sb, &message);
 		hook_arg1 = "message";
@@ -997,7 +999,8 @@ static const char *find_author_by_nickname(const char *name)
 		struct pretty_print_context ctx = {0};
 		ctx.date_mode.type = DATE_NORMAL;
 		strbuf_release(&buf);
-		format_commit_message(commit, "%aN <%aE>", &buf, &ctx);
+		repo_format_commit_message(the_repository, commit,
+					   "%aN <%aE>", &buf, &ctx);
 		clear_mailmap(&mailmap);
 		return strbuf_detach(&buf, NULL);
 	}
diff --git a/builtin/describe.c b/builtin/describe.c
index 1fde0563fe..290218c0d9 100644
--- a/builtin/describe.c
+++ b/builtin/describe.c
@@ -254,7 +254,7 @@ static unsigned long finish_depth_computation(
 			best->depth++;
 		while (parents) {
 			struct commit *p = parents->item;
-			parse_commit(p);
+			repo_parse_commit(the_repository, p);
 			if (!(p->object.flags & SEEN))
 				commit_list_insert_by_date(p, list);
 			p->object.flags |= c->object.flags;
@@ -381,7 +381,7 @@ static void describe_commit(struct object_id *oid, struct strbuf *dst)
 		}
 		while (parents) {
 			struct commit *p = parents->item;
-			parse_commit(p);
+			repo_parse_commit(the_repository, p);
 			if (!(p->object.flags & SEEN))
 				commit_list_insert_by_date(p, &list);
 			p->object.flags |= c->object.flags;
diff --git a/builtin/difftool.c b/builtin/difftool.c
index e7023e3adf..23720d4d64 100644
--- a/builtin/difftool.c
+++ b/builtin/difftool.c
@@ -307,7 +307,8 @@ static char *get_symlink(const struct object_id *oid, const char *path)
 	} else {
 		enum object_type type;
 		unsigned long size;
-		data = read_object_file(oid, &type, &size);
+		data = repo_read_object_file(the_repository, oid, &type,
+					     &size);
 		if (!data)
 			die(_("could not read object %s for symlink %s"),
 				oid_to_hex(oid), path);
diff --git a/builtin/fast-export.c b/builtin/fast-export.c
index 9bd4a95a47..9e65697524 100644
--- a/builtin/fast-export.c
+++ b/builtin/fast-export.c
@@ -239,7 +239,7 @@ static void export_blob(const struct object_id *oid)
 		object = (struct object *)lookup_blob(the_repository, oid);
 		eaten = 0;
 	} else {
-		buf = read_object_file(oid, &type, &size);
+		buf = repo_read_object_file(the_repository, oid, &type, &size);
 		if (!buf)
 			die("could not read blob %s", oid_to_hex(oid));
 		if (check_object_signature(oid, buf, size, type_name(type)) < 0)
@@ -617,7 +617,7 @@ static void handle_commit(struct commit *commit, struct rev_info *rev,
 			  ? strlen(message) : 0),
 	       reencoded ? reencoded : message ? message : "");
 	free(reencoded);
-	unuse_commit_buffer(commit, commit_buffer);
+	repo_unuse_commit_buffer(the_repository, commit, commit_buffer);
 
 	for (i = 0, p = commit->parents; p; p = p->next) {
 		int mark = get_object_mark(&p->item->object);
@@ -689,7 +689,8 @@ static void handle_tag(const char *name, struct tag *tag)
 		return;
 	}
 
-	buf = read_object_file(&tag->object.oid, &type, &size);
+	buf = repo_read_object_file(the_repository, &tag->object.oid, &type,
+				    &size);
 	if (!buf)
 		die("could not read tag %s", oid_to_hex(&tag->object.oid));
 	message = memmem(buf, size, "\n\n", 2);
diff --git a/builtin/fmt-merge-msg.c b/builtin/fmt-merge-msg.c
index 1adc84ed87..4248e3982d 100644
--- a/builtin/fmt-merge-msg.c
+++ b/builtin/fmt-merge-msg.c
@@ -264,7 +264,7 @@ static void record_person(int which, struct string_list *people,
 {
 	const char *buffer = get_commit_buffer(commit, NULL);
 	record_person_from_buf(which, people, buffer);
-	unuse_commit_buffer(commit, buffer);
+	repo_unuse_commit_buffer(the_repository, commit, buffer);
 }
 
 static int cmp_string_list_util_as_integral(const void *a_, const void *b_)
@@ -375,7 +375,8 @@ static void shortlog(const char *name,
 		if (subjects.nr > limit)
 			continue;
 
-		format_commit_message(commit, "%s", &sb, &ctx);
+		repo_format_commit_message(the_repository, commit, "%s", &sb,
+					   &ctx);
 		strbuf_ltrim(&sb);
 
 		if (!sb.len)
@@ -493,7 +494,8 @@ static void fmt_merge_msg_sigs(struct strbuf *out)
 		struct object_id *oid = origins.items[i].util;
 		enum object_type type;
 		unsigned long size, len;
-		char *buf = read_object_file(oid, &type, &size);
+		char *buf = repo_read_object_file(the_repository, oid, &type,
+						  &size);
 		struct strbuf sig = STRBUF_INIT;
 
 		if (!buf || type != OBJ_TAG)
diff --git a/builtin/grep.c b/builtin/grep.c
index 0c3527242e..06f7711330 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -308,7 +308,7 @@ static void *lock_and_read_oid_file(const struct object_id *oid, enum object_typ
 	void *data;
 
 	grep_read_lock();
-	data = read_object_file(oid, type, size);
+	data = repo_read_object_file(the_repository, oid, type, size);
 	grep_read_unlock();
 	return data;
 }
diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index 9582ead950..887c44e111 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -817,7 +817,8 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,
 			die(_("cannot read existing object info %s"), oid_to_hex(oid));
 		if (has_type != type || has_size != size)
 			die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(oid));
-		has_data = read_object_file(oid, &has_type, &has_size);
+		has_data = repo_read_object_file(the_repository, oid,
+						 &has_type, &has_size);
 		read_unlock();
 		if (!data)
 			data = new_data = get_data_from_pack(obj_entry);
@@ -1379,8 +1380,9 @@ static void fix_unresolved_deltas(struct hashfile *f)
 
 		if (objects[d->obj_no].real_type != OBJ_REF_DELTA)
 			continue;
-		base_obj->data = read_object_file(&d->oid, &type,
-						  &base_obj->size);
+		base_obj->data = repo_read_object_file(the_repository,
+						       &d->oid, &type,
+						       &base_obj->size);
 		if (!base_obj->data)
 			continue;
 
diff --git a/builtin/log.c b/builtin/log.c
index 717d20e115..d4709ec63d 100644
--- a/builtin/log.c
+++ b/builtin/log.c
@@ -525,7 +525,7 @@ static int show_tag_object(const struct object_id *oid, struct rev_info *rev)
 {
 	unsigned long size;
 	enum object_type type;
-	char *buf = read_object_file(oid, &type, &size);
+	char *buf = repo_read_object_file(the_repository, oid, &type, &size);
 	int offset = 0;
 
 	if (!buf)
@@ -1032,7 +1032,7 @@ static void make_cover_letter(struct rev_info *rev, int use_stdout,
 		const char *buf = get_commit_buffer(list[i], NULL);
 		if (has_non_ascii(buf))
 			need_8bit_cte = 1;
-		unuse_commit_buffer(list[i], buf);
+		repo_unuse_commit_buffer(the_repository, list[i], buf);
 	}
 
 	if (!branch_name)
diff --git a/builtin/merge-base.c b/builtin/merge-base.c
index 08d91b1f0c..82567bf79c 100644
--- a/builtin/merge-base.c
+++ b/builtin/merge-base.c
@@ -127,7 +127,7 @@ static void add_one_commit(struct object_id *oid, struct rev_collect *revs)
 	commit = lookup_commit(the_repository, oid);
 	if (!commit ||
 	    (commit->object.flags & TMP_MARK) ||
-	    parse_commit(commit))
+	    repo_parse_commit(the_repository, commit))
 		return;
 
 	ALLOC_GROW(revs->commit, revs->nr + 1, revs->alloc);
diff --git a/builtin/merge-tree.c b/builtin/merge-tree.c
index f32941fdab..99403f70f4 100644
--- a/builtin/merge-tree.c
+++ b/builtin/merge-tree.c
@@ -62,7 +62,9 @@ static void *result(struct merge_list *entry, unsigned long *size)
 	const char *path = entry->path;
 
 	if (!entry->stage)
-		return read_object_file(&entry->blob->object.oid, &type, size);
+		return repo_read_object_file(the_repository,
+					     &entry->blob->object.oid, &type,
+					     size);
 	base = NULL;
 	if (entry->stage == 1) {
 		base = entry->blob;
@@ -84,8 +86,9 @@ static void *origin(struct merge_list *entry, unsigned long *size)
 	enum object_type type;
 	while (entry) {
 		if (entry->stage == 2)
-			return read_object_file(&entry->blob->object.oid,
-						&type, size);
+			return repo_read_object_file(the_repository,
+						     &entry->blob->object.oid,
+						     &type, size);
 		entry = entry->link;
 	}
 	return NULL;
diff --git a/builtin/mktag.c b/builtin/mktag.c
index 6fb7dc8578..be4366b03f 100644
--- a/builtin/mktag.c
+++ b/builtin/mktag.c
@@ -25,7 +25,8 @@ static int verify_object(const struct object_id *oid, const char *expected_type)
 	int ret = -1;
 	enum object_type type;
 	unsigned long size;
-	void *buffer = read_object_file(oid, &type, &size);
+	void *buffer = repo_read_object_file(the_repository, oid, &type,
+					     &size);
 	const struct object_id *repl = lookup_replace_object(the_repository, oid);
 
 	if (buffer) {
diff --git a/builtin/name-rev.c b/builtin/name-rev.c
index f1cb45c227..41f2de37e4 100644
--- a/builtin/name-rev.c
+++ b/builtin/name-rev.c
@@ -87,7 +87,7 @@ static void name_rev(struct commit *commit,
 	int parent_number = 1;
 	char *to_free = NULL;
 
-	parse_commit(commit);
+	repo_parse_commit(the_repository, commit);
 
 	if (commit->date < cutoff)
 		return;
diff --git a/builtin/notes.c b/builtin/notes.c
index c05cd004ab..7ce3e50b45 100644
--- a/builtin/notes.c
+++ b/builtin/notes.c
@@ -124,7 +124,7 @@ static void copy_obj_to_fd(int fd, const struct object_id *oid)
 {
 	unsigned long size;
 	enum object_type type;
-	char *buf = read_object_file(oid, &type, &size);
+	char *buf = repo_read_object_file(the_repository, oid, &type, &size);
 	if (buf) {
 		if (size)
 			write_or_die(fd, buf, size);
@@ -255,7 +255,7 @@ static int parse_reuse_arg(const struct option *opt, const char *arg, int unset)
 
 	if (get_oid(arg, &object))
 		die(_("failed to resolve '%s' as a valid ref."), arg);
-	if (!(buf = read_object_file(&object, &type, &len))) {
+	if (!(buf = repo_read_object_file(the_repository, &object, &type, &len))) {
 		free(buf);
 		die(_("failed to read object '%s'."), arg);
 	}
@@ -610,7 +610,8 @@ static int append_edit(int argc, const char **argv, const char *prefix)
 		/* Append buf to previous note contents */
 		unsigned long size;
 		enum object_type type;
-		char *prev_buf = read_object_file(note, &type, &size);
+		char *prev_buf = repo_read_object_file(the_repository, note,
+						       &type, &size);
 
 		strbuf_grow(&d.buf, size + 1);
 		if (d.buf.len && prev_buf && size)
@@ -714,7 +715,7 @@ static int merge_commit(struct notes_merge_options *o)
 		die(_("failed to read ref NOTES_MERGE_PARTIAL"));
 	else if (!(partial = lookup_commit_reference(the_repository, &oid)))
 		die(_("could not find commit from NOTES_MERGE_PARTIAL."));
-	else if (parse_commit(partial))
+	else if (repo_parse_commit(the_repository, partial))
 		die(_("could not parse commit from NOTES_MERGE_PARTIAL."));
 
 	if (partial->parents)
@@ -735,7 +736,8 @@ static int merge_commit(struct notes_merge_options *o)
 
 	/* Reuse existing commit message in reflog message */
 	memset(&pretty_ctx, 0, sizeof(pretty_ctx));
-	format_commit_message(partial, "%s", &msg, &pretty_ctx);
+	repo_format_commit_message(the_repository, partial, "%s", &msg,
+				   &pretty_ctx);
 	strbuf_trim(&msg);
 	strbuf_insert(&msg, 0, "notes: ", 7);
 	update_ref(msg.buf, o->local_ref, &oid,
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index 3383ba43d0..2d55b64433 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -138,11 +138,13 @@ static void *get_delta(struct object_entry *entry)
 	void *buf, *base_buf, *delta_buf;
 	enum object_type type;
 
-	buf = read_object_file(&entry->idx.oid, &type, &size);
+	buf = repo_read_object_file(the_repository, &entry->idx.oid, &type,
+				    &size);
 	if (!buf)
 		die(_("unable to read %s"), oid_to_hex(&entry->idx.oid));
-	base_buf = read_object_file(&DELTA(entry)->idx.oid, &type,
-				    &base_size);
+	base_buf = repo_read_object_file(the_repository,
+					 &DELTA(entry)->idx.oid, &type,
+					 &base_size);
 	if (!base_buf)
 		die("unable to read %s",
 		    oid_to_hex(&DELTA(entry)->idx.oid));
@@ -292,7 +294,9 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
 		    (st = open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL)
 			buf = NULL;
 		else {
-			buf = read_object_file(&entry->idx.oid, &type, &size);
+			buf = repo_read_object_file(the_repository,
+						    &entry->idx.oid, &type,
+						    &size);
 			if (!buf)
 				die(_("unable to read %s"),
 				    oid_to_hex(&entry->idx.oid));
@@ -1218,7 +1222,7 @@ static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)
 	/* Did not find one.  Either we got a bogus request or
 	 * we need to read and perhaps cache.
 	 */
-	data = read_object_file(oid, &type, &size);
+	data = repo_read_object_file(the_repository, oid, &type, &size);
 	if (!data)
 		return NULL;
 	if (type != OBJ_TREE) {
@@ -1989,7 +1993,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
 	/* Load data if not already done */
 	if (!trg->data) {
 		read_lock();
-		trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz);
+		trg->data = repo_read_object_file(the_repository,
+						  &trg_entry->idx.oid, &type,
+						  &sz);
 		read_unlock();
 		if (!trg->data)
 			die(_("object %s cannot be read"),
@@ -2002,7 +2008,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
 	}
 	if (!src->data) {
 		read_lock();
-		src->data = read_object_file(&src_entry->idx.oid, &type, &sz);
+		src->data = repo_read_object_file(the_repository,
+						  &src_entry->idx.oid, &type,
+						  &sz);
 		read_unlock();
 		if (!src->data) {
 			if (src_entry->preferred_base) {
diff --git a/builtin/reflog.c b/builtin/reflog.c
index b5941c1ff3..53a7f0c8e5 100644
--- a/builtin/reflog.c
+++ b/builtin/reflog.c
@@ -77,7 +77,8 @@ static int tree_is_complete(const struct object_id *oid)
 	if (!tree->buffer) {
 		enum object_type type;
 		unsigned long size;
-		void *data = read_object_file(oid, &type, &size);
+		void *data = repo_read_object_file(the_repository, oid, &type,
+						   &size);
 		if (!data) {
 			tree->object.flags |= INCOMPLETE;
 			return 0;
@@ -235,7 +236,7 @@ static void mark_reachable(struct expire_reflog_policy_cb *cb)
 		struct commit *commit = pop_commit(&pending);
 		if (commit->object.flags & REACHABLE)
 			continue;
-		if (parse_commit(commit))
+		if (repo_parse_commit(the_repository, commit))
 			continue;
 		commit->object.flags |= REACHABLE;
 		if (commit->date < expire_limit) {
diff --git a/builtin/replace.c b/builtin/replace.c
index e0b16ad44b..48b44dba3e 100644
--- a/builtin/replace.c
+++ b/builtin/replace.c
@@ -449,7 +449,7 @@ static int create_graft(int argc, const char **argv, int force, int gentle)
 
 	buffer = get_commit_buffer(commit, &size);
 	strbuf_add(&buf, buffer, size);
-	unuse_commit_buffer(commit, buffer);
+	repo_unuse_commit_buffer(the_repository, commit, buffer);
 
 	if (replace_parents(&buf, argc - 1, &argv[1]) < 0) {
 		strbuf_release(&buf);
diff --git a/builtin/shortlog.c b/builtin/shortlog.c
index 88f88e97b2..d46df5af34 100644
--- a/builtin/shortlog.c
+++ b/builtin/shortlog.c
@@ -170,12 +170,13 @@ void shortlog_add_commit(struct shortlog *log, struct commit *commit)
 		(log->email ? "%cN <%cE>" : "%cN") :
 		(log->email ? "%aN <%aE>" : "%aN");
 
-	format_commit_message(commit, fmt, &author, &ctx);
+	repo_format_commit_message(the_repository, commit, fmt, &author, &ctx);
 	if (!log->summary) {
 		if (log->user_format)
 			pretty_print_commit(&ctx, commit, &oneline);
 		else
-			format_commit_message(commit, "%s", &oneline, &ctx);
+			repo_format_commit_message(the_repository, commit,
+						   "%s", &oneline, &ctx);
 	}
 
 	insert_one_record(log, author.buf, oneline.len ? oneline.buf : "<none>");
diff --git a/builtin/show-branch.c b/builtin/show-branch.c
index 363cf8509a..d7e8d6f15c 100644
--- a/builtin/show-branch.c
+++ b/builtin/show-branch.c
@@ -238,7 +238,7 @@ static void join_revs(struct commit_list **list_p,
 			parents = parents->next;
 			if ((this_flag & flags) == flags)
 				continue;
-			parse_commit(p);
+			repo_parse_commit(the_repository, p);
 			if (mark_seen(p, seen_p) && !still_interesting)
 				extra--;
 			p->object.flags |= flags;
@@ -835,7 +835,7 @@ int cmd_show_branch(int ac, const char **av, const char *prefix)
 		if (!commit)
 			die(_("cannot find commit %s (%s)"),
 			    ref_name[num_rev], oid_to_hex(&revkey));
-		parse_commit(commit);
+		repo_parse_commit(the_repository, commit);
 		mark_seen(commit, &seen);
 
 		/* rev#0 uses bit REV_SHIFT, rev#1 uses bit REV_SHIFT+1,
diff --git a/builtin/tag.c b/builtin/tag.c
index 9a19ffb49f..3aa1ea7f15 100644
--- a/builtin/tag.c
+++ b/builtin/tag.c
@@ -169,7 +169,7 @@ static void write_tag_body(int fd, const struct object_id *oid)
 	enum object_type type;
 	char *buf, *sp;
 
-	buf = read_object_file(oid, &type, &size);
+	buf = repo_read_object_file(the_repository, oid, &type, &size);
 	if (!buf)
 		return;
 	/* skip header */
@@ -305,7 +305,7 @@ static void create_reflog_msg(const struct object_id *oid, struct strbuf *sb)
 		strbuf_addstr(sb, "object of unknown type");
 		break;
 	case OBJ_COMMIT:
-		if ((buf = read_object_file(oid, &type, &size)) != NULL) {
+		if ((buf = repo_read_object_file(the_repository, oid, &type, &size)) != NULL) {
 			subject_len = find_commit_subject(buf, &subject_start);
 			strbuf_insert(sb, sb->len, subject_start, subject_len);
 		} else {
diff --git a/builtin/unpack-file.c b/builtin/unpack-file.c
index 58652229f2..df4a47bae7 100644
--- a/builtin/unpack-file.c
+++ b/builtin/unpack-file.c
@@ -10,7 +10,7 @@ static char *create_temp_file(struct object_id *oid)
 	unsigned long size;
 	int fd;
 
-	buf = read_object_file(oid, &type, &size);
+	buf = repo_read_object_file(the_repository, oid, &type, &size);
 	if (!buf || type != OBJ_BLOB)
 		die("unable to read blob object %s", oid_to_hex(oid));
 
diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c
index 30d9413b4b..1e2bc1be3a 100644
--- a/builtin/unpack-objects.c
+++ b/builtin/unpack-objects.c
@@ -425,7 +425,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 	if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
 		return;
 
-	base = read_object_file(&base_oid, &type, &base_size);
+	base = repo_read_object_file(the_repository, &base_oid, &type,
+				     &base_size);
 	if (!base) {
 		error("failed to read delta-pack base object %s",
 		      oid_to_hex(&base_oid));
diff --git a/builtin/verify-commit.c b/builtin/verify-commit.c
index 7772c07ed7..3caa6cf8e9 100644
--- a/builtin/verify-commit.c
+++ b/builtin/verify-commit.c
@@ -47,7 +47,7 @@ static int verify_commit(const char *name, unsigned flags)
 	if (get_oid(name, &oid))
 		return error("commit '%s' not found.", name);
 
-	buf = read_object_file(&oid, &type, &size);
+	buf = repo_read_object_file(the_repository, &oid, &type, &size);
 	if (!buf)
 		return error("%s: unable to read file.", name);
 	if (type != OBJ_COMMIT)
diff --git a/bundle.c b/bundle.c
index a5a71d059e..8080cf1ed1 100644
--- a/bundle.c
+++ b/bundle.c
@@ -224,7 +224,7 @@ static int is_tag_in_date_range(struct object *tag, struct rev_info *revs)
 	if (revs->max_age == -1 && revs->min_age == -1)
 		goto out;
 
-	buf = read_object_file(&tag->oid, &type, &size);
+	buf = repo_read_object_file(the_repository, &tag->oid, &type, &size);
 	if (!buf)
 		goto out;
 	line = memmem(buf, size, "\ntagger ", 8);
diff --git a/combine-diff.c b/combine-diff.c
index 41ab5b01de..5593052ae9 100644
--- a/combine-diff.c
+++ b/combine-diff.c
@@ -308,7 +308,7 @@ static char *grab_blob(struct repository *r,
 		*size = fill_textconv(r, textconv, df, &blob);
 		free_filespec(df);
 	} else {
-		blob = read_object_file(oid, &type, size);
+		blob = repo_read_object_file(the_repository, oid, &type, size);
 		if (type != OBJ_BLOB)
 			die("object '%s' is not a blob!", oid_to_hex(oid));
 	}
diff --git a/commit-graph.c b/commit-graph.c
index 8a1bec7b8a..fd2f5380e1 100644
--- a/commit-graph.c
+++ b/commit-graph.c
@@ -435,7 +435,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
 		int edge_value;
 		uint32_t packedDate[2];
 
-		parse_commit(*list);
+		repo_parse_commit(the_repository, *list);
 		hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
 
 		parent = (*list)->parents;
@@ -606,7 +606,7 @@ static void close_reachable(struct packed_oid_list *oids)
 	for (i = 0; i < oids->nr; i++) {
 		commit = lookup_commit(the_repository, &oids->list[i]);
 
-		if (commit && !parse_commit(commit))
+		if (commit && !repo_parse_commit(the_repository, commit))
 			add_missing_parents(oids, commit);
 	}
 
@@ -783,7 +783,7 @@ void write_commit_graph(const char *obj_dir,
 			continue;
 
 		commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]);
-		parse_commit(commits.list[commits.nr]);
+		repo_parse_commit(the_repository, commits.list[commits.nr]);
 
 		for (parent = commits.list[commits.nr]->parents;
 		     parent; parent = parent->next)
@@ -954,7 +954,7 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
 
 		graph_commit = lookup_commit(r, &cur_oid);
 		odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
-		if (parse_commit_internal(odb_commit, 0, 0)) {
+		if (repo_parse_commit_internal(the_repository, odb_commit, 0, 0)) {
 			graph_report("failed to parse %s from object database",
 				     oid_to_hex(&cur_oid));
 			continue;
diff --git a/commit.c b/commit.c
index 526b33758d..e86241b802 100644
--- a/commit.c
+++ b/commit.c
@@ -70,7 +70,7 @@ struct commit *lookup_commit_reference_by_name(const char *name)
 	if (get_oid_committish(name, &oid))
 		return NULL;
 	commit = lookup_commit_reference(the_repository, &oid);
-	if (parse_commit(commit))
+	if (repo_parse_commit(the_repository, commit))
 		return NULL;
 	return commit;
 }
@@ -491,7 +491,7 @@ int repo_parse_commit_gently(struct repository *r,
 
 void parse_commit_or_die(struct commit *item)
 {
-	if (parse_commit(item))
+	if (repo_parse_commit(the_repository, item))
 		die("unable to parse commit %s",
 		    item ? oid_to_hex(&item->object.oid) : "(null)");
 }
@@ -596,7 +596,7 @@ struct commit *pop_most_recent_commit(struct commit_list **list,
 
 	while (parents) {
 		struct commit *commit = parents->item;
-		if (!parse_commit(commit) && !(commit->object.flags & mark)) {
+		if (!repo_parse_commit(the_repository, commit) && !(commit->object.flags & mark)) {
 			commit->object.flags |= mark;
 			commit_list_insert_by_date(commit, list);
 		}
@@ -689,7 +689,7 @@ static void record_author_date(struct author_date_slab *author_date,
 	*(author_date_slab_at(author_date, commit)) = date;
 
 fail_exit:
-	unuse_commit_buffer(commit, buffer);
+	repo_unuse_commit_buffer(the_repository, commit, buffer);
 }
 
 static int compare_commits_by_author_date(const void *a_, const void *b_,
@@ -1302,7 +1302,7 @@ int parse_signed_commit(const struct commit *commit,
 		}
 		line = next;
 	}
-	unuse_commit_buffer(commit, buffer);
+	repo_unuse_commit_buffer(the_repository, commit, buffer);
 	return saw_signature;
 }
 
@@ -1351,7 +1351,8 @@ static void handle_signed_tag(struct commit *parent, struct commit_extra_header
 	desc = merge_remote_util(parent);
 	if (!desc || !desc->obj)
 		return;
-	buf = read_object_file(&desc->obj->oid, &type, &size);
+	buf = repo_read_object_file(the_repository, &desc->obj->oid, &type,
+				    &size);
 	if (!buf || type != OBJ_TAG)
 		goto free_return;
 	len = parse_signature(buf, size);
@@ -1429,7 +1430,7 @@ struct commit_extra_header *read_commit_extra_headers(struct commit *commit,
 	unsigned long size;
 	const char *buffer = get_commit_buffer(commit, &size);
 	extra = read_commit_extra_header_lines(buffer, size, exclude);
-	unuse_commit_buffer(commit, buffer);
+	repo_unuse_commit_buffer(the_repository, commit, buffer);
 	return extra;
 }
 
diff --git a/config.c b/config.c
index 3461993f0a..e2b72403a6 100644
--- a/config.c
+++ b/config.c
@@ -1605,7 +1605,7 @@ int git_config_from_blob_oid(config_fn_t fn,
 	unsigned long size;
 	int ret;
 
-	buf = read_object_file(oid, &type, &size);
+	buf = repo_read_object_file(the_repository, oid, &type, &size);
 	if (!buf)
 		return error(_("unable to load config blob object '%s'"), name);
 	if (type != OBJ_BLOB) {
diff --git a/diff.c b/diff.c
index c5b5e7ac41..ab16d6dcad 100644
--- a/diff.c
+++ b/diff.c
@@ -3939,7 +3939,8 @@ int diff_populate_filespec(struct repository *r,
 				return 0;
 			}
 		}
-		s->data = read_object_file(&s->oid, &type, &s->size);
+		s->data = repo_read_object_file(the_repository, &s->oid,
+						&type, &s->size);
 		if (!s->data)
 			die("unable to read %s", oid_to_hex(&s->oid));
 		s->should_free = 1;
diff --git a/dir.c b/dir.c
index aceb0d4869..8722b1e5d5 100644
--- a/dir.c
+++ b/dir.c
@@ -245,7 +245,7 @@ static int do_read_blob(const struct object_id *oid, struct oid_stat *oid_stat,
 	*size_out = 0;
 	*data_out = NULL;
 
-	data = read_object_file(oid, &type, &sz);
+	data = repo_read_object_file(the_repository, oid, &type, &sz);
 	if (!data || type != OBJ_BLOB) {
 		free(data);
 		return -1;
diff --git a/entry.c b/entry.c
index 2a2ab6c839..45723fc958 100644
--- a/entry.c
+++ b/entry.c
@@ -86,7 +86,8 @@ static int create_file(const char *path, unsigned int mode)
 static void *read_blob_entry(const struct cache_entry *ce, unsigned long *size)
 {
 	enum object_type type;
-	void *blob_data = read_object_file(&ce->oid, &type, size);
+	void *blob_data = repo_read_object_file(the_repository, &ce->oid,
+						&type, size);
 
 	if (blob_data) {
 		if (type == OBJ_BLOB)
diff --git a/fast-import.c b/fast-import.c
index 89bb0c9db3..879b1fbf56 100644
--- a/fast-import.c
+++ b/fast-import.c
@@ -1371,7 +1371,7 @@ static void load_tree(struct tree_entry *root)
 			die("Can't load tree %s", oid_to_hex(oid));
 	} else {
 		enum object_type type;
-		buf = read_object_file(oid, &type, &size);
+		buf = repo_read_object_file(the_repository, oid, &type, &size);
 		if (!buf || type != OBJ_TREE)
 			die("Can't load tree %s", oid_to_hex(oid));
 	}
@@ -2931,7 +2931,7 @@ static void cat_blob(struct object_entry *oe, struct object_id *oid)
 	char *buf;
 
 	if (!oe || oe->pack_id == MAX_PACK_ID) {
-		buf = read_object_file(oid, &type, &size);
+		buf = repo_read_object_file(the_repository, oid, &type, &size);
 	} else {
 		type = oe->type;
 		buf = gfi_unpack_entry(oe, &size);
@@ -3037,7 +3037,8 @@ static struct object_entry *dereference(struct object_entry *oe,
 		buf = gfi_unpack_entry(oe, &size);
 	} else {
 		enum object_type unused;
-		buf = read_object_file(oid, &unused, &size);
+		buf = repo_read_object_file(the_repository, oid, &unused,
+					    &size);
 	}
 	if (!buf)
 		die("Can't load object %s", oid_to_hex(oid));
diff --git a/fsck.c b/fsck.c
index a0cee0be59..0d3e8886e2 100644
--- a/fsck.c
+++ b/fsck.c
@@ -447,7 +447,7 @@ static int fsck_walk_commit(struct commit *commit, void *data, struct fsck_optio
 	int result;
 	const char *name;
 
-	if (parse_commit(commit))
+	if (repo_parse_commit(the_repository, commit))
 		return -1;
 
 	name = get_object_name(options, &commit->object);
@@ -859,7 +859,7 @@ static int fsck_commit(struct commit *commit, const char *data,
 	const char *buffer = data ?  data : get_commit_buffer(commit, &size);
 	int ret = fsck_commit_buffer(commit, buffer, size, options);
 	if (!data)
-		unuse_commit_buffer(commit, buffer);
+		repo_unuse_commit_buffer(the_repository, commit, buffer);
 	return ret;
 }
 
@@ -879,7 +879,8 @@ static int fsck_tag_buffer(struct tag *tag, const char *data,
 		enum object_type type;
 
 		buffer = to_free =
-			read_object_file(&tag->object.oid, &type, &size);
+			repo_read_object_file(the_repository,
+					      &tag->object.oid, &type, &size);
 		if (!buffer)
 			return report(options, &tag->object,
 				FSCK_MSG_MISSING_TAG_OBJECT,
@@ -1091,7 +1092,7 @@ int fsck_finish(struct fsck_options *options)
 			continue;
 		}
 
-		buf = read_object_file(oid, &type, &size);
+		buf = repo_read_object_file(the_repository, oid, &type, &size);
 		if (!buf) {
 			if (is_promisor_object(&blob->object.oid))
 				continue;
diff --git a/grep.c b/grep.c
index f6bd89e40b..216c38a0d5 100644
--- a/grep.c
+++ b/grep.c
@@ -2112,7 +2112,8 @@ static int grep_source_load_oid(struct grep_source *gs)
 	enum object_type type;
 
 	grep_read_lock();
-	gs->buf = read_object_file(gs->identifier, &type, &gs->size);
+	gs->buf = repo_read_object_file(the_repository, gs->identifier, &type,
+					&gs->size);
 	grep_read_unlock();
 
 	if (!gs->buf)
diff --git a/http-push.c b/http-push.c
index df504ab6a3..c6206838ed 100644
--- a/http-push.c
+++ b/http-push.c
@@ -364,7 +364,8 @@ static void start_put(struct transfer_request *request)
 	ssize_t size;
 	git_zstream stream;
 
-	unpacked = read_object_file(&request->obj->oid, &type, &len);
+	unpacked = repo_read_object_file(the_repository, &request->obj->oid,
+					 &type, &len);
 	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", type_name(type), len) + 1;
 
 	/* Set it up */
diff --git a/log-tree.c b/log-tree.c
index 7443e5fcc7..360dd0b4e6 100644
--- a/log-tree.c
+++ b/log-tree.c
@@ -332,7 +332,8 @@ void fmt_output_commit(struct strbuf *filename,
 	struct pretty_print_context ctx = {0};
 	struct strbuf subject = STRBUF_INIT;
 
-	format_commit_message(commit, "%f", &subject, &ctx);
+	repo_format_commit_message(the_repository, commit, "%f", &subject,
+				   &ctx);
 	fmt_output_subject(filename, subject.buf, info);
 	strbuf_release(&subject);
 }
diff --git a/mailmap.c b/mailmap.c
index 962fd86d6d..f3ffea0e17 100644
--- a/mailmap.c
+++ b/mailmap.c
@@ -225,7 +225,7 @@ static int read_mailmap_blob(struct string_list *map,
 	if (get_oid(name, &oid) < 0)
 		return 0;
 
-	buf = read_object_file(&oid, &type, &size);
+	buf = repo_read_object_file(the_repository, &oid, &type, &size);
 	if (!buf)
 		return error("unable to read mailmap object at %s", name);
 	if (type != OBJ_BLOB)
diff --git a/match-trees.c b/match-trees.c
index 37653308d3..3fb649ad08 100644
--- a/match-trees.c
+++ b/match-trees.c
@@ -55,7 +55,7 @@ static void *fill_tree_desc_strict(struct tree_desc *desc,
 	enum object_type type;
 	unsigned long size;
 
-	buffer = read_object_file(hash, &type, &size);
+	buffer = repo_read_object_file(the_repository, hash, &type, &size);
 	if (!buffer)
 		die("unable to read tree (%s)", oid_to_hex(hash));
 	if (type != OBJ_TREE)
@@ -190,7 +190,7 @@ static int splice_tree(const struct object_id *oid1, const char *prefix,
 	if (*subpath)
 		subpath++;
 
-	buf = read_object_file(oid1, &type, &sz);
+	buf = repo_read_object_file(the_repository, oid1, &type, &sz);
 	if (!buf)
 		die("cannot read tree %s", oid_to_hex(oid1));
 	init_tree_desc(&desc, buf, sz);
diff --git a/merge-blobs.c b/merge-blobs.c
index ee0a0e90c9..757e338715 100644
--- a/merge-blobs.c
+++ b/merge-blobs.c
@@ -12,7 +12,8 @@ static int fill_mmfile_blob(mmfile_t *f, struct blob *obj)
 	unsigned long size;
 	enum object_type type;
 
-	buf = read_object_file(&obj->object.oid, &type, &size);
+	buf = repo_read_object_file(the_repository, &obj->object.oid, &type,
+				    &size);
 	if (!buf)
 		return -1;
 	if (type != OBJ_BLOB) {
@@ -75,7 +76,8 @@ void *merge_blobs(struct index_state *istate, const char *path,
 			return NULL;
 		if (!our)
 			our = their;
-		return read_object_file(&our->object.oid, &type, size);
+		return repo_read_object_file(the_repository, &our->object.oid,
+					     &type, size);
 	}
 
 	if (fill_mmfile_blob(&f1, our) < 0)
diff --git a/merge-recursive.c b/merge-recursive.c
index 82e9f1d24a..646e265625 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -299,7 +299,7 @@ static void output_commit_title(struct merge_options *o, struct commit *commit)
 		strbuf_add_unique_abbrev(&o->obuf, &commit->object.oid,
 					 DEFAULT_ABBREV);
 		strbuf_addch(&o->obuf, ' ');
-		if (parse_commit(commit) != 0)
+		if (repo_parse_commit(the_repository, commit) != 0)
 			strbuf_addstr(&o->obuf, _("(bad commit)\n"));
 		else {
 			const char *title;
@@ -307,7 +307,7 @@ static void output_commit_title(struct merge_options *o, struct commit *commit)
 			int len = find_commit_subject(msg, &title);
 			if (len)
 				strbuf_addf(&o->obuf, "%.*s\n", len, title);
-			unuse_commit_buffer(commit, msg);
+			repo_unuse_commit_buffer(the_repository, commit, msg);
 		}
 	}
 	flush_output(o);
@@ -957,7 +957,7 @@ static int update_file_flags(struct merge_options *o,
 			goto update_index;
 		}
 
-		buf = read_object_file(oid, &type, &size);
+		buf = repo_read_object_file(the_repository, oid, &type, &size);
 		if (!buf)
 			return err(o, _("cannot read object %s '%s'"), oid_to_hex(oid), path);
 		if (type != OBJ_BLOB) {
@@ -1160,7 +1160,8 @@ static void print_commit(struct commit *commit)
 	struct strbuf sb = STRBUF_INIT;
 	struct pretty_print_context ctx = {0};
 	ctx.date_mode.type = DATE_NORMAL;
-	format_commit_message(commit, " %h: %m %s", &sb, &ctx);
+	repo_format_commit_message(the_repository, commit, " %h: %m %s", &sb,
+				   &ctx);
 	fprintf(stderr, "%s\n", sb.buf);
 	strbuf_release(&sb);
 }
@@ -2944,7 +2945,7 @@ static int read_oid_strbuf(struct merge_options *o,
 	void *buf;
 	enum object_type type;
 	unsigned long size;
-	buf = read_object_file(oid, &type, &size);
+	buf = repo_read_object_file(the_repository, oid, &type, &size);
 	if (!buf)
 		return err(o, _("cannot read object %s"), oid_to_hex(oid));
 	if (type != OBJ_BLOB) {
@@ -3522,7 +3523,7 @@ static struct commit *get_ref(const struct object_id *oid, const char *name)
 		return make_virtual_commit((struct tree*)object, name);
 	if (object->type != OBJ_COMMIT)
 		return NULL;
-	if (parse_commit((struct commit *)object))
+	if (repo_parse_commit(the_repository, (struct commit *)object))
 		return NULL;
 	return (struct commit *)object;
 }
diff --git a/negotiator/default.c b/negotiator/default.c
index 4b78f6bf36..19071a97df 100644
--- a/negotiator/default.c
+++ b/negotiator/default.c
@@ -25,7 +25,7 @@ static void rev_list_push(struct negotiation_state *ns,
 	if (!(commit->object.flags & mark)) {
 		commit->object.flags |= mark;
 
-		if (parse_commit(commit))
+		if (repo_parse_commit(the_repository, commit))
 			return;
 
 		prio_queue_put(&ns->rev_list, commit);
@@ -68,7 +68,7 @@ static void mark_common(struct negotiation_state *ns, struct commit *commit,
 			if (!ancestors_only && !(o->flags & POPPED))
 				ns->non_common_revs--;
 			if (!o->parsed && !dont_parse)
-				if (parse_commit(commit))
+				if (repo_parse_commit(the_repository, commit))
 					return;
 
 			for (parents = commit->parents;
@@ -95,7 +95,7 @@ static const struct object_id *get_rev(struct negotiation_state *ns)
 			return NULL;
 
 		commit = prio_queue_get(&ns->rev_list);
-		parse_commit(commit);
+		repo_parse_commit(the_repository, commit);
 		parents = commit->parents;
 
 		commit->object.flags |= POPPED;
diff --git a/negotiator/skipping.c b/negotiator/skipping.c
index dffbc76c49..9b76f95654 100644
--- a/negotiator/skipping.c
+++ b/negotiator/skipping.c
@@ -177,7 +177,7 @@ static const struct object_id *get_rev(struct data *data)
 		if (!(commit->object.flags & COMMON) && !entry->ttl)
 			to_send = commit;
 
-		parse_commit(commit);
+		repo_parse_commit(the_repository, commit);
 		for (p = commit->parents; p; p = p->next)
 			parent_pushed |= push_parent(data, entry, p->item);
 
diff --git a/notes-cache.c b/notes-cache.c
index d87e7ca91c..94196ef0f6 100644
--- a/notes-cache.c
+++ b/notes-cache.c
@@ -21,7 +21,8 @@ static int notes_cache_match_validity(const char *ref, const char *validity)
 		return 0;
 
 	memset(&pretty_ctx, 0, sizeof(pretty_ctx));
-	format_commit_message(commit, "%s", &msg, &pretty_ctx);
+	repo_format_commit_message(the_repository, commit, "%s", &msg,
+				   &pretty_ctx);
 	strbuf_trim(&msg);
 
 	ret = !strcmp(msg.buf, validity);
@@ -79,7 +80,7 @@ char *notes_cache_get(struct notes_cache *c, struct object_id *key_oid,
 	value_oid = get_note(&c->tree, key_oid);
 	if (!value_oid)
 		return NULL;
-	value = read_object_file(value_oid, &type, &size);
+	value = repo_read_object_file(the_repository, value_oid, &type, &size);
 
 	*outsize = size;
 	return value;
diff --git a/notes-merge.c b/notes-merge.c
index 13dd9ba158..af1f5306dd 100644
--- a/notes-merge.c
+++ b/notes-merge.c
@@ -324,7 +324,7 @@ static void write_note_to_worktree(const struct object_id *obj,
 {
 	enum object_type type;
 	unsigned long size;
-	void *buf = read_object_file(note, &type, &size);
+	void *buf = repo_read_object_file(the_repository, note, &type, &size);
 
 	if (!buf)
 		die("cannot read note %s for object %s",
@@ -723,7 +723,7 @@ int notes_merge_commit(struct notes_merge_options *o,
 
 	create_notes_commit(partial_tree, partial_commit->parents, msg,
 			    strlen(msg), result_oid);
-	unuse_commit_buffer(partial_commit, buffer);
+	repo_unuse_commit_buffer(the_repository, partial_commit, buffer);
 	if (o->verbosity >= 4)
 		printf("Finalized notes merge commit: %s\n",
 			oid_to_hex(result_oid));
diff --git a/notes-utils.c b/notes-utils.c
index 14ea03178e..f25ef54a77 100644
--- a/notes-utils.c
+++ b/notes-utils.c
@@ -22,7 +22,7 @@ void create_notes_commit(struct notes_tree *t, struct commit_list *parents,
 		if (!read_ref(t->ref, &parent_oid)) {
 			struct commit *parent = lookup_commit(the_repository,
 							      &parent_oid);
-			if (parse_commit(parent))
+			if (repo_parse_commit(the_repository, parent))
 				die("Failed to find/parse commit %s", t->ref);
 			commit_list_insert(parent, &parents);
 		}
diff --git a/notes.c b/notes.c
index 32d3dbcc1e..766d540dd1 100644
--- a/notes.c
+++ b/notes.c
@@ -797,13 +797,15 @@ int combine_notes_concatenate(struct object_id *cur_oid,
 
 	/* read in both note blob objects */
 	if (!is_null_oid(new_oid))
-		new_msg = read_object_file(new_oid, &new_type, &new_len);
+		new_msg = repo_read_object_file(the_repository, new_oid,
+						&new_type, &new_len);
 	if (!new_msg || !new_len || new_type != OBJ_BLOB) {
 		free(new_msg);
 		return 0;
 	}
 	if (!is_null_oid(cur_oid))
-		cur_msg = read_object_file(cur_oid, &cur_type, &cur_len);
+		cur_msg = repo_read_object_file(the_repository, cur_oid,
+						&cur_type, &cur_len);
 	if (!cur_msg || !cur_len || cur_type != OBJ_BLOB) {
 		free(cur_msg);
 		free(new_msg);
@@ -859,7 +861,7 @@ static int string_list_add_note_lines(struct string_list *list,
 		return 0;
 
 	/* read_sha1_file NUL-terminates */
-	data = read_object_file(oid, &t, &len);
+	data = repo_read_object_file(the_repository, oid, &t, &len);
 	if (t != OBJ_BLOB || !data || !len) {
 		free(data);
 		return t != OBJ_BLOB || !data;
@@ -1218,7 +1220,7 @@ static void format_note(struct notes_tree *t, const struct object_id *object_oid
 	if (!oid)
 		return;
 
-	if (!(msg = read_object_file(oid, &type, &msglen)) || type != OBJ_BLOB) {
+	if (!(msg = repo_read_object_file(the_repository, oid, &type, &msglen)) || type != OBJ_BLOB) {
 		free(msg);
 		return;
 	}
diff --git a/pretty.c b/pretty.c
index 948f5346cf..4ab5b44da0 100644
--- a/pretty.c
+++ b/pretty.c
@@ -1799,7 +1799,8 @@ void pretty_print_commit(struct pretty_print_context *pp,
 	int need_8bit_cte = pp->need_8bit_cte;
 
 	if (pp->fmt == CMIT_FMT_USERFORMAT) {
-		format_commit_message(commit, user_format, sb, pp);
+		repo_format_commit_message(the_repository, commit,
+					   user_format, sb, pp);
 		return;
 	}
 
@@ -1861,7 +1862,7 @@ void pretty_print_commit(struct pretty_print_context *pp,
 	if (cmit_fmt_is_mail(pp->fmt) && sb->len <= beginning_of_body)
 		strbuf_addch(sb, '\n');
 
-	unuse_commit_buffer(commit, reencoded);
+	repo_unuse_commit_buffer(the_repository, commit, reencoded);
 }
 
 void pp_commit_easy(enum cmit_fmt fmt, const struct commit *commit,
diff --git a/read-cache.c b/read-cache.c
index b707edd044..e0669b174e 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -232,7 +232,7 @@ static int ce_compare_link(const struct cache_entry *ce, size_t expected_size)
 	if (strbuf_readlink(&sb, ce->name, expected_size))
 		return -1;
 
-	buffer = read_object_file(&ce->oid, &type, &size);
+	buffer = repo_read_object_file(the_repository, &ce->oid, &type, &size);
 	if (buffer) {
 		if (size == sb.len)
 			match = memcmp(buffer, sb.buf, size);
@@ -2901,7 +2901,8 @@ void *read_blob_data_from_index(const struct index_state *istate,
 	}
 	if (pos < 0)
 		return NULL;
-	data = read_object_file(&istate->cache[pos]->oid, &type, &sz);
+	data = repo_read_object_file(the_repository, &istate->cache[pos]->oid,
+				     &type, &sz);
 	if (!data || type != OBJ_BLOB) {
 		free(data);
 		return NULL;
diff --git a/remote-testsvn.c b/remote-testsvn.c
index 3af708c5b6..c846c9ee3b 100644
--- a/remote-testsvn.c
+++ b/remote-testsvn.c
@@ -62,7 +62,7 @@ static char *read_ref_note(const struct object_id *oid)
 	init_notes(NULL, notes_ref, NULL, 0);
 	if (!(note_oid = get_note(NULL, oid)))
 		return NULL;	/* note tree not found */
-	if (!(msg = read_object_file(note_oid, &type, &msglen)))
+	if (!(msg = repo_read_object_file(the_repository, note_oid, &type, &msglen)))
 		error("Empty notes tree. %s", notes_ref);
 	else if (!msglen || type != OBJ_BLOB) {
 		error("Note contains unusable content. "
@@ -109,7 +109,7 @@ static int note2mark_cb(const struct object_id *object_oid,
 	enum object_type type;
 	struct rev_note note;
 
-	if (!(msg = read_object_file(note_oid, &type, &msglen)) ||
+	if (!(msg = repo_read_object_file(the_repository, note_oid, &type, &msglen)) ||
 			!msglen || type != OBJ_BLOB) {
 		free(msg);
 		return 1;
diff --git a/remote.c b/remote.c
index 348417f0a7..e16e8e002f 100644
--- a/remote.c
+++ b/remote.c
@@ -1822,7 +1822,7 @@ int ref_newer(const struct object_id *new_oid, const struct object_id *old_oid)
 		return 0;
 	new_commit = (struct commit *) o;
 
-	if (parse_commit(new_commit) < 0)
+	if (repo_parse_commit(the_repository, new_commit) < 0)
 		return 0;
 
 	used = list = NULL;
diff --git a/rerere.c b/rerere.c
index 8d4ac8426b..084bedfe55 100644
--- a/rerere.c
+++ b/rerere.c
@@ -988,8 +988,9 @@ static int handle_cache(struct index_state *istate, const char *path,
 			break;
 		i = ce_stage(ce) - 1;
 		if (!mmfile[i].ptr) {
-			mmfile[i].ptr = read_object_file(&ce->oid, &type,
-							 &size);
+			mmfile[i].ptr = repo_read_object_file(the_repository,
+							      &ce->oid, &type,
+							      &size);
 			mmfile[i].size = size;
 		}
 	}
diff --git a/revision.c b/revision.c
index 28366eaccf..01c11107b8 100644
--- a/revision.c
+++ b/revision.c
@@ -274,7 +274,7 @@ static struct commit *handle_commit(struct rev_info *revs,
 	if (object->type == OBJ_COMMIT) {
 		struct commit *commit = (struct commit *)object;
 
-		if (parse_commit(commit) < 0)
+		if (repo_parse_commit(the_repository, commit) < 0)
 			die("unable to parse commit %s", name);
 		if (flags & UNINTERESTING) {
 			mark_parents_uninteresting(commit);
@@ -688,7 +688,7 @@ static void try_to_simplify_commit(struct rev_info *revs, struct commit *commit)
 					ts->treesame[0] = 1;
 			}
 		}
-		if (parse_commit(p) < 0)
+		if (repo_parse_commit(the_repository, p) < 0)
 			die("cannot simplify commit %s (because of %s)",
 			    oid_to_hex(&commit->object.oid),
 			    oid_to_hex(&p->object.oid));
@@ -721,7 +721,7 @@ static void try_to_simplify_commit(struct rev_info *revs, struct commit *commit)
 				 * IOW, we pretend this parent is a
 				 * "root" commit.
 				 */
-				if (parse_commit(p) < 0)
+				if (repo_parse_commit(the_repository, p) < 0)
 					die("cannot simplify commit %s (invalid %s)",
 					    oid_to_hex(&commit->object.oid),
 					    oid_to_hex(&p->object.oid));
@@ -800,7 +800,7 @@ static int add_parents_to_list(struct rev_info *revs, struct commit *commit,
 			parent = parent->next;
 			if (p)
 				p->object.flags |= UNINTERESTING;
-			if (parse_commit_gently(p, 1) < 0)
+			if (repo_parse_commit_gently(the_repository, p, 1) < 0)
 				continue;
 			if (p->parents)
 				mark_parents_uninteresting(p);
@@ -828,7 +828,7 @@ static int add_parents_to_list(struct rev_info *revs, struct commit *commit,
 		struct commit *p = parent->item;
 		int gently = revs->ignore_missing_links ||
 			     revs->exclude_promisor_objects;
-		if (parse_commit_gently(p, gently) < 0) {
+		if (repo_parse_commit_gently(the_repository, p, gently) < 0) {
 			if (revs->exclude_promisor_objects &&
 			    is_promisor_object(&p->object.oid)) {
 				if (revs->first_parent_only)
@@ -3095,7 +3095,7 @@ static int commit_match(struct commit *commit, struct rev_info *opt)
 		retval = grep_buffer(&opt->grep_filter,
 				     (char *)message, strlen(message));
 	strbuf_release(&buf);
-	unuse_commit_buffer(commit, message);
+	repo_unuse_commit_buffer(the_repository, commit, message);
 	return opt->invert_grep ? !retval : retval;
 }
 
diff --git a/sequencer.c b/sequencer.c
index da4e727cc3..5d612831e9 100644
--- a/sequencer.c
+++ b/sequencer.c
@@ -344,7 +344,7 @@ static void free_message(struct commit *commit, struct commit_message *msg)
 	free(msg->parent_label);
 	free(msg->label);
 	free(msg->subject);
-	unuse_commit_buffer(commit, msg->message);
+	repo_unuse_commit_buffer(the_repository, commit, msg->message);
 }
 
 static void print_advice(int show_hint, struct replay_opts *opts)
@@ -604,7 +604,7 @@ static int is_index_unchanged(void)
 	 * the commit is invalid, parse_commit() will complain.  So
 	 * there is nothing for us to say here.  Just return failure.
 	 */
-	if (parse_commit(head_commit))
+	if (repo_parse_commit(the_repository, head_commit))
 		return -1;
 
 	if (!(cache_tree_oid = get_cache_tree_oid()))
@@ -1142,13 +1142,15 @@ void print_commit_summary(const char *prefix, const struct object_id *oid,
 	commit = lookup_commit(the_repository, oid);
 	if (!commit)
 		die(_("couldn't look up newly created commit"));
-	if (parse_commit(commit))
+	if (repo_parse_commit(the_repository, commit))
 		die(_("could not parse newly created commit"));
 
 	strbuf_addstr(&format, "format:%h] %s");
 
-	format_commit_message(commit, "%an <%ae>", &author_ident, &pctx);
-	format_commit_message(commit, "%cn <%ce>", &committer_ident, &pctx);
+	repo_format_commit_message(the_repository, commit, "%an <%ae>",
+				   &author_ident, &pctx);
+	repo_format_commit_message(the_repository, commit, "%cn <%ce>",
+				   &committer_ident, &pctx);
 	if (strbuf_cmp(&author_ident, &committer_ident)) {
 		strbuf_addstr(&format, "\n Author: ");
 		strbuf_addbuf_percentquote(&format, &author_ident);
@@ -1156,7 +1158,8 @@ void print_commit_summary(const char *prefix, const struct object_id *oid,
 	if (flags & SUMMARY_SHOW_AUTHOR_DATE) {
 		struct strbuf date = STRBUF_INIT;
 
-		format_commit_message(commit, "%ad", &date, &pctx);
+		repo_format_commit_message(the_repository, commit, "%ad",
+					   &date, &pctx);
 		strbuf_addstr(&format, "\n Date: ");
 		strbuf_addbuf_percentquote(&format, &date);
 		strbuf_release(&date);
@@ -1221,7 +1224,7 @@ static int parse_head(struct commit **head)
 			warning(_("HEAD %s is not a commit!"),
 				oid_to_hex(&oid));
 		}
-		if (parse_commit(current_head))
+		if (repo_parse_commit(the_repository, current_head))
 			return error(_("could not parse HEAD commit"));
 	}
 	*head = current_head;
@@ -1271,7 +1274,8 @@ static int try_to_commit(struct strbuf *msg, const char *author,
 			hook_commit = "HEAD";
 		}
 		author = amend_author = get_author(message);
-		unuse_commit_buffer(current_head, message);
+		repo_unuse_commit_buffer(the_repository, current_head,
+					 message);
 		if (!author) {
 			res = error(_("unable to parse commit author"));
 			goto out;
@@ -1381,12 +1385,12 @@ static int is_original_commit_empty(struct commit *commit)
 {
 	const struct object_id *ptree_oid;
 
-	if (parse_commit(commit))
+	if (repo_parse_commit(the_repository, commit))
 		return error(_("could not parse commit %s"),
 			     oid_to_hex(&commit->object.oid));
 	if (commit->parents) {
 		struct commit *parent = commit->parents->item;
-		if (parse_commit(parent))
+		if (repo_parse_commit(the_repository, parent))
 			return error(_("could not parse parent commit %s"),
 				oid_to_hex(&parent->object.oid));
 		ptree_oid = get_commit_tree_oid(parent);
@@ -1557,7 +1561,8 @@ static int update_squash_messages(enum todo_command command,
 		find_commit_subject(head_message, &body);
 		if (write_message(body, strlen(body),
 				  rebase_path_fixup_msg(), 0)) {
-			unuse_commit_buffer(head_commit, head_message);
+			repo_unuse_commit_buffer(the_repository, head_commit,
+						 head_message);
 			return error(_("cannot write '%s'"),
 				     rebase_path_fixup_msg());
 		}
@@ -1569,7 +1574,8 @@ static int update_squash_messages(enum todo_command command,
 		strbuf_addstr(&buf, "\n\n");
 		strbuf_addstr(&buf, body);
 
-		unuse_commit_buffer(head_commit, head_message);
+		repo_unuse_commit_buffer(the_repository, head_commit,
+					 head_message);
 	}
 
 	if (!(message = get_commit_buffer(commit, NULL)))
@@ -1592,7 +1598,7 @@ static int update_squash_messages(enum todo_command command,
 		strbuf_add_commented_lines(&buf, body, strlen(body));
 	} else
 		return error(_("unknown command: %d"), command);
-	unuse_commit_buffer(commit, message);
+	repo_unuse_commit_buffer(the_repository, commit, message);
 
 	res = write_message(buf.buf, buf.len, rebase_path_squash_msg(), 0);
 	strbuf_release(&buf);
@@ -1729,7 +1735,7 @@ static int do_pick_commit(enum todo_command command, struct commit *commit,
 		msg_file = NULL;
 		goto fast_forward_edit;
 	}
-	if (parent && parse_commit(parent) < 0)
+	if (parent && repo_parse_commit(the_repository, parent) < 0)
 		/* TRANSLATORS: The first %s will be a "todo" command like
 		   "revert" or "pick", the second %s a SHA1. */
 		return error(_("%s: cannot parse parent commit %s"),
@@ -2357,7 +2363,8 @@ static int walk_revs_populate_todo(struct todo_list *todo_list,
 		subject_len = find_commit_subject(commit_buffer, &subject);
 		strbuf_addf(&todo_list->buf, "%s %s %.*s\n", command_string,
 			short_commit_name(commit), subject_len, subject);
-		unuse_commit_buffer(commit, commit_buffer);
+		repo_unuse_commit_buffer(the_repository, commit,
+					 commit_buffer);
 	}
 
 	if (!todo_list->nr)
@@ -2616,7 +2623,8 @@ static int make_patch(struct commit *commit, struct replay_opts *opts)
 		const char *commit_buffer = get_commit_buffer(commit, NULL);
 		find_commit_subject(commit_buffer, &subject);
 		res |= write_message(subject, strlen(subject), buf.buf, 1);
-		unuse_commit_buffer(commit, commit_buffer);
+		repo_unuse_commit_buffer(the_repository, commit,
+					 commit_buffer);
 	}
 	strbuf_release(&buf);
 
@@ -3013,7 +3021,7 @@ static int do_merge(struct commit *commit, const char *arg, int arg_len,
 		find_commit_subject(message, &body);
 		len = strlen(body);
 		ret = write_message(body, len, git_path_merge_msg(the_repository), 0);
-		unuse_commit_buffer(commit, message);
+		repo_unuse_commit_buffer(the_repository, commit, message);
 		if (ret) {
 			error_errno(_("could not write '%s'"),
 				    git_path_merge_msg(the_repository));
@@ -3658,11 +3666,13 @@ static int commit_staged_changes(struct replay_opts *opts,
 				if (parse_head(&commit) ||
 				    !(p = get_commit_buffer(commit, NULL)) ||
 				    write_message(p, strlen(p), path, 0)) {
-					unuse_commit_buffer(commit, p);
+					repo_unuse_commit_buffer(the_repository,
+								 commit, p);
 					return error(_("could not write file: "
 						       "'%s'"), path);
 				}
-				unuse_commit_buffer(commit, p);
+				repo_unuse_commit_buffer(the_repository,
+							 commit, p);
 			}
 		}
 
@@ -4564,7 +4574,7 @@ int skip_unnecessary_picks(void)
 			continue;
 		if (item->command != TODO_PICK)
 			break;
-		if (parse_commit(item->commit)) {
+		if (repo_parse_commit(the_repository, item->commit)) {
 			todo_list_release(&todo_list);
 			return error(_("could not parse commit '%s'"),
 				oid_to_hex(&item->commit->object.oid));
@@ -4691,12 +4701,13 @@ int rearrange_squash(void)
 
 		*commit_todo_item_at(&commit_todo, item->commit) = item;
 
-		parse_commit(item->commit);
+		repo_parse_commit(the_repository, item->commit);
 		commit_buffer = get_commit_buffer(item->commit, NULL);
 		find_commit_subject(commit_buffer, &subject);
 		format_subject(&buf, subject, " ");
 		subject = subjects[i] = strbuf_detach(&buf, &subject_len);
-		unuse_commit_buffer(item->commit, commit_buffer);
+		repo_unuse_commit_buffer(the_repository, item->commit,
+					 commit_buffer);
 		if ((skip_prefix(subject, "fixup! ", &p) ||
 		     skip_prefix(subject, "squash! ", &p))) {
 			struct commit *commit2;
diff --git a/sha1-file.c b/sha1-file.c
index ce47524679..4f86034d25 100644
--- a/sha1-file.c
+++ b/sha1-file.c
@@ -1456,7 +1456,8 @@ void *read_object_with_reference(const struct object_id *oid,
 		int ref_length = -1;
 		const char *ref_type = NULL;
 
-		buffer = read_object_file(&actual_oid, &type, &isize);
+		buffer = repo_read_object_file(the_repository, &actual_oid,
+					       &type, &isize);
 		if (!buffer)
 			return NULL;
 		if (type == required_type) {
diff --git a/sha1-name.c b/sha1-name.c
index c9cc1318b7..71e6bb7a9a 100644
--- a/sha1-name.c
+++ b/sha1-name.c
@@ -357,7 +357,8 @@ static int show_ambiguous_object(const struct object_id *oid, void *data)
 		if (commit) {
 			struct pretty_print_context pp = {0};
 			pp.date_mode.type = DATE_SHORT;
-			format_commit_message(commit, " %ad - %s", &desc, &pp);
+			repo_format_commit_message(the_repository, commit,
+						   " %ad - %s", &desc, &pp);
 		}
 	} else if (type == OBJ_TAG) {
 		struct tag *tag = lookup_tag(the_repository, oid);
@@ -849,7 +850,7 @@ static int get_parent(const char *name, int len,
 	if (ret)
 		return ret;
 	commit = lookup_commit_reference(the_repository, &oid);
-	if (parse_commit(commit))
+	if (repo_parse_commit(the_repository, commit))
 		return -1;
 	if (!idx) {
 		oidcpy(result, &commit->object.oid);
@@ -881,7 +882,7 @@ static int get_nth_ancestor(const char *name, int len,
 		return -1;
 
 	while (generation--) {
-		if (parse_commit(commit) || !commit->parents)
+		if (repo_parse_commit(the_repository, commit) || !commit->parents)
 			return -1;
 		commit = commit->parents->item;
 	}
@@ -1152,7 +1153,7 @@ static int get_oid_oneline(const char *prefix, struct object_id *oid,
 		buf = get_commit_buffer(commit, NULL);
 		p = strstr(buf, "\n\n");
 		matches = negative ^ (p && !regexec(&regex, p + 2, 0, NULL, 0));
-		unuse_commit_buffer(commit, buf);
+		repo_unuse_commit_buffer(the_repository, commit, buf);
 
 		if (matches) {
 			oidcpy(oid, &commit->object.oid);
diff --git a/shallow.c b/shallow.c
index e656ce8b9c..124c03f003 100644
--- a/shallow.c
+++ b/shallow.c
@@ -210,7 +210,7 @@ struct commit_list *get_shallow_commits_by_rev_list(int ac, const char **av,
 		struct commit *c = p->item;
 		struct commit_list *parent;
 
-		if (parse_commit(c))
+		if (repo_parse_commit(the_repository, c))
 			die("unable to parse commit %s",
 			    oid_to_hex(&c->object.oid));
 
@@ -533,7 +533,7 @@ static void paint_down(struct paint_info *info, const struct object_id *oid,
 		if (c->object.flags & BOTTOM)
 			continue;
 
-		if (parse_commit(c))
+		if (repo_parse_commit(the_repository, c))
 			die("unable to parse commit %s",
 			    oid_to_hex(&c->object.oid));
 
diff --git a/submodule-config.c b/submodule-config.c
index fc2c41b947..4b2104979b 100644
--- a/submodule-config.c
+++ b/submodule-config.c
@@ -552,7 +552,8 @@ static const struct submodule *config_from(struct submodule_cache *cache,
 	if (submodule)
 		goto out;
 
-	config = read_object_file(&oid, &type, &config_size);
+	config = repo_read_object_file(the_repository, &oid, &type,
+				       &config_size);
 	if (!config || type != OBJ_BLOB)
 		goto out;
 
diff --git a/t/helper/test-revision-walking.c b/t/helper/test-revision-walking.c
index 625b2dbf82..a03cfcdae1 100644
--- a/t/helper/test-revision-walking.c
+++ b/t/helper/test-revision-walking.c
@@ -19,7 +19,8 @@ static void print_commit(struct commit *commit)
 	struct strbuf sb = STRBUF_INIT;
 	struct pretty_print_context ctx = {0};
 	ctx.date_mode.type = DATE_NORMAL;
-	format_commit_message(commit, " %m %s", &sb, &ctx);
+	repo_format_commit_message(the_repository, commit, " %m %s", &sb,
+				   &ctx);
 	printf("%s\n", sb.buf);
 	strbuf_release(&sb);
 }
diff --git a/tag.c b/tag.c
index 1db663d716..b5a5a4c152 100644
--- a/tag.c
+++ b/tag.c
@@ -52,7 +52,7 @@ int gpg_verify_tag(const struct object_id *oid, const char *name_to_report,
 				find_unique_abbrev(oid, DEFAULT_ABBREV),
 				type_name(type));
 
-	buf = read_object_file(oid, &type, &size);
+	buf = repo_read_object_file(the_repository, oid, &type, &size);
 	if (!buf)
 		return error("%s: unable to read file.",
 				name_to_report ?
@@ -200,7 +200,8 @@ int parse_tag(struct tag *item)
 
 	if (item->object.parsed)
 		return 0;
-	data = read_object_file(&item->object.oid, &type, &size);
+	data = repo_read_object_file(the_repository, &item->object.oid, &type,
+				     &size);
 	if (!data)
 		return error("Could not read %s",
 			     oid_to_hex(&item->object.oid));
diff --git a/tree-walk.c b/tree-walk.c
index 77b37f36fa..ec6045828b 100644
--- a/tree-walk.c
+++ b/tree-walk.c
@@ -715,8 +715,10 @@ enum follow_symlinks_result get_tree_entry_follow_symlinks(struct object_id *tre
 			 */
 			retval = DANGLING_SYMLINK;
 
-			contents = read_object_file(&current_tree_oid, &type,
-						    &link_len);
+			contents = repo_read_object_file(the_repository,
+							 &current_tree_oid,
+							 &type,
+							 &link_len);
 
 			if (!contents)
 				goto done;
diff --git a/tree.c b/tree.c
index 215d3fdc7c..e741180482 100644
--- a/tree.c
+++ b/tree.c
@@ -105,7 +105,7 @@ static int read_tree_1(struct tree *tree, struct strbuf *base,
 				    oid_to_hex(entry.oid),
 				    base->buf, entry.path);
 
-			if (parse_commit(commit))
+			if (repo_parse_commit(the_repository, commit))
 				die("Invalid commit %s in submodule path %s%s",
 				    oid_to_hex(entry.oid),
 				    base->buf, entry.path);
@@ -221,7 +221,8 @@ int parse_tree_gently(struct tree *item, int quiet_on_missing)
 
 	if (item->object.parsed)
 		return 0;
-	buffer = read_object_file(&item->object.oid, &type, &size);
+	buffer = repo_read_object_file(the_repository, &item->object.oid,
+				       &type, &size);
 	if (!buffer)
 		return quiet_on_missing ? -1 :
 			error("Could not read %s",
diff --git a/walker.c b/walker.c
index 96990d84da..3e398cd485 100644
--- a/walker.c
+++ b/walker.c
@@ -78,7 +78,7 @@ static int process_commit(struct walker *walker, struct commit *commit)
 {
 	struct commit_list *parents;
 
-	if (parse_commit(commit))
+	if (repo_parse_commit(the_repository, commit))
 		return -1;
 
 	while (complete && complete->item->date >= commit->date) {
diff --git a/xdiff-interface.c b/xdiff-interface.c
index ec6e574e4a..742553de49 100644
--- a/xdiff-interface.c
+++ b/xdiff-interface.c
@@ -192,7 +192,7 @@ void read_mmblob(mmfile_t *ptr, const struct object_id *oid)
 		return;
 	}
 
-	ptr->ptr = read_object_file(oid, &type, &size);
+	ptr->ptr = repo_read_object_file(the_repository, oid, &type, &size);
 	if (!ptr->ptr || type != OBJ_BLOB)
 		die("unable to read blob object %s", oid_to_hex(oid));
 	ptr->size = size;
-- 
2.19.0


^ permalink raw reply	[relevance 2%]

* Re: [PATCH 18/19] submodule: don't add submodule as odb for push
      [irrelevant]   ` <20181011230028.200488-1-jonathantanmy@google.com>
@ 2018-10-11 23:09     ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-11 23:09 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: git

> Do you know if pushing of submodules is exercised by any test?

t5531-deep-submodule-push.sh (all of them)
t5545-push-options.sh (ok 4 - push options and submodules)

^ permalink raw reply	[relevance 8%]

* Re: [RFC PATCH 00/19] Bring more repository handles into our code base
  2018-10-11 21:17 [RFC PATCH 00/19] Bring more repository handles into our code base Stefan Beller
                   ` (2 preceding siblings ...)
  2018-10-11 21:17 ` [PATCH 19/19] Apply semantic patches from previous patches Stefan Beller
@ 2018-10-11 23:31 ` Junio C Hamano
  3 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-10-11 23:31 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git, jonathantanmy

Stefan Beller <sbeller@google.com> writes:

> Additionally each patch adds a semantic patch, that would port from the old to
> the new function. These semantic patches are all applied in the very last patch,
> but we could omit applying the last patch if it causes too many merge conflicts
> and trickl in the semantic patches over time when there are no merge conflicts.

That's an interesting approach ;-)

> The original goal of all these refactoring series was to remove add_submodule_odb 
> in submodule.c, which was partially reached with this series.

Yup, that is a very good goalpost to keep in mind.

> remaining calls in another series, but it shows we're close to be done with these
> large refactorings as far as I am concerned.

Nice.

^ permalink raw reply	[relevance 2%]

* Git Test Coverage Report (Friday, Oct 12)
@ 2018-10-12 12:59 Derrick Stolee
  0 siblings, 0 replies; 200+ results
From: Derrick Stolee @ 2018-10-12 12:59 UTC (permalink / raw)
  To: Git List


In an effort to ensure new code is reasonably covered by the test suite, 
we now have contrib/coverage-diff.sh to combine the gcov output from 
'make coverage-test ; make coverage-report' with the output from 'git 
diff A B' to discover _new_ lines of code that are not covered.

This report takes the output of these results after running on four 
branches:

         pu: 13cafb433ca0dd31b3ea229a79cc8507aa89ee19 (tests are broken, 
so coverage not reported)

        jch: fda196ac82002ede984896861cf28a354044d1a0

       next: 76f2f5c1e34c4dbef1029e2984c2892894c444ce

     master: 04861070400d3ca9d487bd0d736ca818b9ffe371

      maint: cae598d9980661a978e2df4fb338518f7bf09572

I ran the test suite on each of these branches on an Ubuntu Linux VM, 
and I'm missing some dependencies (like apache, svn, and perforce) so 
not all tests are run.

I submit this output without comment. I'm taking a look especially at my 
own lines to see where coverage can be improved.

Thanks,

-Stolee

Uncovered code new in jch, compared to next. Build logs at [1]
--------------------------------------------------------------

builtin/archive.c
e001fd3a50 builtin/archive.c  78) die(_("git archive: expected ACK/NAK, 
got a flush packet"));
e001fd3a50 builtin/archive.c  80) if (starts_with(reader.line, "NACK "))
e001fd3a50 builtin/archive.c  81) die(_("git archive: NACK %s"), 
reader.line + 5);
e001fd3a50 builtin/archive.c  82) if (starts_with(reader.line, "ERR "))
e001fd3a50 builtin/archive.c  83) die(_("remote error: %s"), reader.line 
+ 4);
e001fd3a50 builtin/archive.c  84) die(_("git archive: protocol error"));
e001fd3a50 builtin/archive.c  89) die(_("git archive: expected a flush"));
fb19d32f05 builtin/archive.c  99) if (version != discover_version(&reader))
fb19d32f05 builtin/archive.c 100) die(_("git archive: received different 
protocol versions in subsequent requests"));

builtin/fetch.c
7bbc53a589 builtin/fetch.c  385) continue; /* can this happen??? */

builtin/rebase--interactive.c
53bbcfbde7 builtin/rebase--interactive2.c  24) return error(_("no HEAD?"));
53bbcfbde7 builtin/rebase--interactive2.c  51) return 
error_errno(_("could not create temporary %s"), path_state_dir());
53bbcfbde7 builtin/rebase--interactive2.c  57) return 
error_errno(_("could not mark as interactive"));
53bbcfbde7 builtin/rebase--interactive2.c  77) return -1;
53bbcfbde7 builtin/rebase--interactive2.c  81) return -1;
53bbcfbde7 builtin/rebase--interactive2.c  87) free(revisions);
53bbcfbde7 builtin/rebase--interactive2.c  88) free(shortrevisions);
53bbcfbde7 builtin/rebase--interactive2.c  90) return -1;
53bbcfbde7 builtin/rebase--interactive2.c  98) free(revisions);
53bbcfbde7 builtin/rebase--interactive2.c  99) free(shortrevisions);
53bbcfbde7 builtin/rebase--interactive2.c 101) return 
error_errno(_("could not open %s"), rebase_path_todo());
53bbcfbde7 builtin/rebase--interactive2.c 106) 
argv_array_push(&make_script_args, restrict_revision);
53bbcfbde7 builtin/rebase--interactive2.c 114) error(_("could not 
generate todo list"));
53bbcfbde7 builtin/rebase--interactive2.c 206) 
usage_with_options(builtin_rebase_interactive_usage, options);
53bbcfbde7 builtin/rebase--interactive2.c 220) 
warning(_("--[no-]rebase-cousins has no effect without "
0af129b2ed builtin/rebase--interactive2.c 226) die(_("a base commit must 
be provided with --upstream or --onto"));
34b47315d9 builtin/rebase--interactive.c  261) ret = rearrange_squash();
34b47315d9 builtin/rebase--interactive.c  262) break;
34b47315d9 builtin/rebase--interactive.c  264) ret = 
sequencer_add_exec_commands(cmd);
34b47315d9 builtin/rebase--interactive.c  265) break;
0af129b2ed builtin/rebase--interactive2.c 267) BUG("invalid command 
'%d'", command);

builtin/rebase.c
55071ea248   61) strbuf_trim(&out);
55071ea248   62) ret = !strcmp("true", out.buf);
55071ea248   63) strbuf_release(&out);
002ee2fe68  115) die(_("%s requires an interactive rebase"), option);
f95736288a  148) return error_errno(_("could not read '%s'"), path);
f95736288a  162) return -1;
f95736288a  167) return error(_("could not get 'onto': '%s'"), buf.buf);
f95736288a  178) return -1;
f95736288a  179) } else if (read_one(state_dir_path("head", opts), &buf))
f95736288a  180) return -1;
f95736288a  182) return error(_("invalid orig-head: '%s'"), buf.buf);
f95736288a  186) return -1;
f95736288a  188) opts->flags &= ~REBASE_NO_QUIET;
73d51ed0a5  196) opts->signoff = 1;
73d51ed0a5  197) opts->flags |= REBASE_FORCE;
ead98c111b  204) return -1;
12026a412c  219) return -1;
ba1905a5fe  227) return -1;
ba1905a5fe  235) return -1;
6defce2b02  255) return error(_("Could not read '%s'"), path);
6defce2b02  271) res = error(_("Cannot store %s"), autostash.buf);
6defce2b02  275) return res;
bc24382c2b  373) argv_array_pushf(&child.args,
bc24382c2b  375) oid_to_hex(&opts->restrict_revision->object.oid));
ac7f467fef  488) BUG("Unhandled rebase type %d", opts->type);
ac7f467fef  507) struct strbuf dir = STRBUF_INIT;
6defce2b02  509) apply_autostash(opts);
ac7f467fef  510) strbuf_addstr(&dir, opts->state_dir);
ac7f467fef  511) remove_dir_recursively(&dir, 0);
ac7f467fef  512) strbuf_release(&dir);
ac7f467fef  513) die("Nothing to do");
d4c569f8f4  540) BUG("Not a fully qualified branch: '%s'", 
switch_to_branch);
ac7f467fef  543) return -1;
ac7f467fef  547) rollback_lock_file(&lock);
ac7f467fef  548) return error(_("could not determine HEAD revision"));
ac7f467fef  565) rollback_lock_file(&lock);
ac7f467fef  566) return error(_("could not read index"));
ac7f467fef  570) error(_("failed to find tree of %s"), oid_to_hex(oid));
ac7f467fef  571) rollback_lock_file(&lock);
ac7f467fef  572) free((void *)desc.buffer);
ac7f467fef  573) return -1;
ac7f467fef  586) ret = error(_("could not write index"));
ac7f467fef  590) return ret;
ac7f467fef  606) } else if (old_orig)
ac7f467fef  607) delete_ref(NULL, "ORIG_HEAD", old_orig, 0);
bff014dac7  635) opts->flags &= !REBASE_DIFFSTAT;
9a48a615b4  669) return 1;
9a48a615b4  685) return 0;
55071ea248  893) const char *path = mkpath("%s/git-legacy-rebase",
55071ea248  896) if (sane_execvp(path, (char **)argv) < 0)
55071ea248  897) die_errno(_("could not exec %s"), path);
55071ea248  899) BUG("sane_execvp() returned???");
0eabf4b95c  915) die(_("It looks like 'git am' is in progress. Cannot 
rebase."));
f28d40d3a9  952) usage_with_options(builtin_rebase_usage,
f95736288a  972) die(_("Cannot read HEAD"));
f95736288a  976) die(_("could not read index"));
f95736288a  990) exit(1);
122420c295 1002) die(_("could not discard worktree changes"));
122420c295 1004) exit(1);
5e5d96197c 1015) exit(1);
5e5d96197c 1018) die(_("could not move back to %s"),
5a61494539 1028) die(_("could not remove '%s'"), options.state_dir);
51e9ea6da7 1042) BUG("action: %d", action);
c54dacb50e 1047) const char *last_slash = strrchr(options.state_dir, '/');
c54dacb50e 1048) const char *state_dir_base =
c54dacb50e 1049) last_slash ? last_slash + 1 : options.state_dir;
c54dacb50e 1050) const char *cmd_live_rebase =
c54dacb50e 1052) strbuf_reset(&buf);
c54dacb50e 1053) strbuf_addf(&buf, "rm -fr \"%s\"", options.state_dir);
c54dacb50e 1054) die(_("It seems that there is already a %s directory, 
and\n"
53f9e5be94 1078) strbuf_addstr(&options.git_am_opt, " --ignore-date");
53f9e5be94 1079) options.flags |= REBASE_FORCE;
7998dbe1ec 1091) strbuf_addf(&options.git_am_opt, " -C%d", opt_c);
3c3588c7d3 1123) else if (strcmp("no-rebase-cousins", rebase_merges))
3c3588c7d3 1124) die(_("Unknown mode: %s"), rebase_merges);
ba1905a5fe 1146) die(_("--strategy requires --merge or --interactive"));
ba1905a5fe 1156) BUG("unhandled rebase type (%d)", options.type);
cda614e489 1164) strbuf_addstr(&options.git_format_patch_opt, " 
--progress");
ac7f467fef 1173) options.state_dir = apply_dir();
ac7f467fef 1174) break;
ac7f467fef 1251) die(_("invalid upstream '%s'"), options.upstream_name);
9dba809a69 1257) die(_("Could not create new root commit"));
e65123a71d 1307) die(_("fatal: no such branch/commit '%s'"),
ac7f467fef 1315) die(_("No such ref: %s"), "HEAD");
ac7f467fef 1327) die(_("Could not resolve HEAD to a revision"));
e65123a71d 1329) BUG("unexpected number of arguments left to parse");
e0333e5c63 1340) die(_("could not read index"));
6defce2b02 1367) die(_("Cannot autostash"));
6defce2b02 1370) die(_("Unexpected stash response: '%s'"),
6defce2b02 1376) die(_("Could not create directory for '%s'"),
6defce2b02 1382) die(_("could not reset --hard"));
e65123a71d 1426) ret = !!error(_("could not parse '%s'"),
e65123a71d 1428) goto cleanup;
e65123a71d 1437) ret = !!error(_("could not switch to "
1ed9c14ff2 1447)  resolve_ref_unsafe("HEAD", 0, NULL, &flag))
1ed9c14ff2 1448) puts(_("HEAD is up to date."));
9a48a615b4 1457)  resolve_ref_unsafe("HEAD", 0, NULL, &flag))
9a48a615b4 1458) puts(_("HEAD is up to date, rebase forced."));

builtin/rev-list.c
7c0fe330d5 builtin/rev-list.c 227) die("unexpected missing %s object '%s'",
7c0fe330d5 builtin/rev-list.c 228)     type_name(obj->type), 
oid_to_hex(&obj->oid));
250edfa8c8 builtin/rev-list.c 434) bisect_flags |= BISECT_FIND_ALL;

builtin/stash.c
93871263d1 builtin/stash--helper.c  130) free_stash_info(info);
93871263d1 builtin/stash--helper.c  131) error(_("'%s' is not a 
stash-like commit"), revision);
93871263d1 builtin/stash--helper.c  132) exit(128);
93871263d1 builtin/stash--helper.c  165) free_stash_info(info);
93871263d1 builtin/stash--helper.c  166) fprintf_ln(stderr, _("No stash 
entries found."));
93871263d1 builtin/stash--helper.c  167) return -1;
93871263d1 builtin/stash--helper.c  202) free_stash_info(info);
31f109a361 builtin/stash--helper.c  229) return error(_("git stash clear 
with parameters is unimplemented"));
93871263d1 builtin/stash--helper.c  244) return -1;
93871263d1 builtin/stash--helper.c  252) return -1;
93871263d1 builtin/stash--helper.c  265) return -1;
93871263d1 builtin/stash--helper.c  268) return error(_("unable to write 
new index file"));
93871263d1 builtin/stash--helper.c  374) remove_path(stash_index_path.buf);
93871263d1 builtin/stash--helper.c  375) return -1;
93871263d1 builtin/stash--helper.c  402) return -1;
93871263d1 builtin/stash--helper.c  405) return error(_("Cannot apply a 
stash in the middle of a merge"));
93871263d1 builtin/stash--helper.c  415) strbuf_release(&out);
93871263d1 builtin/stash--helper.c  416) return -1;
93871263d1 builtin/stash--helper.c  422) return -1;
93871263d1 builtin/stash--helper.c  427) return -1;
93871263d1 builtin/stash--helper.c  434) return error(_("Could not 
restore untracked files from stash"));
93871263d1 builtin/stash--helper.c  465) return -1;
93871263d1 builtin/stash--helper.c  470) strbuf_release(&out);
93871263d1 builtin/stash--helper.c  475) strbuf_release(&out);
93871263d1 builtin/stash--helper.c  476) return -1;
31f109a361 builtin/stash--helper.c  551) return error(_("%s: Could not 
drop stash entry"),
b3513da4bd builtin/stash--helper.c  623) printf_ln(_("The stash entry is 
kept in case you need it again."));
8ceb24b2c3 builtin/stash--helper.c  754) free_stash_info(&info);
129f0b0a00 builtin/stash.c          755) 
usage_with_options(git_stash_show_usage, options);
0ac06fb81f builtin/stash--helper.c  808) fprintf_ln(stderr, _("\"git 
stash store\" requires one <commit> argument"));
0ac06fb81f builtin/stash--helper.c  809) return -1;
f6f191b3f2 builtin/stash--helper.c  884) return 1;
f6f191b3f2 builtin/stash--helper.c  925) ret = -1;
f6f191b3f2 builtin/stash--helper.c  926) goto done;
f6f191b3f2 builtin/stash--helper.c  931) ret = -1;
f6f191b3f2 builtin/stash--helper.c  932) goto done;
f6f191b3f2 builtin/stash--helper.c  937) ret = -1;
f6f191b3f2 builtin/stash--helper.c  938) goto done;
f6f191b3f2 builtin/stash--helper.c  966) ret = -1;
f6f191b3f2 builtin/stash--helper.c  967) goto done;
f6f191b3f2 builtin/stash--helper.c  978) ret = -1;
f6f191b3f2 builtin/stash--helper.c  979) goto done;
f6f191b3f2 builtin/stash--helper.c  984) ret = -1;
f6f191b3f2 builtin/stash--helper.c  985) goto done;
f6f191b3f2 builtin/stash--helper.c  992) ret = -1;
f6f191b3f2 builtin/stash--helper.c  993) goto done;
f6f191b3f2 builtin/stash--helper.c 1018) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1019) goto done;
f6f191b3f2 builtin/stash--helper.c 1031) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1032) goto done;
f6f191b3f2 builtin/stash--helper.c 1037) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1038) goto done;
f6f191b3f2 builtin/stash--helper.c 1049) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1050) goto done;
f6f191b3f2 builtin/stash--helper.c 1055) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1056) goto done;
8002b9e626 builtin/stash--helper.c 1089) fprintf_ln(stderr, _("You do 
not have the initial commit yet"));
8002b9e626 builtin/stash--helper.c 1110) if (!quiet)
8002b9e626 builtin/stash--helper.c 1111) fprintf_ln(stderr, _("Cannot 
save the current index state"));
f6f191b3f2 builtin/stash--helper.c 1112) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1113) *stash_msg = NULL;
f6f191b3f2 builtin/stash--helper.c 1114) goto done;
8002b9e626 builtin/stash--helper.c 1119) if (!quiet)
8002b9e626 builtin/stash--helper.c 1120) fprintf_ln(stderr, _("Cannot 
save the untracked files"));
f6f191b3f2 builtin/stash--helper.c 1121) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1122) *stash_msg = NULL;
f6f191b3f2 builtin/stash--helper.c 1123) goto done;
8002b9e626 builtin/stash--helper.c 1131) if (!quiet)
8002b9e626 builtin/stash--helper.c 1132) fprintf_ln(stderr, _("Cannot 
save the current worktree state"));
f6f191b3f2 builtin/stash--helper.c 1133) goto done;
8002b9e626 builtin/stash--helper.c 1139) if (!quiet)
8002b9e626 builtin/stash--helper.c 1140) fprintf_ln(stderr, _("Cannot 
save the current worktree state"));
f6f191b3f2 builtin/stash--helper.c 1141) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1142) *stash_msg = NULL;
f6f191b3f2 builtin/stash--helper.c 1143) goto done;
8002b9e626 builtin/stash--helper.c 1166) if (!quiet)
8002b9e626 builtin/stash--helper.c 1167) fprintf_ln(stderr, _("Cannot 
record working tree state"));
f6f191b3f2 builtin/stash--helper.c 1168) ret = -1;
f6f191b3f2 builtin/stash--helper.c 1169) goto done;
48c061fa44 builtin/stash--helper.c 1251) return -1;
8002b9e626 builtin/stash--helper.c 1260) if (!quiet)
8002b9e626 builtin/stash--helper.c 1261) fprintf_ln(stderr, _("Cannot 
initialize stash"));
48c061fa44 builtin/stash--helper.c 1262) return -1;
8002b9e626 builtin/stash--helper.c 1272) if (!quiet)
8002b9e626 builtin/stash--helper.c 1273) fprintf_ln(stderr, _("Cannot 
save the current status"));
48c061fa44 builtin/stash--helper.c 1274) ret = -1;
48c061fa44 builtin/stash--helper.c 1275) goto done;
48c061fa44 builtin/stash--helper.c 1292) ret = -1;
48c061fa44 builtin/stash--helper.c 1311) ret = -1;
48c061fa44 builtin/stash--helper.c 1312) goto done;
48c061fa44 builtin/stash--helper.c 1321) ret = -1;
48c061fa44 builtin/stash--helper.c 1322) goto done;
48c061fa44 builtin/stash--helper.c 1330) ret = -1;
48c061fa44 builtin/stash--helper.c 1339) ret = -1;
48c061fa44 builtin/stash--helper.c 1350) ret = -1;
48c061fa44 builtin/stash--helper.c 1359) ret = -1;
48c061fa44 builtin/stash--helper.c 1360) goto done;
48c061fa44 builtin/stash--helper.c 1368) ret = -1;
48c061fa44 builtin/stash--helper.c 1394) ret = -1;
129f0b0a00 builtin/stash.c         1524) 
usage_msg_opt(xstrfmt(_("unknown subcommand: %s"), argv[0]),
129f0b0a00 builtin/stash.c         1554) continue;

builtin/upload-archive.c
e001fd3a50 builtin/upload-archive.c 111) if (version == protocol_v0 || 
version == protocol_v1)
e001fd3a50 builtin/upload-archive.c 112) packet_write_fmt(1, "NACK 
unable to spawn subprocess\n");
e001fd3a50 builtin/upload-archive.c 113) else if (version == protocol_v2)
e001fd3a50 builtin/upload-archive.c 114) error_clnt("unable to spawn 
subprocess\n");

config.c
c780b9cfe8 2298) return val;
c780b9cfe8 2301) if (is_bool)
c780b9cfe8 2302) return val ? 0 : 1;
c780b9cfe8 2304) return val;

help.c
26c7d06783 help.c         500) static int get_alias(const char *var, 
const char *value, void *data)
26c7d06783 help.c         502) struct string_list *list = data;
26c7d06783 help.c         504) if (skip_prefix(var, "alias.", &var))
26c7d06783 help.c         505) string_list_append(list, var)->util = 
xstrdup(value);
26c7d06783 help.c         507) return 0;
26c7d06783 help.c         530) printf("\n%s\n", _("Command aliases"));
26c7d06783 help.c         531) ALLOC_ARRAY(aliases, alias_list.nr + 1);
26c7d06783 help.c         532) for (i = 0; i < alias_list.nr; i++) {
26c7d06783 help.c         533) aliases[i].name = alias_list.items[i].string;
26c7d06783 help.c         534) aliases[i].help = alias_list.items[i].util;
26c7d06783 help.c         535) aliases[i].category = 1;
26c7d06783 help.c         537) aliases[alias_list.nr].name = NULL;
26c7d06783 help.c         538) print_command_list(aliases, 1, longest);
26c7d06783 help.c         539) free(aliases);

http-backend.c
fb19d32f05 646) argv[1] = ".";
fb19d32f05 647) argv[2] = NULL;

list-objects-filter-options.c
bc5975d24f  55) if (errbuf) {
bc5975d24f  56) strbuf_addstr(
bc5975d24f  60) return 1;
cc0b05a4cc  86) if (errbuf)

list-objects-filter.c
696aa73905  47) BUG("unknown filter_situation: %d", filter_situation);
bc5975d24f 101) BUG("unknown filter_situation: %d", filter_situation);
696aa73905 152) BUG("unknown filter_situation: %d", filter_situation);
696aa73905 257) BUG("unknown filter_situation: %d", filter_situation);
696aa73905 438) BUG("invalid list-objects filter choice: %d",

list-objects.c
f447a499db 197) ctx->show_object(obj, base->buf, ctx->show_data);

midx.c
e43d2dcce1  285) if (hasheq(oid.hash,
e43d2dcce1  286)    p->bad_object_sha1 + the_hash_algo->rawsz * i))

oidset.c
8b2f8cbcb1 29) kh_del_oid(&set->set, pos);
8b2f8cbcb1 30) return 1;

packfile.c
fb571c5fa7  117) return error("index file %s is too small", path);
fb571c5fa7  119) return error("empty data");

preload-index.c
ae9af12287  73) struct progress_data *pd = p->progress;
ae9af12287  75) pthread_mutex_lock(&pd->mutex);
ae9af12287  76) pd->n += last_nr - nr;
ae9af12287  77) display_progress(pd->progress, pd->n);
ae9af12287  78) pthread_mutex_unlock(&pd->mutex);
ae9af12287  79) last_nr = nr;
ae9af12287  93) struct progress_data *pd = p->progress;
ae9af12287  95) pthread_mutex_lock(&pd->mutex);
ae9af12287  96) display_progress(pd->progress, pd->n + last_nr);
ae9af12287  97) pthread_mutex_unlock(&pd->mutex);
ae9af12287 128) pd.progress = start_delayed_progress(_("Refreshing 
index"), index->cache_nr);
ae9af12287 129) pthread_mutex_init(&pd.mutex, NULL);
ae9af12287 140) p->progress = &pd;

prio-queue.c
2d181390f3 94) return queue->array[queue->nr - 1].data;

read-cache.c
ae9af12287 1490) progress = start_delayed_progress(_("Refresh index"),
ae9af12287 1491)   istate->cache_nr);
ae9af12287 1533) display_progress(progress, i);
ae9af12287 1566) display_progress(progress, istate->cache_nr);
ae9af12287 1567) stop_progress(&progress);
252d079cbd 1778) const unsigned char *cp = (const unsigned char *)name;
252d079cbd 1782) strip_len = decode_varint(&cp);
77ff1127a4 1783) if (previous_ce) {
77ff1127a4 1784) previous_len = previous_ce->ce_namelen;
77ff1127a4 1785) if (previous_len < strip_len)
252d079cbd 1786) die(_("malformed name field in the index, near path '%s'"),
77ff1127a4 1787) previous_ce->name);
77ff1127a4 1788) copy_len = previous_len - strip_len;
77ff1127a4 1790) copy_len = 0;
252d079cbd 1792) name = (const char *)cp;
252d079cbd 1798) len += copy_len;
252d079cbd 1819) if (copy_len)
252d079cbd 1820) memcpy(ce->name, previous_ce->name, copy_len);
252d079cbd 1821) memcpy(ce->name + copy_len, name, len + 1 - copy_len);
252d079cbd 1822) *ent_size = (name - ((char *)ondisk)) + len + 1 - copy_len;
abb4bb8384 1959) munmap((void *)p->mmap, p->mmap_size);
abb4bb8384 1960) die(_("index file corrupt"));
77ff1127a4 2001) mem_pool_init(&istate->ce_mem_pool,
77ff1127a4 2041) static void *load_cache_entries_thread(void *_data)
77ff1127a4 2043) struct load_cache_entries_thread_data *p = _data;
77ff1127a4 2047) for (i = p->ieot_start; i < p->ieot_start + 
p->ieot_blocks; i++) {
77ff1127a4 2048) p->consumed += load_cache_entry_block(p->istate, 
p->ce_mem_pool,
77ff1127a4 2049) p->offset, p->ieot->entries[i].nr, p->mmap, 
p->ieot->entries[i].offset, NULL);
77ff1127a4 2050) p->offset += p->ieot->entries[i].nr;
77ff1127a4 2052) return NULL;
77ff1127a4 2055) static unsigned long load_cache_entries_threaded(struct 
index_state *istate, const char *mmap, size_t mmap_size,
77ff1127a4 2060) unsigned long consumed = 0;
77ff1127a4 2063) if (istate->name_hash_initialized)
77ff1127a4 2064) BUG("the name hash isn't thread safe");
77ff1127a4 2066) mem_pool_init(&istate->ce_mem_pool, 0);
77ff1127a4 2069) if (nr_threads > ieot->nr)
77ff1127a4 2070) nr_threads = ieot->nr;
77ff1127a4 2071) data = xcalloc(nr_threads, sizeof(*data));
77ff1127a4 2073) offset = ieot_start = 0;
77ff1127a4 2074) ieot_blocks = DIV_ROUND_UP(ieot->nr, nr_threads);
77ff1127a4 2075) for (i = 0; i < nr_threads; i++) {
77ff1127a4 2076) struct load_cache_entries_thread_data *p = &data[i];
77ff1127a4 2079) if (ieot_start + ieot_blocks > ieot->nr)
77ff1127a4 2080) ieot_blocks = ieot->nr - ieot_start;
77ff1127a4 2082) p->istate = istate;
77ff1127a4 2083) p->offset = offset;
77ff1127a4 2084) p->mmap = mmap;
77ff1127a4 2085) p->ieot = ieot;
77ff1127a4 2086) p->ieot_start = ieot_start;
77ff1127a4 2087) p->ieot_blocks = ieot_blocks;
77ff1127a4 2090) nr = 0;
77ff1127a4 2091) for (j = p->ieot_start; j < p->ieot_start + 
p->ieot_blocks; j++)
77ff1127a4 2092) nr += p->ieot->entries[j].nr;
77ff1127a4 2093) if (istate->version == 4) {
77ff1127a4 2094) mem_pool_init(&p->ce_mem_pool,
77ff1127a4 2097) mem_pool_init(&p->ce_mem_pool,
77ff1127a4 2101) err = pthread_create(&p->pthread, NULL, 
load_cache_entries_thread, p);
77ff1127a4 2102) if (err)
77ff1127a4 2103) die(_("unable to create load_cache_entries thread: 
%s"), strerror(err));
77ff1127a4 2106) for (j = 0; j < ieot_blocks; j++)
77ff1127a4 2107) offset += ieot->entries[ieot_start + j].nr;
77ff1127a4 2108) ieot_start += ieot_blocks;
77ff1127a4 2111) for (i = 0; i < nr_threads; i++) {
77ff1127a4 2112) struct load_cache_entries_thread_data *p = &data[i];
77ff1127a4 2114) err = pthread_join(p->pthread, NULL);
77ff1127a4 2115) if (err)
77ff1127a4 2116) die(_("unable to join load_cache_entries thread: %s"), 
strerror(err));
77ff1127a4 2117) mem_pool_combine(istate->ce_mem_pool, p->ce_mem_pool);
77ff1127a4 2118) consumed += p->consumed;
77ff1127a4 2121) free(data);
77ff1127a4 2123) return consumed;
77ff1127a4 2192) nr_threads = cpus;
abb4bb8384 2196) extension_offset = read_eoie_extension(mmap, mmap_size);
abb4bb8384 2197) if (extension_offset) {
abb4bb8384 2200) p.src_offset = extension_offset;
abb4bb8384 2201) err = pthread_create(&p.pthread, NULL, 
load_index_extensions, &p);
abb4bb8384 2202) if (err)
abb4bb8384 2203) die(_("unable to create load_index_extensions thread: 
%s"), strerror(err));
abb4bb8384 2205) nr_threads--;
77ff1127a4 2214) ieot = read_ieot_extension(mmap, mmap_size, 
extension_offset);
77ff1127a4 2217) src_offset += load_cache_entries_threaded(istate, mmap, 
mmap_size, src_offset, nr_threads, ieot);
77ff1127a4 2218) free(ieot);
abb4bb8384 2232) int ret = pthread_join(p.pthread, NULL);
abb4bb8384 2233) if (ret)
abb4bb8384 2234) die(_("unable to join load_index_extensions thread: 
%s"), strerror(ret));
3255089ada 2775) ieot_blocks = nr_threads;
77ff1127a4 2776) if (ieot_blocks > istate->cache_nr)
77ff1127a4 2777) ieot_blocks = istate->cache_nr;
3255089ada 2785) ieot = xcalloc(1, sizeof(struct index_entry_offset_table)
3255089ada 2786) + (ieot_blocks * sizeof(struct index_entry_offset)));
77ff1127a4 2787) ieot_entries = DIV_ROUND_UP(entries, ieot_blocks);
3255089ada 2794) free(ieot);
3b1d9e045e 2795) return -1;
3255089ada 2821) ieot->entries[ieot->nr].nr = nr;
3255089ada 2822) ieot->entries[ieot->nr].offset = offset;
3255089ada 2823) ieot->nr++;
3255089ada 2829) if (previous_name)
3255089ada 2830) previous_name->buf[0] = 0;
3255089ada 2831) nr = 0;
3255089ada 2832) offset = lseek(newfd, 0, SEEK_CUR);
3255089ada 2833) if (offset < 0) {
3255089ada 2834) free(ieot);
3255089ada 2835) return -1;
3255089ada 2837) offset += write_buffer_len;
3255089ada 2847) ieot->entries[ieot->nr].nr = nr;
3255089ada 2848) ieot->entries[ieot->nr].offset = offset;
3255089ada 2849) ieot->nr++;
3255089ada 2861) free(ieot);
3b1d9e045e 2862) return -1;
3255089ada 2876) struct strbuf sb = STRBUF_INIT;
3255089ada 2878) write_ieot_extension(&sb, ieot);
3255089ada 2879) err = write_index_ext_header(&c, &eoie_c, newfd, 
CACHE_EXT_INDEXENTRYOFFSETTABLE, sb.len) < 0
3255089ada 2880) || ce_write(&c, newfd, sb.buf, sb.len) < 0;
3255089ada 2881) strbuf_release(&sb);
3255089ada 2882) free(ieot);
3255089ada 2883) if (err)
3255089ada 2884) return -1;
3b1d9e045e 3372) static size_t read_eoie_extension(const char *mmap, 
size_t mmap_size)
3b1d9e045e 3390) if (mmap_size < sizeof(struct cache_header) + 
EOIE_SIZE_WITH_HEADER + the_hash_algo->rawsz)
3b1d9e045e 3391) return 0;
3b1d9e045e 3394) index = eoie = mmap + mmap_size - EOIE_SIZE_WITH_HEADER 
- the_hash_algo->rawsz;
3b1d9e045e 3395) if (CACHE_EXT(index) != CACHE_EXT_ENDOFINDEXENTRIES)
3b1d9e045e 3396) return 0;
3b1d9e045e 3397) index += sizeof(uint32_t);
3b1d9e045e 3400) extsize = get_be32(index);
3b1d9e045e 3401) if (extsize != EOIE_SIZE)
3b1d9e045e 3402) return 0;
3b1d9e045e 3403) index += sizeof(uint32_t);
3b1d9e045e 3409) offset = get_be32(index);
3b1d9e045e 3410) if (mmap + offset < mmap + sizeof(struct cache_header))
3b1d9e045e 3411) return 0;
3b1d9e045e 3412) if (mmap + offset >= eoie)
3b1d9e045e 3413) return 0;
3b1d9e045e 3414) index += sizeof(uint32_t);
3b1d9e045e 3425) src_offset = offset;
3b1d9e045e 3426) the_hash_algo->init_fn(&c);
3b1d9e045e 3427) while (src_offset < mmap_size - the_hash_algo->rawsz - 
EOIE_SIZE_WITH_HEADER) {
3b1d9e045e 3435) memcpy(&extsize, mmap + src_offset + 4, 4);
3b1d9e045e 3436) extsize = ntohl(extsize);
3b1d9e045e 3439) if (src_offset + 8 + extsize < src_offset)
3b1d9e045e 3440) return 0;
3b1d9e045e 3442) the_hash_algo->update_fn(&c, mmap + src_offset, 8);
3b1d9e045e 3444) src_offset += 8;
3b1d9e045e 3445) src_offset += extsize;
3b1d9e045e 3447) the_hash_algo->final_fn(hash, &c);
3b1d9e045e 3448) if (!hasheq(hash, (const unsigned char *)index))
3b1d9e045e 3449) return 0;
3b1d9e045e 3452) if (src_offset != mmap_size - the_hash_algo->rawsz - 
EOIE_SIZE_WITH_HEADER)
3b1d9e045e 3453) return 0;
3b1d9e045e 3455) return offset;
3255089ada 3475) static struct index_entry_offset_table 
*read_ieot_extension(const char *mmap, size_t mmap_size, size_t offset)
3255089ada 3477)        const char *index = NULL;
3255089ada 3483)        if (!offset)
3255089ada 3484)        return NULL;
3255089ada 3485)        while (offset <= mmap_size - 
the_hash_algo->rawsz - 8) {
3255089ada 3486)        extsize = get_be32(mmap + offset + 4);
3255089ada 3487)        if (CACHE_EXT((mmap + offset)) == 
CACHE_EXT_INDEXENTRYOFFSETTABLE) {
3255089ada 3488)        index = mmap + offset + 4 + 4;
3255089ada 3489)        break;
3255089ada 3491)        offset += 8;
3255089ada 3492)        offset += extsize;
3255089ada 3494)        if (!index)
3255089ada 3495)        return NULL;
3255089ada 3498)        ext_version = get_be32(index);
3255089ada 3499)        if (ext_version != IEOT_VERSION) {
3255089ada 3500)        error("invalid IEOT version %d", ext_version);
3255089ada 3501)        return NULL;
3255089ada 3503)        index += sizeof(uint32_t);
3255089ada 3506)        nr = (extsize - sizeof(uint32_t)) / 
(sizeof(uint32_t) + sizeof(uint32_t));
3255089ada 3507)        if (!nr) {
3255089ada 3508)        error("invalid number of IEOT entries %d", nr);
3255089ada 3509)        return NULL;
3255089ada 3511)        ieot = xmalloc(sizeof(struct 
index_entry_offset_table)
3255089ada 3512)        + (nr * sizeof(struct index_entry_offset)));
3255089ada 3513)        ieot->nr = nr;
3255089ada 3514)        for (i = 0; i < nr; i++) {
3255089ada 3515)        ieot->entries[i].offset = get_be32(index);
3255089ada 3516)        index += sizeof(uint32_t);
3255089ada 3517)        ieot->entries[i].nr = get_be32(index);
3255089ada 3518)        index += sizeof(uint32_t);
3255089ada 3521)        return ieot;
3255089ada 3524) static void write_ieot_extension(struct strbuf *sb, 
struct index_entry_offset_table *ieot)
3255089ada 3530)        put_be32(&buffer, IEOT_VERSION);
3255089ada 3531)        strbuf_add(sb, &buffer, sizeof(uint32_t));
3255089ada 3534)        for (i = 0; i < ieot->nr; i++) {
3255089ada 3537)        put_be32(&buffer, ieot->entries[i].offset);
3255089ada 3538)        strbuf_add(sb, &buffer, sizeof(uint32_t));
3255089ada 3541)        put_be32(&buffer, ieot->entries[i].nr);
3255089ada 3542)        strbuf_add(sb, &buffer, sizeof(uint32_t));
3255089ada 3544) }

rebase-interactive.c
64a43cbd5d 61) return error_errno(_("could not read '%s'."), todo_file);
64a43cbd5d 65) strbuf_release(&buf);
64a43cbd5d 66) return -1;
a9f5476fbc 74) return error_errno(_("could not read '%s'."), todo_file);
a9f5476fbc 78) strbuf_release(&buf);
a9f5476fbc 79) return -1;
64a43cbd5d 85) return -1;

revision.c
4943d28849 2931) return;
4943d28849 2934) return;
4943d28849 2937) c->object.flags |= UNINTERESTING;
4943d28849 2940) return;
4943d28849 2943) mark_parents_uninteresting(c);
4943d28849 2966) return;
4943d28849 2969) return;
4943d28849 2974) return;
4943d28849 3019) info->topo_queue.compare = compare_commits_by_commit_date;
4943d28849 3020) break;
4943d28849 3022) init_author_date_slab(&info->author_date);
4943d28849 3023) info->topo_queue.compare = compare_commits_by_author_date;
4943d28849 3024) info->topo_queue.cb_data = &info->author_date;
4943d28849 3025) break;
4943d28849 3038) continue;
4943d28849 3048) record_author_date(&info->author_date, c);
6c04ff3001 3086) if (!revs->ignore_missing_links)
6c04ff3001 3087) die("Failed to traverse parents of commit %s",
4943d28849 3088) oid_to_hex(&commit->object.oid));
4943d28849 3096) continue;

sequencer.c
65850686cf 2276) return;
65850686cf 2373) write_file(rebase_path_quiet(), "%s\n", quiet);
2c58483a59 3371) return error(_("could not checkout %s"), commit);
4df66c40b0 3385) return error(_("%s: not a valid OID"), orig_head);
b97e187364 4748) return -1;
b97e187364 4751) return -1;
b97e187364 4757) return error_errno(_("could not read '%s'."), todo_file);
b97e187364 4760) todo_list_release(&todo_list);
b97e187364 4761) return error(_("unusable todo list: '%s'"), todo_file);
b97e187364 4780) todo_list_release(&todo_list);
b97e187364 4781) return -1;
b97e187364 4785) return error(_("could not copy '%s' to '%s'."), todo_file,
b97e187364 4789) return error(_("could not transform the todo list"));
b97e187364 4818) return error(_("could not transform the todo list"));
b97e187364 4821) return error(_("could not skip unnecessary pick 
commands"));
b97e187364 4827) return -1;

split-index.c
568f3a6073 310) const unsigned int ondisk_flags =
568f3a6073 314) ce_flags = ce->ce_flags;
568f3a6073 315) base_flags = base->ce_flags;
568f3a6073 317) ce->ce_flags   &= ondisk_flags;
568f3a6073 318) base->ce_flags &= ondisk_flags;
568f3a6073 319) ret = memcmp(&ce->ce_stat_data, &base->ce_stat_data,
568f3a6073 322) ce->ce_flags = ce_flags;
568f3a6073 323) base->ce_flags = base_flags;
568f3a6073 324) if (ret)
568f3a6073 325) ce->ce_flags |= CE_UPDATE_IN_BASE;

strbuf.c
f95736288a  127) --sb->len;

transport-helper.c
fb19d32f05  643) if (!data->connect && !data->stateless_connect)

transport.c
99bcb883cb  299) BUG("buffer must be empty at the end of handshake()");

Commits introducing uncovered code:
Alban Gruin      0af129b2e: rebase--interactive2: rewrite the submodes 
of interactive rebase in C
Alban Gruin      2c58483a5: rebase -i: rewrite setup_reflog_action() in C
Alban Gruin      34b47315d: rebase -i: move rebase--helper modes to 
rebase--interactive
Alban Gruin      4df66c40b: rebase -i: rewrite checkout_onto() in C
Alban Gruin      53bbcfbde: rebase -i: implement the main part of 
interactive rebase as a builtin
Alban Gruin      64a43cbd5: rebase -i: rewrite the edit-todo 
functionality in C
Alban Gruin      65850686c: rebase -i: rewrite write_basic_state() in C
Alban Gruin      a9f5476fb: sequencer: refactor append_todo_help() to 
write its message to a buffer
Alban Gruin      b97e18736: rebase -i: rewrite complete_action() in C
Ben Peart      3255089ad: ieot: add Index Entry Offset Table (IEOT) 
extension
Ben Peart      3b1d9e045: eoie: add End of Index Entry (EOIE) extension
Ben Peart      77ff1127a: read-cache: load cache entries on worker threads
Ben Peart      abb4bb838: read-cache: load cache extensions on a worker 
thread
Ben Peart      c780b9cfe: config: add new index.threads config setting
Derrick Stolee      2d181390f: prio-queue: add 'peek' operation
Derrick Stolee      4943d2884: revision.c: refactor basic topo-order logic
Derrick Stolee      6c04ff300: revision.c: begin refactoring 
--topo-order logic
Harald Nordgren      250edfa8c: bisect: create 'bisect_flags' parameter 
in find_bisection()
Jeff King      e43d2dcce: more oideq/hasheq conversions
Joel Teichroeb      31f109a36: stash: convert drop and clear to builtin
Joel Teichroeb      93871263d: stash: convert apply to builtin
Joel Teichroeb      b3513da4b: stash: convert pop to builtin
Johannes Schindelin      bc24382c2: builtin rebase: prepare for builtin 
rebase -i
Jonathan Tan      99bcb883c: transport: allow skipping of ref listing
Josh Steadmon      e001fd3a5: archive: implement protocol v2 archive command
Josh Steadmon      fb19d32f0: archive: allow archive over HTTP(S) with 
proto v2
Josh Steadmon      fb571c5fa: fuzz: add fuzz testing for packfile indices
Junio C Hamano      7bbc53a58: fetch: replace string-list used as a 
look-up table with a hashmap
Matthew DeVore      696aa7390: list-objects-filter: use BUG rather than die
Matthew DeVore      7c0fe330d: rev-list: handle missing tree objects 
properly
Matthew DeVore      bc5975d24: list-objects-filter: implement filter tree:0
Matthew DeVore      cc0b05a4c: list-objects-filter-options: do not 
over-strbuf_init
Matthew DeVore      f447a499d: list-objects: store common func args in 
struct
Nguyễn Thái Ngọc Duy      252d079cb: read-cache.c: optimize reading 
index format v4
Nguyễn Thái Ngọc Duy      26c7d0678: help -a: improve and make --verbose 
default
Nguyễn Thái Ngọc Duy      ae9af1228: status: show progress bar if 
refreshing the index takes too long
Paul-Sebastian Ungureanu      0ac06fb81: stash: convert store to builtin
Paul-Sebastian Ungureanu      129f0b0a0: stash: convert 
`stash--helper.c` into `stash.c`
Paul-Sebastian Ungureanu      48c061fa4: stash: convert push to builtin
Paul-Sebastian Ungureanu      8002b9e62: stash: make push -q quiet
Paul-Sebastian Ungureanu      8ceb24b2c: stash: convert show to builtin
Paul-Sebastian Ungureanu      f6f191b3f: stash: convert create to builtin
Pratik Karki      002ee2fe6: builtin rebase: support `keep-empty` option
Pratik Karki      0eabf4b95: builtin rebase: stop if `git am` is in progress
Pratik Karki      12026a412: builtin rebase: support `--gpg-sign` option
Pratik Karki      122420c29: builtin rebase: support --skip
Pratik Karki      1ed9c14ff: builtin rebase: support --force-rebase
Pratik Karki      3c3588c7d: builtin rebase: support 
--rebase-merges[=[no-]rebase-cousins]
Pratik Karki      51e9ea6da: builtin rebase: support --edit-todo and 
--show-current-patch
Pratik Karki      53f9e5be9: builtin rebase: support `ignore-date` option
Pratik Karki      55071ea24: rebase: start implementing it as a builtin
Pratik Karki      5a6149453: builtin rebase: support --quit
Pratik Karki      5e5d96197: builtin rebase: support --abort
Pratik Karki      6defce2b0: builtin rebase: support `--autostash` option
Pratik Karki      73d51ed0a: builtin rebase: support --signoff
Pratik Karki      7998dbe1e: builtin rebase: support `-C` and 
`--whitespace=<type>`
Pratik Karki      9a48a615b: builtin rebase: try to fast forward when 
possible
Pratik Karki      9dba809a6: builtin rebase: support --root
Pratik Karki      ac7f467fe: builtin/rebase: support running "git rebase 
<upstream>"
Pratik Karki      ba1905a5f: builtin rebase: add support for custom 
merge strategies
Pratik Karki      bff014dac: builtin rebase: support the `verbose` and 
`diffstat` options
Pratik Karki      c54dacb50: builtin rebase: start a new rebase only if 
none is in progress
Pratik Karki      cda614e48: builtin rebase: show progress when 
connected to a terminal
Pratik Karki      d4c569f8f: builtin rebase: only store fully-qualified 
refs in `options.head_name`
Pratik Karki      e0333e5c6: builtin rebase: require a clean worktree
Pratik Karki      e65123a71: builtin rebase: support `git rebase 
<upstream> <switch-to>`
Pratik Karki      ead98c111: builtin rebase: support --rerere-autoupdate
Pratik Karki      f28d40d3a: builtin rebase: support --onto
Pratik Karki      f95736288: builtin rebase: support --continue
René Scharfe      8b2f8cbcb: oidset: use khash
SZEDER Gábor      568f3a607: split-index: don't compare stat data of 
entries already marked for split index

Uncovered code new in next, compared to master. Build logs at [2]
-----------------------------------------------------------------

blame.c
a470beea39  113)  !strcmp(r->index->cache[-1 - pos]->name, path))
a470beea39  272) int pos = index_name_pos(r->index, path, len);
a470beea39  274) mode = r->index->cache[pos]->ce_mode;

builtin/am.c
2abf350385 1414) repo_init_revisions(the_repository, &rev_info, NULL);

builtin/gc.c
3029970275 builtin/gc.c 461) ret = error_errno(_("cannot stat '%s'"), 
gc_log_path);
3029970275 builtin/gc.c 470) ret = error_errno(_("cannot read '%s'"), 
gc_log_path);
fec2ed2187 builtin/gc.c 495) die(FAILED_RUN, pack_refs_cmd.argv[0]);
fec2ed2187 builtin/gc.c 498) die(FAILED_RUN, reflog.argv[0]);a
3029970275 builtin/gc.c 585) exit(128);
fec2ed2187 builtin/gc.c 637) die(FAILED_RUN, repack.argv[0]);
fec2ed2187 builtin/gc.c 647) die(FAILED_RUN, prune.argv[0]);
fec2ed2187 builtin/gc.c 654) die(FAILED_RUN, prune_worktrees.argv[0]);
fec2ed2187 builtin/gc.c 658) die(FAILED_RUN, rerere.argv[0]);

builtin/pack-objects.c
2fa233a554 builtin/pack-objects.c 1512) hashcpy(base_oid.hash, base_sha1);
2fa233a554 builtin/pack-objects.c 1513) if 
(!in_same_island(&delta->idx.oid, &base_oid))
2fa233a554 builtin/pack-objects.c 1514) return 0;

commit-graph.c
20fd6d5799   79) return 0;

diff.c
b78ea5fc35 4117) add_external_diff_name(o->repo, &argv, other, two);

refs.c
4a6067cda5 1405) return 0;

revision.c
2abf350385 1526) if (ce_path_match(istate, ce, &revs->prune_data, NULL)) {
2abf350385 1532) while ((i+1 < istate->cache_nr) &&
2abf350385 1533)        ce_same_name(ce, istate->cache[i+1]))

wt-status.c
f3bd35fa0d  671) s->committable = 1;
73ba5d78b4 1953) if (s->state.rebase_in_progress ||
73ba5d78b4 1954)     s->state.rebase_interactive_in_progress)
73ba5d78b4 1955) branch_name = s->state.onto;
73ba5d78b4 1956) else if (s->state.detached_from)
73ba5d78b4 1957) branch_name = s->state.detached_from;

Commits introducing uncovered code:
Derrick Stolee      20fd6d579: commit-graph: not compatible with grafts
Jeff King      2fa233a55: pack-objects: handle island check for 
"external" delta base
Jonathan Nieder      302997027: gc: do not return error for prior errors 
in daemonized mode
Jonathan Nieder      fec2ed218: gc: exit with status 128 on failure
Nguyễn Thái Ngọc Duy      2abf35038: revision.c: remove implicit 
dependency on the_index
Nguyễn Thái Ngọc Duy      a470beea3: blame.c: rename "repo" argument to "r"
Nguyễn Thái Ngọc Duy      b78ea5fc3: diff.c: reduce implicit dependency 
on the_index
Stefan Beller      4a6067cda: refs.c: migrate internal ref iteration to 
pass thru repository argument
Stephen P. Smith      73ba5d78b: roll wt_status_state into wt_status and 
populate in the collect phase
Stephen P. Smith      f3bd35fa0: wt-status.c: set the committable flag 
in the collect phase


Uncovered code new in master, compared to maint. Build logs at [3]
------------------------------------------------------------------

builtin/checkout.c
fa655d8411 builtin/checkout.c  537) return 0;
fa655d8411 builtin/checkout.c  951) return error(_("index file corrupt"));

builtin/commit.c
859fdc0c3c builtin/commit.c 1657) 
write_commit_graph_reachable(get_object_directory(), 0);

builtin/difftool.c
4a7e27e957 441) if (oideq(&loid, &roid))

builtin/fsck.c
454ea2e4d7 builtin/fsck.c 743) for (p = get_all_packs(the_repository); p;
66ec0390e7 builtin/fsck.c 862) midx_argv[2] = "--object-dir";
66ec0390e7 builtin/fsck.c 863) midx_argv[3] = alt->path;
66ec0390e7 builtin/fsck.c 864) if (run_command(&midx_verify))
66ec0390e7 builtin/fsck.c 865) errors_found |= ERROR_COMMIT_GRAPH;

builtin/log.c
2e6fd71a52 builtin/log.c 1461) die(_("failed to infer range-diff ranges"));
ee6cbf712e builtin/log.c 1807) die(_("--interdiff requires 
--cover-letter or single patch"));
8631bf1cdd builtin/log.c 1817) else if (!rdiff_prev)
8631bf1cdd builtin/log.c 1818) die(_("--creation-factor requires 
--range-diff"));
40ce41604d builtin/log.c 1822) die(_("--range-diff requires 
--cover-letter or single patch"));

builtin/multi-pack-index.c
6d68e6a461 35) usage_with_options(builtin_multi_pack_index_usage,
6d68e6a461 39) die(_("too many arguments"));
6d68e6a461 48) die(_("unrecognized verb: %s"), argv[0]);

builtin/pack-objects.c
6a22d52126 builtin/pack-objects.c 1079) if (fill_midx_entry(oid, &e, m)) {
6a22d52126 builtin/pack-objects.c 1080) struct packed_git *p = e.p;
6a22d52126 builtin/pack-objects.c 1083) if (p == *found_pack)
6a22d52126 builtin/pack-objects.c 1084) offset = *found_offset;
6a22d52126 builtin/pack-objects.c 1086) offset = 
find_pack_entry_one(oid->hash, p);
6a22d52126 builtin/pack-objects.c 1088) if (offset) {
6a22d52126 builtin/pack-objects.c 1089) if (!*found_pack) {
6a22d52126 builtin/pack-objects.c 1090) if (!is_pack_valid(p))
6a22d52126 builtin/pack-objects.c 1091) continue;
6a22d52126 builtin/pack-objects.c 1092) *found_offset = offset;
6a22d52126 builtin/pack-objects.c 1093) *found_pack = p;
6a22d52126 builtin/pack-objects.c 1095) want = 
want_found_object(exclude, p);
6a22d52126 builtin/pack-objects.c 1096) if (want != -1)
6a22d52126 builtin/pack-objects.c 1097) return want;
28b8a73080 builtin/pack-objects.c 2778) depth++;
108f530385 builtin/pack-objects.c 2782) oe_set_tree_depth(&to_pack, ent, 
depth);
454ea2e4d7 builtin/pack-objects.c 2966) p = get_all_packs(the_repository);

builtin/pack-redundant.c
454ea2e4d7 builtin/pack-redundant.c 580) struct packed_git *p = 
get_all_packs(the_repository);
454ea2e4d7 builtin/pack-redundant.c 595) struct packed_git *p = 
get_all_packs(the_repository);

builtin/remote.c
5025425dff builtin/remote.c  864) return error(_("No such remote: 
'%s'"), name);

builtin/repack.c
16d75fa48d  48) use_delta_islands = git_config_bool(var, value);
16d75fa48d  49) return 0;

builtin/rerere.c
2373b65059 builtin/rerere.c  78) warning(_("'git rerere forget' without 
paths is deprecated"));
2373b65059 builtin/rerere.c 110) die(_("unable to generate diff for 
'%s'"), rerere_path(id, NULL));

builtin/show-branch.c
9001dc2a74 builtin/show-branch.c 430) if (get_oid(refname + ofs, &tmp) 
|| !oideq(&tmp, oid))

builtin/submodule--helper.c
ee69b2a90c 1462) die(_("Invalid update mode '%s' for submodule path '%s'"),
ee69b2a90c 1466) die(_("Invalid update mode '%s' configured for 
submodule path '%s'"),
ee69b2a90c 1469) trace_printf("loaded thing");
ee69b2a90c 1470) out->type = sub->update_strategy.type;
ee69b2a90c 1471) out->command = sub->update_strategy.command;
ee69b2a90c 1491) die("submodule--helper update-module-clone expects 
<just-cloned> <path> [<update>]");
74d4731da1 2033) BUG("submodule--helper connect-gitdir-workingtree 
<name> <path>");
74d4731da1 2039) BUG("We could get the submodule handle before?");
74d4731da1 2042) die(_("could not get a repository handle for submodule 
'%s'"), path);

builtin/unpack-objects.c
4a7e27e957 builtin/unpack-objects.c 306) if (oideq(&info->base_oid, 
&obj_list[nr].oid) ||

builtin/update-index.c
4a7e27e957 builtin/update-index.c  672) if (oideq(&ce_2->oid, &ce_3->oid) &&

builtin/worktree.c
e5353bef55  60) error_errno(_("failed to delete '%s'"), sb.buf);
e19831c94f 251)     die(_("unable to re-add worktree '%s'"), path);
68a6b3a1bd 793) die(_("cannot move a locked working tree, lock reason: 
%s\nuse 'move -f -f' to override or unlock first"),
f4143101cb 906) die(_("cannot remove a locked working tree, lock reason: 
%s\nuse 'remove -f -f' to override or unlock first"),

cache-tree.c
4592e6080f 762) BUG("%s with flags 0x%x should not be in cache-tree",
4592e6080f 770) BUG("bad subtree '%.*s'", entlen, name);
4592e6080f 785) BUG("cache-tree for path %.*s does not match. "

commit-graph.c
6cc017431c  247) return 0;

commit-reach.c
5227c38566  63) BUG("bad generation skip %8x > %8x at %s",
5227c38566 134) return ret;
5227c38566 282) return 1;
5227c38566 314) return ret;
5227c38566 317) return ret;
5227c38566 323) return ret;
1d614d41e5 395) return 0;
1d614d41e5 401) return 0;
1d614d41e5 405) return 0;
920f93ca1c 459) return CONTAINS_NO;
920f93ca1c 484) cutoff = c->generation;
b67f6b26e3 559) continue;
b67f6b26e3 569) from->objects[i].item->flags |= assign_flag;
b67f6b26e3 570) continue;
b67f6b26e3 576) result = 0;
b67f6b26e3 577) goto cleanup;

delta-islands.c
c8d521faf7  53) memcpy(b, old, size);
c8d521faf7  73) return 1;
c8d521faf7 118) return 0;
c8d521faf7 130) return 0;
c8d521faf7 187) b->refcount--;
c8d521faf7 188) b = kh_value(island_marks, pos) = island_bitmap_new(b);
c8d521faf7 202) continue;
c8d521faf7 212) obj = ((struct tag *)obj)->tagged;
c8d521faf7 213) if (obj) {
c8d521faf7 214) parse_object(the_repository, &obj->oid);
c8d521faf7 215) marks = create_or_get_island_marks(obj);
c8d521faf7 216) island_bitmap_set(marks, island_counter);
c8d521faf7 248) return;
c8d521faf7 268) progress_state = start_progress(_("Propagating island 
marks"), nr);
c8d521faf7 286) die(_("bad tree object %s"), oid_to_hex(&ent->idx.oid));
c8d521faf7 293) continue;
c8d521faf7 297) continue;
c8d521faf7 321) return config_error_nonbool(k);
c8d521faf7 330) die(_("failed to load island regex for '%s': %s"), k, 
re.buf);
c8d521faf7 386) warning(_("island regex from config has "
c8d521faf7 397) strbuf_addch(&island_name, '-');
c8d521faf7 433) continue;
c8d521faf7 436) list[dst] = list[src];

diff-lib.c
9001dc2a74 diff-lib.c 345)     (!oideq(oid, &old_entry->oid) || 
!oideq(&old_entry->oid, &new_entry->oid))) {

dir.c
c46c406ae1 2275) trace_performance_leave("read directory %.*s", len, path);

entry.c
b878579ae7 402) static void mark_colliding_entries(const struct checkout 
*state,
b878579ae7 405) int i, trust_ino = check_stat;
b878579ae7 411) ce->ce_flags |= CE_MATCHED;
b878579ae7 413) for (i = 0; i < state->istate->cache_nr; i++) {
b878579ae7 414) struct cache_entry *dup = state->istate->cache[i];
b878579ae7 416) if (dup == ce)
b878579ae7 417) break;
b878579ae7 419) if (dup->ce_flags & (CE_MATCHED | CE_VALID | 
CE_SKIP_WORKTREE))
b878579ae7 420) continue;
b878579ae7 422) if ((trust_ino && dup->ce_stat_data.sd_ino == st->st_ino) ||
b878579ae7 423)     (!trust_ino && !fspathcmp(ce->name, dup->name))) {
b878579ae7 424) dup->ce_flags |= CE_MATCHED;
b878579ae7 425) break;
b878579ae7 428) }
b878579ae7 488) mark_colliding_entries(state, ce, &st);

fsck.c
fb8952077d  214) die_errno("Could not read '%s'", path);

ll-merge.c
d64324cb60 379) marker_size = DEFAULT_CONFLICT_MARKER_SIZE;

log-tree.c
4a7e27e957 477) if (oideq(&parent->item->object.oid, oid))

mailinfo.c
3aa4d81f88  992) len--;
3aa4d81f88  998) handle_filter(mi, prev);
3aa4d81f88  999) strbuf_reset(prev);
3aa4d81f88 1090) handle_filter(mi, &prev);

midx.c
4d80560c54   58) error_errno(_("failed to read %s"), midx_name);
4d80560c54   59) goto cleanup_fail;
4d80560c54   65) error(_("multi-pack-index file %s is too small"), 
midx_name);
4d80560c54   66) goto cleanup_fail;
0d5b3a5ef7  146) die(_("multi-pack-index missing required OID lookup 
chunk"));
662148c435  148) die(_("multi-pack-index missing required object offsets 
chunk"));
4d80560c54  173) munmap(midx_map, midx_size);
4d80560c54  175) close(fd);
a40498a126  188) close_pack(m->packs[i]);
a40498a126  189) free(m->packs);
3715a6335c  262) return 0;
3715a6335c  278) return 0;
c39b02ae0a  283) nth_midxed_object_oid(&oid, m, pos);
c39b02ae0a  284) for (i = 0; i < p->num_bad_objects; i++)
c39b02ae0a  285) if (!hashcmp(oid.hash,
c39b02ae0a  286)      p->bad_object_sha1 + the_hash_algo->rawsz * i))
c39b02ae0a  287) return 0;
c4d25228eb  341) return 1;
396f257018  398) warning(_("failed to add packfile '%s'"),
396f257018  400) return;
fe1ed56f5e  404) warning(_("failed to open pack-index '%s'"),
fe1ed56f5e  406) close_pack(packs->list[packs->nr]);
fe1ed56f5e  407) FREE_AND_NULL(packs->list[packs->nr]);
fe1ed56f5e  408) return;
a40498a126  481) return 1;
fe1ed56f5e  498) die(_("failed to locate object %d in packfile"), 
cur_object);
32f3c541e3  611) BUG("incorrect pack-file order: %s before %s",
0d5b3a5ef7  673) BUG("OIDs not in order: %s >= %s",
662148c435  700) BUG("object %s requires a large offset (%"PRIx64") but 
the MIDX is not writing large offsets!",
fc59e74844  754) die_errno(_("unable to create leading directories of %s"),
a40498a126  784) goto cleanup;
32f3c541e3  843) BUG("incorrect chunk offsets: %"PRIu64" before %"PRIu64,
32f3c541e3  848) BUG("chunk offset %"PRIu64" is not properly aligned",
32f3c541e3  860) BUG("incorrect chunk offset (%"PRIu64" != %"PRIu64") 
for chunk id %"PRIx32,
32f3c541e3  887) BUG("trying to write unknown chunk id %"PRIx32,
32f3c541e3  893) BUG("incorrect final offset %"PRIu64" != %"PRIu64,
525e18c04b  923) die(_("failed to clear multi-pack-index at %s"), midx);
56ee7ff156  949) return 0;
cc6af73c02  990) midx_report(_("failed to load pack-index for packfile %s"),
cc6af73c02  991)     e.p->pack_name);
cc6af73c02  992) break;

pack-bitmap.c
30cdc33fba 1130) return 0;

pack-objects.c
108f530385 169) REALLOC_ARRAY(pdata->tree_depth, pdata->nr_alloc);
fe0ac2fb7f 172) REALLOC_ARRAY(pdata->layer, pdata->nr_alloc);
108f530385 189) pdata->tree_depth[pdata->nr_objects - 1] = 0;
fe0ac2fb7f 192) pdata->layer[pdata->nr_objects - 1] = 0;

packfile.c
fe1ed56f5e  205) if (open_pack_index(p))
fe1ed56f5e  206) return 0;
fe1ed56f5e  207) level1_ofs = p->index_data;
17c35c8969  479) break;
17c35c8969  483) return error("packfile %s index unavailable", 
p->pack_name);
17c35c8969  537) return 0;

refs/packed-backend.c
9001dc2a74 1163) } else if (!oideq(&update->old_oid, iter->oid)) {

refs/ref-cache.c
9001dc2a74 275) if (!oideq(&ref1->u.value.oid, &ref2->u.value.oid))

rerere.c
2373b65059  216) die(_("corrupt MERGE_RR"));
2373b65059  225) die(_("corrupt MERGE_RR"));
2373b65059  228) die(_("corrupt MERGE_RR"));
2373b65059  263) die(_("unable to write rerere record"));
2373b65059  268) die(_("unable to write rerere record"));
4af32207bc  375) break;
4af32207bc  379) strbuf_addbuf(&two, &conflict);
c0f16f8e14  383) break;
c0f16f8e14  387) break;
c0f16f8e14  391) break;
2373b65059  476) return error_errno(_("could not open '%s'"), path);
2373b65059  481) error_errno(_("could not write '%s'"), output);
2373b65059  491) error(_("there were errors while writing '%s' (%s)"),
2373b65059  494) io.io.wrerror = error_errno(_("failed to flush '%s'"), 
path);
2373b65059  560) return error(_("index file corrupt"));
2373b65059  593) return error(_("index file corrupt"));
2373b65059  676) warning_errno(_("failed utime() on '%s'"),
2373b65059  682) return error_errno(_("could not open '%s'"), path);
2373b65059  684) error_errno(_("could not write '%s'"), path);
2373b65059  686) return error_errno(_("writing '%s' failed"), path);
2373b65059  712) die(_("unable to write new index file"));
2373b65059  794) die_errno(_("cannot unlink stray '%s'"), path);
2373b65059 1043) error(_("failed to update conflicted state in '%s'"), 
path);
2373b65059 1061) error(_("no remembered resolution for '%s'"), path);
2373b65059 1063) error_errno(_("cannot unlink '%s'"), filename);
2373b65059 1097) return error(_("index file corrupt"));
2373b65059 1185) die_errno(_("unable to open rr-cache directory"));

revision.c
4a7e27e957 3241)     oideq(&p->item->object.oid, &commit->object.oid))

sha1-file.c
67947c34ae sha1-file.c 2216) if (!hasheq(expected_sha1, real_sha1)) {

sha1-name.c
8aac67a174 sha1-name.c  154) static void unique_in_midx(struct 
multi_pack_index *m,
8aac67a174 sha1-name.c  157) uint32_t num, i, first = 0;
8aac67a174 sha1-name.c  158) const struct object_id *current = NULL;
8aac67a174 sha1-name.c  159) num = m->num_objects;
8aac67a174 sha1-name.c  161) if (!num)
8aac67a174 sha1-name.c  162) return;
8aac67a174 sha1-name.c  164) bsearch_midx(&ds->bin_pfx, m, &first);
8aac67a174 sha1-name.c  171) for (i = first; i < num && !ds->ambiguous; 
i++) {
8aac67a174 sha1-name.c  173) current = nth_midxed_object_oid(&oid, m, i);
8aac67a174 sha1-name.c  174) if (!match_sha(ds->len, ds->bin_pfx.hash, 
current->hash))
8aac67a174 sha1-name.c  175) break;
8aac67a174 sha1-name.c  176) update_candidates(ds, current);
8aac67a174 sha1-name.c  212)      m = m->next)
8aac67a174 sha1-name.c  213) unique_in_midx(m, ds);
8aac67a174 sha1-name.c  573) return;

trace.c
c46c406ae1 189) now = getnanotime();
c46c406ae1 190) perf_start_times[perf_indent] = now;
c46c406ae1 191) if (perf_indent + 1 < ARRAY_SIZE(perf_start_times))
c46c406ae1 192) perf_indent++;
c46c406ae1 194) BUG("Too deep indentation");
c46c406ae1 195) return now;
c46c406ae1 211) if (perf_indent >= strlen(space))
c46c406ae1 212) BUG("Too deep indentation");
c46c406ae1 214) strbuf_addf(&buf, ":%.*s ", perf_indent, space);
c46c406ae1 317) void trace_performance_leave_fl(const char *file, int line,
c46c406ae1 323) if (perf_indent)
c46c406ae1 324) perf_indent--;
c46c406ae1 326) if (!format) /* Allow callers to leave without tracing 
anything */
c46c406ae1 327) return;
c46c406ae1 329) since = perf_start_times[perf_indent];
c46c406ae1 330) va_start(ap, format);
c46c406ae1 331) trace_performance_vprintf_fl(file, line, nanos - since, 
format, ap);
c46c406ae1 332) va_end(ap);
c46c406ae1 477) trace_performance_leave("git command:%s", command_line.buf);
c46c406ae1 485) if (!command_line.len)
c46c406ae1 490) trace_performance_enter();

unpack-trees.c
b878579ae7  360) string_list_append(&list, ce->name);
b878579ae7  361) ce->ce_flags &= ~CE_MATCHED;
b878579ae7  368) warning(_("the following paths have collided (e.g. 
case-sensitive paths\n"
b878579ae7  372) for (i = 0; i < list.nr; i++)
b878579ae7  373) fprintf(stderr, "  '%s'\n", list.items[i].string);
b4da37380b  715) BUG("This is a directory and should not exist in index");
b4da37380b  719) BUG("pos must point at the first entry in this directory");
b4da37380b  740) BUG("We need cache-tree to do this optimization");
f1e11c6510  777) free(tree_ce);
b4da37380b  778) return rc;
b4da37380b  785) printf("Unpacked %d entries from %s to %s using 
cache-tree\n",
b4da37380b  787)        o->src_index->cache[pos]->name,
b4da37380b  788)        o->src_index->cache[pos + nr_entries - 1]->name);
b4da37380b  811) BUG("Wrong condition to get here buddy");

Commits introducing uncovered code:
Ben Peart      fa655d841: checkout: optimize "git checkout -b <new_branch>"
Christian Couder      108f53038: pack-objects: move tree_depth into 
'struct packing_data'
Christian Couder      fe0ac2fb7: pack-objects: move 'layer' into 'struct 
packing_data'
Derrick Stolee      0d5b3a5ef: midx: write object ids in a chunk
Derrick Stolee      17c35c896: packfile: skip loading index if in 
multi-pack-index
Derrick Stolee      1d614d41e: commit-reach: move ref_newer from remote.c
Derrick Stolee      32f3c541e: multi-pack-index: write pack names in chunk
Derrick Stolee      3715a6335: midx: read objects from multi-pack-index
Derrick Stolee      396f25701: multi-pack-index: read packfile list
Derrick Stolee      454ea2e4d: treewide: use get_all_packs
Derrick Stolee      4d80560c5: multi-pack-index: load into memory
Derrick Stolee      5227c3856: commit-reach: move walk methods from commit.c
Derrick Stolee      525e18c04: midx: clear midx on repack
Derrick Stolee      56ee7ff15: multi-pack-index: add 'verify' verb
Derrick Stolee      662148c43: midx: write object offsets
Derrick Stolee      66ec0390e: fsck: verify multi-pack-index
Derrick Stolee      6a22d5212: pack-objects: consider packs in 
multi-pack-index
Derrick Stolee      6cc017431: commit-reach: use can_all_from_reach
Derrick Stolee      6d68e6a46: multi-pack-index: provide more helpful 
usage info
Derrick Stolee      859fdc0c3: commit-graph: define GIT_TEST_COMMIT_GRAPH
Derrick Stolee      8aac67a17: midx: use midx in abbreviation calculations
Derrick Stolee      920f93ca1: commit-reach: move commit_contains from 
ref-filter
Derrick Stolee      a40498a12: midx: use existing midx when writing new one
Derrick Stolee      b67f6b26e: commit-reach: properly peel tags
Derrick Stolee      c39b02ae0: midx: mark bad packed objects
Derrick Stolee      c4d25228e: config: create core.multiPackIndex setting
Derrick Stolee      cc6af73c0: multi-pack-index: verify object offsets
Derrick Stolee      fc59e7484: midx: write header information to lockfile
Derrick Stolee      fe1ed56f5: midx: sort and deduplicate objects from 
packfiles
Duy Nguyen      b878579ae: clone: report duplicate entries on 
case-insensitive filesystems
Eric Sunshine      2e6fd71a5: format-patch: extend --range-diff to 
accept revision range
Eric Sunshine      40ce41604: format-patch: allow --range-diff to apply 
to a lone-patch
Eric Sunshine      68a6b3a1b: worktree: teach 'move' to override lock 
when --force given twice
Eric Sunshine      8631bf1cd: format-patch: add --creation-factor tweak 
for --range-diff
Eric Sunshine      e19831c94: worktree: teach 'add' to respect --force 
for registered but missing path
Eric Sunshine      e5353bef5: worktree: move delete_git_dir() earlier in 
file for upcoming new callers
Eric Sunshine      ee6cbf712: format-patch: allow --interdiff to apply 
to a lone-patch
Eric Sunshine      f4143101c: worktree: teach 'remove' to override lock 
when --force given twice
Jeff King      16d75fa48: repack: add delta-islands support
Jeff King      28b8a7308: pack-objects: add delta-islands support
Jeff King      30cdc33fb: pack-bitmap: save "have" bitmap from walk
Jeff King      4a7e27e95: convert "oidcmp() == 0" to oideq()
Jeff King      67947c34a: convert "hashcmp() != 0" to "!hasheq()"
Jeff King      9001dc2a7: convert "oidcmp() != 0" to "!oideq()"
Jeff King      c8d521faf: Add delta-islands.{c,h}
Nguyễn Thái Ngọc Duy      4592e6080: cache-tree: verify valid cache-tree 
in the test suite
Nguyễn Thái Ngọc Duy      b4da37380: unpack-trees: optimize walking same 
trees with cache-tree
Nguyễn Thái Ngọc Duy      c46c406ae: trace.h: support nested performance 
tracing
Nguyễn Thái Ngọc Duy      f1e11c651: unpack-trees: reduce malloc in 
cache-tree walk
René Scharfe      3aa4d81f8: mailinfo: support format=flowed
René Scharfe      fb8952077: fsck: use strbuf_getline() to read skiplist 
file
Shulhan      5025425df: builtin/remote: quote remote name on error to 
display empty name
Stefan Beller      74d4731da: submodule--helper: replace 
connect-gitdir-workingtree by ensure-core-worktree
Stefan Beller      ee69b2a90: submodule--helper: introduce new 
update-module-mode helper
Thomas Gummerer      2373b6505: rerere: mark strings for translation
Thomas Gummerer      4af32207b: rerere: teach rerere to handle nested 
conflicts
Thomas Gummerer      c0f16f8e1: rerere: factor out handle_conflict function
Torsten Bögershausen      d64324cb6: Make git_check_attr() a void function


[1] https://dev.azure.com/git/git/_build/results?buildId=172&view=logs
     Build running coverage-test for 'jch' and coverage-diff.sh against 
'next'

[2] https://git.visualstudio.com/git/_build/results?buildId=166&view=logs
     Build running coverage-test for 'next' and coverage-diff.sh against 
'master'

[3] https://dev.azure.com/git/git/_build/results?buildId=171&view=logs
     Build running coverage-test for 'master' and coverage-diff.sh 
against 'maint'

^ permalink raw reply	[relevance 2%]

* Re: What's cooking in git.git (Oct 2018, #02; Sat, 13)
      [irrelevant] <xmqqh8hr9pxb.fsf@gitster-ct.c.googlers.com>
@ 2018-10-12 19:44 ` Stefan Beller
  2018-10-12 23:37   ` Stefan Beller
  2018-10-13  1:03   ` Junio C Hamano
  0 siblings, 2 replies; 200+ results
From: Stefan Beller @ 2018-10-12 19:44 UTC (permalink / raw)
  To: Junio C Hamano; +Cc: git

>
> * sb/strbuf-h-update (2018-09-29) 1 commit
>  - strbuf.h: format according to coding guidelines
>
>  Code clean-up to serve as a BCP example.
>
>  What's the status of this one after the discussion thread stopped here?
>  cf. <CAGZ79kbV6QjsFKcD2uG_P9j1AvzSNQSi-_jXGQ9w0YU9fjhEGg@mail.gmail.com>

I started rewriting the documentation according to your proposal of having
parameters with name, then referred to as NAME in the docs.

After a few examples, I must admit I do not like that style,
so I would just want to do the minimal part that would get this patch landed,
i.e.
(a) convince you that the patch is good as-is or
(b) resend with fewer parameters made explicit if
    we desire to be concise instead.

> * nd/the-index (2018-09-21) 23 commits
>   (merged to 'next' on 2018-10-10 at 16e2e2e947)
[...]
>  (this branch is used by sb/more-repo-in-api.)
>
>  Various codepaths in the core-ish part learn to work on an
>  arbitrary in-core index structure, not necessarily the default
>  instance "the_index".
>
>  Will merge to 'master'.

Cool!

sb/more-repo-in-api is not part of this message,
but it requires at least one resend, so I'll do that.


> * sb/submodule-recursive-fetch-gets-the-tip (2018-10-11) 9 commits
>  . builtin/fetch: check for submodule updates for non branch fetches
>  . fetch: retry fetching submodules if needed objects were not fetched
>  . submodule: fetch in submodules git directory instead of in worktree
>  . repository: repo_submodule_init to take a submodule struct
>  . submodule.c: do not copy around submodule list
>  . submodule.c: move global changed_submodule_names into fetch submodule struct
>  . submodule.c: sort changed_submodule_names before searching it
>  . submodule.c: fix indentation
>  . sha1-array: provide oid_array_filter
>
>  "git fetch --recurse-submodules" may not fetch the necessary commit
>  that is bound to the superproject, which is getting corrected.
>
>  Ejected for now, as it has fallouts in places like t/helper/.

This is the first time I hear about that, I'll look into that.
The tipmost commit there is also shoddy, I'll redo that.

^ permalink raw reply	[relevance 2%]

* [PATCH] submodule helper: convert relative URL to absolute URL if needed
@ 2018-10-12 21:53 Stefan Beller
  2018-10-12 22:27 ` Jonathan Nieder
  2018-10-16  0:19 ` Stefan Beller
  0 siblings, 2 replies; 200+ results
From: Stefan Beller @ 2018-10-12 21:53 UTC (permalink / raw)
  To: git; +Cc: Stefan Beller

The submodule helper update_clone called by "git submodule update",
clones submodules if needed. As submodules used to have the URL indicating
if they were active, the step to resolve relative URLs was done in the
"submodule init" step. Nowadays submodules can be configured active without
calling an explicit init, e.g. via configuring submodule.active.

Then we'll fallback to the URL found in the .gitmodules, which may be
relative to the superproject, but we do not resolve it, yet.

To do so, factor out the function that resolves the relative URLs in
"git submodule init" (in the submodule helper in the init_submodule
function) and cal it at the appropriate place in the update_clone helper.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/submodule--helper.c | 48 ++++++++++++++++++++++++-------------
 t/t7400-submodule-basic.sh  | 24 +++++++++++++++++++
 2 files changed, 55 insertions(+), 17 deletions(-)

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index f6fb8991f3..a9a3ac38be 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -584,6 +584,27 @@ static int module_foreach(int argc, const char **argv, const char *prefix)
 	return 0;
 }
 
+
+char *compute_submodule_clone_url(const char *rel_url)
+{
+	char *remoteurl, *relurl;
+	char *remote = get_default_remote();
+	struct strbuf remotesb = STRBUF_INIT;
+
+	strbuf_addf(&remotesb, "remote.%s.url", remote);
+	free(remote);
+
+	if (git_config_get_string(remotesb.buf, &remoteurl)) {
+		warning(_("could not lookup configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);
+		remoteurl = xgetcwd();
+	}
+	relurl = relative_url(remoteurl, rel_url, NULL);
+	strbuf_release(&remotesb);
+	free(remoteurl);
+
+	return relurl;
+}
+
 struct init_cb {
 	const char *prefix;
 	unsigned int flags;
@@ -634,21 +655,9 @@ static void init_submodule(const char *path, const char *prefix,
 		/* Possibly a url relative to parent */
 		if (starts_with_dot_dot_slash(url) ||
 		    starts_with_dot_slash(url)) {
-			char *remoteurl, *relurl;
-			char *remote = get_default_remote();
-			struct strbuf remotesb = STRBUF_INIT;
-			strbuf_addf(&remotesb, "remote.%s.url", remote);
-			free(remote);
-
-			if (git_config_get_string(remotesb.buf, &remoteurl)) {
-				warning(_("could not lookup configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);
-				remoteurl = xgetcwd();
-			}
-			relurl = relative_url(remoteurl, url, NULL);
-			strbuf_release(&remotesb);
-			free(remoteurl);
-			free(url);
-			url = relurl;
+			char *to_free = url;
+			url = compute_submodule_clone_url(url);
+			free(to_free);
 		}
 
 		if (git_config_set_gently(sb.buf, url))
@@ -1562,8 +1571,13 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce,
 
 	strbuf_reset(&sb);
 	strbuf_addf(&sb, "submodule.%s.url", sub->name);
-	if (repo_config_get_string_const(the_repository, sb.buf, &url))
-		url = sub->url;
+	if (repo_config_get_string_const(the_repository, sb.buf, &url)) {
+		if (starts_with_dot_slash(sub->url) ||
+		    starts_with_dot_dot_slash(sub->url))
+			url = compute_submodule_clone_url(sub->url);
+		else
+			url = sub->url;
+	}
 
 	strbuf_reset(&sb);
 	strbuf_addf(&sb, "%s/.git", ce->name);
diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh
index c0ffc1022a..3f5dd5e4ef 100755
--- a/t/t7400-submodule-basic.sh
+++ b/t/t7400-submodule-basic.sh
@@ -1224,6 +1224,30 @@ test_expect_success 'submodule update and setting submodule.<name>.active' '
 	test_cmp expect actual
 '
 
+test_expect_success 'clone active submodule without submodule url set' '
+	test_when_finished "rm -rf test/test" &&
+	mkdir test &&
+	# another dir breaks accidental relative paths still being correct
+	git clone file://"$pwd"/multisuper test/test &&
+	(
+		cd test/test &&
+		git config submodule.active "." &&
+
+		# do not pass --init flag, as it is already active:
+		git submodule update &&
+		git submodule status >actual_raw &&
+
+		cut -c 1,43- actual_raw >actual &&
+		cat >expect <<-\EOF &&
+		 sub0 (test2)
+		 sub1 (test2)
+		 sub2 (test2)
+		 sub3 (test2)
+		EOF
+		test_cmp expect actual
+	)
+'
+
 test_expect_success 'clone --recurse-submodules with a pathspec works' '
 	test_when_finished "rm -rf multisuper_clone" &&
 	cat >expected <<-\EOF &&
-- 
2.19.0


^ permalink raw reply	[relevance 24%]

* Re: [PATCH] submodule helper: convert relative URL to absolute URL if needed
  2018-10-12 21:53 [PATCH] submodule helper: convert relative URL to absolute URL if needed Stefan Beller
@ 2018-10-12 22:27 ` Jonathan Nieder
  2018-10-16  0:19 ` Stefan Beller
  1 sibling, 0 replies; 200+ results
From: Jonathan Nieder @ 2018-10-12 22:27 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git, Jaewoong Jung

Hi,

Stefan Beller wrote:

> The submodule helper update_clone called by "git submodule update",
> clones submodules if needed. As submodules used to have the URL indicating
> if they were active, the step to resolve relative URLs was done in the
> "submodule init" step. Nowadays submodules can be configured active without
> calling an explicit init, e.g. via configuring submodule.active.
>
> Then we'll fallback to the URL found in the .gitmodules, which may be
> relative to the superproject, but we do not resolve it, yet.

Oh!  Good catch.

> To do so, factor out the function that resolves the relative URLs in
> "git submodule init" (in the submodule helper in the init_submodule
> function) and cal it at the appropriate place in the update_clone helper.

s/cal/call/

> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
>  builtin/submodule--helper.c | 48 ++++++++++++++++++++++++-------------
>  t/t7400-submodule-basic.sh  | 24 +++++++++++++++++++
>  2 files changed, 55 insertions(+), 17 deletions(-)

What is the symptom when this fails?

> diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
> index f6fb8991f3..a9a3ac38be 100644
> --- a/builtin/submodule--helper.c
> +++ b/builtin/submodule--helper.c
> @@ -584,6 +584,27 @@ static int module_foreach(int argc, const char **argv, const char *prefix)
>  	return 0;
>  }
>  
> +
> +char *compute_submodule_clone_url(const char *rel_url)

Should be static.

Is the caller responsible for freeing the returned buffer?

> +{
> +	char *remoteurl, *relurl;
> +	char *remote = get_default_remote();
> +	struct strbuf remotesb = STRBUF_INIT;

optional: could rename, something like

	struct strbuf key = STRBUF_INIT;

	remote = get_default_remote();
	strbuf_addf(&key, "remote.%s.url", remote);

 [...]
	strbuf_release(&key);
	free(remote);
	return result;


> +
> +	strbuf_addf(&remotesb, "remote.%s.url", remote);
> +	free(remote);
> +
> +	if (git_config_get_string(remotesb.buf, &remoteurl)) {
> +		warning(_("could not lookup configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);

nit: lookup is the noun, "look up" is the verb

> +		remoteurl = xgetcwd();
> +	}
> +	relurl = relative_url(remoteurl, rel_url, NULL);
> +	strbuf_release(&remotesb);
> +	free(remoteurl);
> +
> +	return relurl;
> +}
> +
>  struct init_cb {
>  	const char *prefix;
>  	unsigned int flags;
> @@ -634,21 +655,9 @@ static void init_submodule(const char *path, const char *prefix,
>  		/* Possibly a url relative to parent */
>  		if (starts_with_dot_dot_slash(url) ||
>  		    starts_with_dot_slash(url)) {
> -			char *remoteurl, *relurl;
> -			char *remote = get_default_remote();
> -			struct strbuf remotesb = STRBUF_INIT;
> -			strbuf_addf(&remotesb, "remote.%s.url", remote);
> -			free(remote);
> -
> -			if (git_config_get_string(remotesb.buf, &remoteurl)) {
> -				warning(_("could not lookup configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);
> -				remoteurl = xgetcwd();
> -			}
> -			relurl = relative_url(remoteurl, url, NULL);
> -			strbuf_release(&remotesb);
> -			free(remoteurl);
> -			free(url);
> -			url = relurl;

Ah, this is moved code. I should have used --color-moved. ;-)

> +			char *to_free = url;
> +			url = compute_submodule_clone_url(url);
> +			free(to_free);

Maybe:

			char *old_url = url;
			url = ...(old_url);
			free(old_url);

>  		}
>  
>  		if (git_config_set_gently(sb.buf, url))
> @@ -1562,8 +1571,13 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce,
>  
>  	strbuf_reset(&sb);
>  	strbuf_addf(&sb, "submodule.%s.url", sub->name);
> -	if (repo_config_get_string_const(the_repository, sb.buf, &url))
> -		url = sub->url;
> +	if (repo_config_get_string_const(the_repository, sb.buf, &url)) {
> +		if (starts_with_dot_slash(sub->url) ||
> +		    starts_with_dot_dot_slash(sub->url))
> +			url = compute_submodule_clone_url(sub->url);
> +		else
> +			url = sub->url;
> +	}

Nice.

>  
>  	strbuf_reset(&sb);
>  	strbuf_addf(&sb, "%s/.git", ce->name);
> diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh
> index c0ffc1022a..3f5dd5e4ef 100755
> --- a/t/t7400-submodule-basic.sh
> +++ b/t/t7400-submodule-basic.sh
> @@ -1224,6 +1224,30 @@ test_expect_success 'submodule update and setting submodule.<name>.active' '
>  	test_cmp expect actual
>  '
>  
> +test_expect_success 'clone active submodule without submodule url set' '

Thanks for the test \o/.

> +	test_when_finished "rm -rf test/test" &&
> +	mkdir test &&
> +	# another dir breaks accidental relative paths still being correct
> +	git clone file://"$pwd"/multisuper test/test &&
> +	(
> +		cd test/test &&
> +		git config submodule.active "." &&
> +
> +		# do not pass --init flag, as it is already active:

What does "it" refer to here?

> +		git submodule update &&
> +		git submodule status >actual_raw &&
> +
> +		cut -c 1,43- actual_raw >actual &&
> +		cat >expect <<-\EOF &&
> +		 sub0 (test2)
> +		 sub1 (test2)
> +		 sub2 (test2)
> +		 sub3 (test2)
> +		EOF
> +		test_cmp expect actual
> +	)
> +'
> +
>  test_expect_success 'clone --recurse-submodules with a pathspec works' '
>  	test_when_finished "rm -rf multisuper_clone" &&
>  	cat >expected <<-\EOF &&

Thanks for the quick fix.

Jonathan

^ permalink raw reply	[relevance 5%]

* Re: What's cooking in git.git (Oct 2018, #02; Sat, 13)
  2018-10-12 19:44 ` What's cooking in git.git (Oct 2018, #02; Sat, 13) Stefan Beller
@ 2018-10-12 23:37   ` Stefan Beller
  2018-10-13  1:03   ` Junio C Hamano
  1 sibling, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-12 23:37 UTC (permalink / raw)
  To: Junio C Hamano; +Cc: git

On Fri, Oct 12, 2018 at 12:44 PM Stefan Beller <sbeller@google.com> wrote:

> > * sb/submodule-recursive-fetch-gets-the-tip (2018-10-11) 9 commits
> >  . builtin/fetch: check for submodule updates for non branch fetches
> >  . fetch: retry fetching submodules if needed objects were not fetched
> >  . submodule: fetch in submodules git directory instead of in worktree
> >  . repository: repo_submodule_init to take a submodule struct
> >  . submodule.c: do not copy around submodule list
> >  . submodule.c: move global changed_submodule_names into fetch submodule struct
> >  . submodule.c: sort changed_submodule_names before searching it
> >  . submodule.c: fix indentation
> >  . sha1-array: provide oid_array_filter
> >
> >  "git fetch --recurse-submodules" may not fetch the necessary commit
> >  that is bound to the superproject, which is getting corrected.
> >
> >  Ejected for now, as it has fallouts in places like t/helper/.
>
> This is the first time I hear about that, I'll look into that.

I looked into that, and merging with origin/next only
had one merge conflict in submodule.c, easy to resolve.
It builds and tests cleanly after that.

What fallouts did you observe?

> The tipmost commit there is also shoddy, I'll redo that.

I confused this series with sb/submodule-update-in-C
which got merged already, I may send a fixup commit there.

So it seems to me that this series is still good.

Thanks,
Stefan

^ permalink raw reply	[relevance 8%]

* Re: [PATCH 17/19] submodule: use submodule repos for object lookup
      [irrelevant]   ` <20181011224052.191281-1-jonathantanmy@google.com>
@ 2018-10-13  0:20     ` Stefan Beller
  2018-10-16 19:30     ` Stefan Beller
  1 sibling, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-13  0:20 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: git

On Thu, Oct 11, 2018 at 3:41 PM Jonathan Tan <jonathantanmy@google.com> wrote:
>
> > +/*
> > + * Initialize 'out' based on the provided submodule path.
> > + *
> > + * Unlike repo_submodule_init, this tolerates submodules not present
> > + * in .gitmodules. NEEDSWORK: The repo_submodule_init behavior is
> > + * preferrable. This function exists only to preserve historical behavior.
>
> What do you mean by "The repo_submodule_init behavior is preferable"?

Preferable in the sense that it follows the definition of a submodule, but this
here works for "any repo" that happens to be at the gitlink.

>  If
> we need to preserve historical behavior, then it seems that the most
> preferable one is something that meets our needs (like open_submodule()
> in this patch).

Yes, I'll reword to drop the preferrable, but still state the difference.

I wonder if instead we'd want to introduce a

    repo_submodule_init(struct repository *submodule \
        struct repository *superproject \
        const char *path, \
        int tolerate_lookalikes)

Another patch proposes to replace the path
by a struct submodule, but for lookalikes, we do not have
a struct submodule to begin with (though in the other
patches we cook up a fake entry in the submodule config)

> > +static int open_submodule(struct repository *out, const char *path)
> > +{
> > +     struct strbuf sb = STRBUF_INIT;
> > +
> > +     if (submodule_to_gitdir(&sb, path) || repo_init(out, sb.buf, NULL)) {
> > +             strbuf_release(&sb);
> > +             return -1;
> > +     }
> > +
> > +     out->submodule_prefix = xstrdup(path);
>
> Do we need to set submodule_prefix?

Good point! Thanks for catching this.

>
> > @@ -507,7 +533,7 @@ static void show_submodule_header(struct diff_options *o, const char *path,
> >       else if (is_null_oid(two))
> >               message = "(submodule deleted)";
> >
> > -     if (add_submodule_odb(path)) {
> > +     if (open_submodule(sub, path) < 0) {
>
> This function, as a side effect, writes the open repository to "sub" for
> its caller to use. I think it's better if its callers open "sub" and
> then pass it to show_submodule_header() if successful. If opening the
> submodule is expected to fail sometimes (like it seems here), then we
> can allow callers to also pass NULL, and document what happens when NULL
> is passed.

That looks intriguing, I'll take a look. Note that we also pass
in **left and **right to have it assigned in there.

>
> Also, repositories open in this way should also be freed.

Yes, thanks!

^ permalink raw reply	[relevance 9%]

* Re: What's cooking in git.git (Oct 2018, #02; Sat, 13)
  2018-10-12 19:44 ` What's cooking in git.git (Oct 2018, #02; Sat, 13) Stefan Beller
  2018-10-12 23:37   ` Stefan Beller
@ 2018-10-13  1:03   ` Junio C Hamano
  2018-10-15 17:32     ` Stefan Beller
  1 sibling, 1 reply; 200+ results
From: Junio C Hamano @ 2018-10-13  1:03 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

Stefan Beller <sbeller@google.com> writes:

>> * sb/submodule-recursive-fetch-gets-the-tip (2018-10-11) 9 commits
>>  . builtin/fetch: check for submodule updates for non branch fetches
>>  . fetch: retry fetching submodules if needed objects were not fetched
>>  . submodule: fetch in submodules git directory instead of in worktree
>>  . repository: repo_submodule_init to take a submodule struct
>>  . submodule.c: do not copy around submodule list
>>  . submodule.c: move global changed_submodule_names into fetch submodule struct
>>  . submodule.c: sort changed_submodule_names before searching it
>>  . submodule.c: fix indentation
>>  . sha1-array: provide oid_array_filter
>>
>>  "git fetch --recurse-submodules" may not fetch the necessary commit
>>  that is bound to the superproject, which is getting corrected.
>>
>>  Ejected for now, as it has fallouts in places like t/helper/.
>
> This is the first time I hear about that, I'll look into that.
> The tipmost commit there is also shoddy, I'll redo that.

This is the first time I saw the breakage with this series, but I
would not be suprised, as this was rerolled recently.  Who knows
what got changed in this series and in other topics---any new
interaction can arise and that is a normal part of distributed
development.

The xx/sb-submodule-recursive-fetch-gets-the-tip-in-pu branch at
git://github.com/gitster/git.git has a merge of this into 'pu', with
textual conflicts all resolved.  

At least t/helper/test-submodule-nested-repo-config.c fails to
build; I didn't check if there are other breakages.


^ permalink raw reply	[relevance 5%]

* [PATCH v2 00/15] Hash function transition part 15
@ 2018-10-15  0:01 brian m. carlson
  0 siblings, 0 replies; 200+ results
From: brian m. carlson @ 2018-10-15  0:01 UTC (permalink / raw)
  To: git
  Cc: Jeff King, Eric Sunshine, Nguyễn Thái Ngọc Duy,
	Stefan Beller

This is the fifteenth series in the ongoing hash function transition.

This series includes several conversions to use the_hash_algo, combined
with some use of parse_oid_hex and GIT_MAX_RAWSZ.

Changes from v1:
* Fix several other substantially similar issues in builtin/repack.
* Fix a comment style issue.
* Improve commit message as suggested by Stefan.
* Pull in Gábor's patch and place it at the beginning of the series.

SZEDER Gábor (1):
  object_id.cocci: match only expressions of type 'struct object_id'

brian m. carlson (14):
  pack-bitmap-write: use GIT_MAX_RAWSZ for allocation
  builtin/repack: replace hard-coded constants
  builtin/mktree: remove hard-coded constant
  builtin/fetch-pack: remove constants with parse_oid_hex
  pack-revindex: express constants in terms of the_hash_algo
  packfile: express constants in terms of the_hash_algo
  refs/packed-backend: express constants using the_hash_algo
  upload-pack: express constants in terms of the_hash_algo
  transport: use parse_oid_hex instead of a constant
  tag: express constant in terms of the_hash_algo
  apply: replace hard-coded constants
  apply: rename new_sha1_prefix and old_sha1_prefix
  submodule: make zero-oid comparison hash function agnostic
  rerere: convert to use the_hash_algo

 apply.c                            |  50 ++++++------
 builtin/fetch-pack.c               |  13 ++--
 builtin/mktree.c                   |   2 +-
 builtin/repack.c                   |  13 ++--
 contrib/coccinelle/object_id.cocci | 117 ++++++++++++++++-------------
 git-submodule.sh                   |   7 +-
 pack-bitmap-write.c                |   2 +-
 pack-revindex.c                    |  10 ++-
 packfile.c                         |   5 +-
 refs/packed-backend.c              |  14 ++--
 rerere.c                           |  81 ++++++++++----------
 tag.c                              |   2 +-
 transport.c                        |   7 +-
 upload-pack.c                      |  13 ++--
 14 files changed, 181 insertions(+), 155 deletions(-)

Range-diff against v1:
 -:  ---------- >  1:  35d9cefd9a object_id.cocci: match only expressions of type 'struct object_id'
 1:  223d6f6695 !  2:  bb2a15a6e9 pack-bitmap-write: use GIT_MAX_RAWSZ for allocation
    @@ -2,6 +2,7 @@
     
         pack-bitmap-write: use GIT_MAX_RAWSZ for allocation
     
    +    Reviewed-by: Stefan Beller <sbeller@google.com>
         Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net>
     
      diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c
 2:  bbdb147d8d <  -:  ---------- builtin/repack: replace hard-coded constant
 -:  ---------- >  3:  95bdac161e builtin/repack: replace hard-coded constants
 3:  bf7b55fe4e =  4:  2a28675cfc builtin/mktree: remove hard-coded constant
 4:  155451f608 =  5:  7941bb0060 builtin/fetch-pack: remove constants with parse_oid_hex
 5:  9ec7065d62 !  6:  2ec9a22ea7 pack-revindex: express constants in terms of the_hash_algo
    @@ -3,7 +3,9 @@
         pack-revindex: express constants in terms of the_hash_algo
     
         Express the various constants used in terms of the_hash_algo.
    +    While we're at it, fix a comment style issue as well.
     
    +    Reviewed-by: Stefan Beller <sbeller@google.com>
         Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net>
     
      diff --git a/pack-revindex.c b/pack-revindex.c
    @@ -37,7 +39,8 @@
      	}
      
     -	/* This knows the pack format -- the 20-byte trailer
    -+	/* This knows the pack format -- the hash trailer
    ++	/*
    ++	 * This knows the pack format -- the hash trailer
      	 * follows immediately after the last object data.
      	 */
     -	p->revindex[num_ent].offset = p->pack_size - 20;
 6:  5e8576c6e4 !  7:  3ccb2b7217 packfile: express constants in terms of the_hash_algo
    @@ -5,6 +5,11 @@
         Replace uses of GIT_SHA1_RAWSZ with references to the_hash_algo to avoid
         dependence on a particular hash length.
     
    +    It's likely that in the future, we'll update the pack format to indicate
    +    what hash algorithm it uses, and then this code will change.  However,
    +    at least on an interim basis, make it easier to develop on a pure
    +    SHA-256 Git by using the_hash_algo here.
    +
         Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net>
     
      diff --git a/packfile.c b/packfile.c
 7:  4c7b6471db =  8:  39eb7e1069 refs/packed-backend: express constants using the_hash_algo
 8:  8318dcb630 =  9:  55269d5fc2 upload-pack: express constants in terms of the_hash_algo
 9:  16916c9fa2 = 10:  d30536a3bd transport: use parse_oid_hex instead of a constant
10:  9d8e2bc4ae = 11:  2139fe1fe1 tag: express constant in terms of the_hash_algo
11:  58f91f2167 = 12:  af815c4215 apply: replace hard-coded constants
12:  6899dfc4af = 13:  f651b226c8 apply: rename new_sha1_prefix and old_sha1_prefix
13:  cc974651cb = 14:  bf1d450aa5 submodule: make zero-oid comparison hash function agnostic
14:  b373f16c12 = 15:  d984036c1c rerere: convert to use the_hash_algo

^ permalink raw reply	[relevance 5%]

* Git Test Coverage Report (Monday, Oct 15)
@ 2018-10-15 16:24 Derrick Stolee
  0 siblings, 0 replies; 200+ results
From: Derrick Stolee @ 2018-10-15 16:24 UTC (permalink / raw)
  To: Git List

In an effort to ensure new code is reasonably covered by the test suite, 
we now have contrib/coverage-diff.sh to combine the gcov output from 
'make coverage-test ; make coverage-report' with the output from 'git 
diff A B' to discover _new_ lines of code that are not covered. This 
report ignores lines including "BUG(".

This report takes the output of these results after running on four 
branches:

         pu: 78bfaf7bc1efe6892fe4dbedf9ed80f9dd48146c

        jch: d67e018e0f57ebbb4fa0354a58a0a6d47e25a948

       next: 152ad8e3369ac77026886a2910e3a407c281df35

     master: 5a0cc8aca797dbd7d2be3b67458ff880ed45cddf

      maint: 04861070400d3ca9d487bd0d736ca818b9ffe371

I ran the test suite on each of these branches on an Ubuntu Linux VM, 
and I'm missing some dependencies (like apache, svn, and perforce) so 
not all tests are run.

I submit this output without comment. I'm taking a look especially at my 
own lines to see where coverage can be improved.

Thanks,

-Stolee

Uncovered code in 'pu' (12227c8) and not in 'jch' (22f2f0f)
-----------------------------------------------------------
builtin/blame.c
74e8221b52 builtin/blame.c    924) blame_date_width = sizeof("Thu Oct 19 
16:00");
74e8221b52 builtin/blame.c    925) break;

builtin/branch.c
ba8ba9df26 builtin/branch.c 452) die(_("could not resolve HEAD"));
ba8ba9df26 builtin/branch.c 458) die(_("HEAD (%s) points outside of 
refs/heads/"), refname);

builtin/grep.c
a8ace17bd4 builtin/grep.c  439) grep_read_unlock();

builtin/help.c
e6e76baaf4 builtin/help.c 429) if (!exclude_guides || alias[0] == '!') {
e6e76baaf4 builtin/help.c 430) printf_ln(_("'%s' is aliased to '%s'"), 
cmd, alias);
e6e76baaf4 builtin/help.c 431) free(alias);
e6e76baaf4 builtin/help.c 432) exit(0);
e6e76baaf4 builtin/help.c 441) fprintf_ln(stderr, _("'%s' is aliased to 
'%s'"), cmd, alias);
e6e76baaf4 builtin/help.c 442) count = split_cmdline(alias, &argv);
e6e76baaf4 builtin/help.c 443) if (count < 0)
e6e76baaf4 builtin/help.c 444) die(_("bad alias.%s string: %s"), cmd,
e6e76baaf4 builtin/help.c 446) free(argv);
e6e76baaf4 builtin/help.c 448) return alias;

builtin/rebase--interactive.c
6424061be4 builtin/rebase--interactive.c   23) return 
error_errno(_("could not read '%s'."), todo_file);
6424061be4 builtin/rebase--interactive.c   28) return 
error_errno(_("could not write '%s'"), todo_file);
7ccfac40bc builtin/rebase--interactive.c   43) return 
error_errno(_("could not read '%s'."), todo_file);
7ccfac40bc builtin/rebase--interactive.c   46) 
todo_list_release(&todo_list);
7ccfac40bc builtin/rebase--interactive.c   47) return error(_("unusable 
todo list: '%s'"), todo_file);
9787d17d40 builtin/rebase--interactive.c  294) ret = 
rearrange_squash_in_todo_file();

date.c
74e8221b52  113) die("Timestamp too large for this system: %"PRItime, time);
74e8221b52  216) if (tm->tm_mon == human_tm->tm_mon) {
74e8221b52  217) if (tm->tm_mday > human_tm->tm_mday) {
74e8221b52  219) } else if (tm->tm_mday == human_tm->tm_mday) {
74e8221b52  220) hide.date = hide.wday = 1;
74e8221b52  221) } else if (tm->tm_mday + 5 > human_tm->tm_mday) {
74e8221b52  223) hide.date = 1;
74e8221b52  231) gettimeofday(&now, NULL);
74e8221b52  232) show_date_relative(time, tz, &now, buf);
74e8221b52  233) return;
74e8221b52  246) hide.seconds = 1;
74e8221b52  247) hide.tz |= !hide.date;
74e8221b52  248) hide.wday = hide.time = !hide.year;
74e8221b52  262) strbuf_rtrim(buf);
74e8221b52  287) gettimeofday(&now, NULL);
74e8221b52  290) human_tz = local_time_tzoffset(now.tv_sec, &human_tm);
74e8221b52  886) static int auto_date_style(void)
74e8221b52  888) return (isatty(1) || pager_in_use()) ? DATE_HUMAN : 
DATE_NORMAL;
74e8221b52  909) return DATE_HUMAN;
74e8221b52  911) return auto_date_style();

git.c
a9a60b94cc 322) fprintf_ln(stderr, _("'%s' is aliased to '%s'"),

rebase-interactive.c
b74a37a5a7  26) warning(_("unrecognized setting %s for option "
6424061be4 107) return error_errno(_("could not write '%s''"), todo_file);
6424061be4 110) return error(_("could not copy '%s' to '%s'."), todo_file,
b74a37a5a7 174) goto leave_check;

sequencer.c
b5d6062402 4425) strbuf_insert(buf, todo_list->items[insert].offset_in_buf +
b5d6062402 4437) int sequencer_add_exec_commands(const char *commands)
06d8136126 4444) return error_errno(_("could not read '%s'."), todo_file);
b5d6062402 4446) if (todo_list_parse_insn_buffer(todo_list.buf.buf, 
&todo_list)) {
b5d6062402 4451) todo_list_add_exec_commands(&todo_list, commands);
b5d6062402 4452) res = write_message(todo_list.buf.buf, 
todo_list.buf.len, todo_file, 0);
0cce4a2756 4453) todo_list_release(&todo_list);
b5d6062402 4455) return res;
b74a37a5a7 4515) goto out;
b74a37a5a7 4520) goto out;
b8dac44d10 4660) todo_list_release(&new_todo);
009173ed7b 4665) todo_list_release(&new_todo);
009173ed7b 4666) return error_errno(_("could not write '%s'"), todo_file);
9787d17d40 4859) int rearrange_squash_in_todo_file(void)
9787d17d40 4861) const char *todo_file = rebase_path_todo();
9787d17d40 4862) struct todo_list todo_list = TODO_LIST_INIT;
9787d17d40 4863) int res = 0;
9787d17d40 4865) if (strbuf_read_file_or_whine(&todo_list.buf, 
todo_file) < 0)
9787d17d40 4866) return -1;
9787d17d40 4867) if (todo_list_parse_insn_buffer(todo_list.buf.buf, 
&todo_list) < 0) {
9787d17d40 4868) todo_list_release(&todo_list);
9787d17d40 4869) return -1;
9787d17d40 4872) res = todo_list_rearrange_squash(&todo_list);
9787d17d40 4873) if (!res)
9787d17d40 4874) res = rewrite_file(todo_file, todo_list.buf.buf, 
todo_list.buf.len);
9787d17d40 4876) todo_list_release(&todo_list);

submodule-config.c
bcbc780d14 739) return CONFIG_INVALID_KEY;
45f5ef3d77 754) warning(_("Could not update .gitmodules entry %s"), key);

wrapper.c
5efde212fc  70) die("Out of memory, malloc failed (tried to allocate %" 
PRIuMAX " bytes)",
5efde212fc  73) error("Out of memory, malloc failed (tried to allocate 
%" PRIuMAX " bytes)",

Commits introducing uncovered code:
Alban Gruin      009173ed7: sequencer: change complete_action() to use 
the refactored functions
Alban Gruin      06d813612: sequencer: fix a call to error() in 
transform_todo_file()
Alban Gruin      6424061be: rebase-interactive: rewrite edit_todo_list() 
to handle the initial edit
Alban Gruin      7ccfac40b: rebase--interactive: move 
transform_todo_file() to rebase--interactive.c
Alban Gruin      9787d17d4: sequencer: refactor rearrange_squash() to 
work on a todo_list
Alban Gruin      b5d606240: sequencer: refactor 
sequencer_add_exec_commands() to work on a todo_list
Alban Gruin      b74a37a5a: sequencer: refactor check_todo_list() to 
work on a todo_list
Alban Gruin      b8dac44d1: sequencer: refactor skip_unnecessary_picks() 
to work on a todo_list
Antonio Ospite      45f5ef3d7: submodule: factor out a 
config_set_in_gitmodules_file_gently function
Antonio Ospite      a8ace17bd: submodule: support reading .gitmodules 
when it's not in the working tree
Antonio Ospite      bcbc780d1: submodule: add a 
print_config_from_gitmodules() helper
Daniels Umanovskis      ba8ba9df2: branch: introduce --show-current 
display option
Liam Beguin      0cce4a275: rebase -i -x: add exec commands via the 
rebase--helper
Linus Torvalds      74e8221b5: Add 'human' date format
Martin Koegler      5efde212f: zlib.c: use size_t for size
Rasmus Villemoes      a9a60b94c: git.c: handle_alias: prepend alias info 
when first argument is -h
Rasmus Villemoes      e6e76baaf: help: redirect to aliased commands for 
"git cmd --help"

Uncovered code in 'jch' (22f2f0f) and not in 'next' (152ad8e)
-------------------------------------------------------------
apply.c
eccb5a5f3d 4071) return get_oid_hex(p->old_oid_prefix, oid);

builtin/fetch.c
7bbc53a589 builtin/fetch.c  385) continue; /* can this happen??? */

builtin/rebase--interactive.c
53bbcfbde7 builtin/rebase--interactive2.c  24) return error(_("no HEAD?"));
53bbcfbde7 builtin/rebase--interactive2.c  51) return 
error_errno(_("could not create temporary %s"), path_state_dir());
53bbcfbde7 builtin/rebase--interactive2.c  57) return 
error_errno(_("could not mark as interactive"));
53bbcfbde7 builtin/rebase--interactive2.c  77) return -1;
53bbcfbde7 builtin/rebase--interactive2.c  81) return -1;
53bbcfbde7 builtin/rebase--interactive2.c  87) free(revisions);
53bbcfbde7 builtin/rebase--interactive2.c  88) free(shortrevisions);
53bbcfbde7 builtin/rebase--interactive2.c  90) return -1;
53bbcfbde7 builtin/rebase--interactive2.c  98) free(revisions);
53bbcfbde7 builtin/rebase--interactive2.c  99) free(shortrevisions);
53bbcfbde7 builtin/rebase--interactive2.c 101) return 
error_errno(_("could not open %s"), rebase_path_todo());
53bbcfbde7 builtin/rebase--interactive2.c 106) 
argv_array_push(&make_script_args, restrict_revision);
53bbcfbde7 builtin/rebase--interactive2.c 114) error(_("could not 
generate todo list"));
53bbcfbde7 builtin/rebase--interactive2.c 206) 
usage_with_options(builtin_rebase_interactive_usage, options);
53bbcfbde7 builtin/rebase--interactive2.c 220) 
warning(_("--[no-]rebase-cousins has no effect without "
0af129b2ed builtin/rebase--interactive2.c 226) die(_("a base commit must 
be provided with --upstream or --onto"));
34b47315d9 builtin/rebase--interactive.c  261) ret = rearrange_squash();
34b47315d9 builtin/rebase--interactive.c  262) break;
34b47315d9 builtin/rebase--interactive.c  264) ret = 
sequencer_add_exec_commands(cmd);
34b47315d9 builtin/rebase--interactive.c  265) break;

builtin/rebase.c
55071ea248   61) strbuf_trim(&out);
55071ea248   62) ret = !strcmp("true", out.buf);
55071ea248   63) strbuf_release(&out);
002ee2fe68  115) die(_("%s requires an interactive rebase"), option);
f95736288a  148) return error_errno(_("could not read '%s'"), path);
f95736288a  162) return -1;
f95736288a  167) return error(_("could not get 'onto': '%s'"), buf.buf);
f95736288a  178) return -1;
f95736288a  179) } else if (read_one(state_dir_path("head", opts), &buf))
f95736288a  180) return -1;
f95736288a  182) return error(_("invalid orig-head: '%s'"), buf.buf);
f95736288a  186) return -1;
f95736288a  188) opts->flags &= ~REBASE_NO_QUIET;
73d51ed0a5  196) opts->signoff = 1;
73d51ed0a5  197) opts->flags |= REBASE_FORCE;
ead98c111b  204) return -1;
12026a412c  219) return -1;
ba1905a5fe  227) return -1;
ba1905a5fe  235) return -1;
6defce2b02  255) return error(_("Could not read '%s'"), path);
6defce2b02  271) res = error(_("Cannot store %s"), autostash.buf);
6defce2b02  275) return res;
bc24382c2b  373) argv_array_pushf(&child.args,
bc24382c2b  375) oid_to_hex(&opts->restrict_revision->object.oid));
ac7f467fef  507) struct strbuf dir = STRBUF_INIT;
6defce2b02  509) apply_autostash(opts);
ac7f467fef  510) strbuf_addstr(&dir, opts->state_dir);
ac7f467fef  511) remove_dir_recursively(&dir, 0);
ac7f467fef  512) strbuf_release(&dir);
ac7f467fef  513) die("Nothing to do");
ac7f467fef  543) return -1;
ac7f467fef  547) rollback_lock_file(&lock);
ac7f467fef  548) return error(_("could not determine HEAD revision"));
ac7f467fef  565) rollback_lock_file(&lock);
ac7f467fef  566) return error(_("could not read index"));
ac7f467fef  570) error(_("failed to find tree of %s"), oid_to_hex(oid));
ac7f467fef  571) rollback_lock_file(&lock);
ac7f467fef  572) free((void *)desc.buffer);
ac7f467fef  573) return -1;
ac7f467fef  586) ret = error(_("could not write index"));
ac7f467fef  590) return ret;
ac7f467fef  606) } else if (old_orig)
ac7f467fef  607) delete_ref(NULL, "ORIG_HEAD", old_orig, 0);
bff014dac7  635) opts->flags &= !REBASE_DIFFSTAT;
9a48a615b4  669) return 1;
9a48a615b4  685) return 0;
55071ea248  893) const char *path = mkpath("%s/git-legacy-rebase",
55071ea248  896) if (sane_execvp(path, (char **)argv) < 0)
55071ea248  897) die_errno(_("could not exec %s"), path);
0eabf4b95c  915) die(_("It looks like 'git am' is in progress. Cannot 
rebase."));
f28d40d3a9  952) usage_with_options(builtin_rebase_usage,
f95736288a  972) die(_("Cannot read HEAD"));
f95736288a  976) die(_("could not read index"));
f95736288a  990) exit(1);
122420c295 1002) die(_("could not discard worktree changes"));
122420c295 1004) exit(1);
5e5d96197c 1015) exit(1);
5e5d96197c 1018) die(_("could not move back to %s"),
5a61494539 1028) die(_("could not remove '%s'"), options.state_dir);
c54dacb50e 1047) const char *last_slash = strrchr(options.state_dir, '/');
c54dacb50e 1048) const char *state_dir_base =
c54dacb50e 1049) last_slash ? last_slash + 1 : options.state_dir;
c54dacb50e 1050) const char *cmd_live_rebase =
c54dacb50e 1052) strbuf_reset(&buf);
c54dacb50e 1053) strbuf_addf(&buf, "rm -fr \"%s\"", options.state_dir);
c54dacb50e 1054) die(_("It seems that there is already a %s directory, 
and\n"
53f9e5be94 1078) strbuf_addstr(&options.git_am_opt, " --ignore-date");
53f9e5be94 1079) options.flags |= REBASE_FORCE;
7998dbe1ec 1091) strbuf_addf(&options.git_am_opt, " -C%d", opt_c);
3c3588c7d3 1123) else if (strcmp("no-rebase-cousins", rebase_merges))
3c3588c7d3 1124) die(_("Unknown mode: %s"), rebase_merges);
ba1905a5fe 1146) die(_("--strategy requires --merge or --interactive"));
cda614e489 1164) strbuf_addstr(&options.git_format_patch_opt, " 
--progress");
ac7f467fef 1173) options.state_dir = apply_dir();
ac7f467fef 1174) break;
ac7f467fef 1251) die(_("invalid upstream '%s'"), options.upstream_name);
9dba809a69 1257) die(_("Could not create new root commit"));
e65123a71d 1307) die(_("fatal: no such branch/commit '%s'"),
ac7f467fef 1315) die(_("No such ref: %s"), "HEAD");
ac7f467fef 1327) die(_("Could not resolve HEAD to a revision"));
e0333e5c63 1340) die(_("could not read index"));
6defce2b02 1367) die(_("Cannot autostash"));
6defce2b02 1370) die(_("Unexpected stash response: '%s'"),
6defce2b02 1376) die(_("Could not create directory for '%s'"),
6defce2b02 1382) die(_("could not reset --hard"));
e65123a71d 1426) ret = !!error(_("could not parse '%s'"),
e65123a71d 1428) goto cleanup;
e65123a71d 1437) ret = !!error(_("could not switch to "
1ed9c14ff2 1447)  resolve_ref_unsafe("HEAD", 0, NULL, &flag))
1ed9c14ff2 1448) puts(_("HEAD is up to date."));
9a48a615b4 1457)  resolve_ref_unsafe("HEAD", 0, NULL, &flag))
9a48a615b4 1458) puts(_("HEAD is up to date, rebase forced."));

builtin/repack.c
2f0c9e9a9b 239) die("repack: Expecting full hex object ID lines only 
from pack-objects.");
2f0c9e9a9b 411) die("repack: Expecting full hex object ID lines only 
from pack-objects.");

builtin/stash.c
3d5ec65ce8 builtin/stash--helper.c  126) error(_("'%s' is not a 
stash-like commit"), revision);
3d5ec65ce8 builtin/stash--helper.c  127) free_stash_info(info);
3d5ec65ce8 builtin/stash--helper.c  128) exit(128);
3d5ec65ce8 builtin/stash--helper.c  161) free_stash_info(info);
3d5ec65ce8 builtin/stash--helper.c  162) fprintf_ln(stderr, _("No stash 
entries found."));
3d5ec65ce8 builtin/stash--helper.c  163) return -1;
3d5ec65ce8 builtin/stash--helper.c  198) free_stash_info(info);
7005771171 builtin/stash--helper.c  225) return error(_("git stash clear 
with parameters is "
3d5ec65ce8 builtin/stash--helper.c  241) return -1;
3d5ec65ce8 builtin/stash--helper.c  249) return -1;
3d5ec65ce8 builtin/stash--helper.c  262) return -1;
3d5ec65ce8 builtin/stash--helper.c  265) return error(_("unable to write 
new index file"));
3d5ec65ce8 builtin/stash--helper.c  377) remove_path(stash_index_path.buf);
3d5ec65ce8 builtin/stash--helper.c  378) return -1;
3d5ec65ce8 builtin/stash--helper.c  405) return -1;
3d5ec65ce8 builtin/stash--helper.c  408) return error(_("cannot apply a 
stash in the middle of a merge"));
3d5ec65ce8 builtin/stash--helper.c  418) strbuf_release(&out);
3d5ec65ce8 builtin/stash--helper.c  419) return error(_("Could not 
generate diff %s^!."),
3d5ec65ce8 builtin/stash--helper.c  426) return error(_("Conflicts in 
index."
3d5ec65ce8 builtin/stash--helper.c  432) return error(_("Could not save 
index tree"));
3d5ec65ce8 builtin/stash--helper.c  439) return error(_("could not 
restore untracked files from stash"));
3d5ec65ce8 builtin/stash--helper.c  470) return -1;
3d5ec65ce8 builtin/stash--helper.c  475) strbuf_release(&out);
3d5ec65ce8 builtin/stash--helper.c  480) strbuf_release(&out);
3d5ec65ce8 builtin/stash--helper.c  481) return -1;
7005771171 builtin/stash--helper.c  557) return error(_("%s: Could not 
drop stash entry"),
5bf62a19c0 builtin/stash--helper.c  632) printf_ln(_("The stash entry is 
kept in case "
104eb50d14 builtin/stash--helper.c  766) free_stash_info(&info);
193c3e3516 builtin/stash.c          767) 
usage_with_options(git_stash_show_usage, options);
813904a0ce builtin/stash--helper.c  783) stash_msg = "Created via \"git 
stash store\".";
813904a0ce builtin/stash--helper.c  789) if (!quiet) {
813904a0ce builtin/stash--helper.c  790) fprintf_ln(stderr, _("Cannot 
update %s with %s"),
813904a0ce builtin/stash--helper.c  793) return -1;
813904a0ce builtin/stash--helper.c  817) if (!quiet)
813904a0ce builtin/stash--helper.c  818) fprintf_ln(stderr, _("\"git 
stash store\" requires one "
813904a0ce builtin/stash--helper.c  820) return -1;
9f630e7480 builtin/stash--helper.c  902) return -1;
9f630e7480 builtin/stash--helper.c  962) ret = -1;
9f630e7480 builtin/stash--helper.c  963) goto done;
9f630e7480 builtin/stash--helper.c  968) ret = -1;
9f630e7480 builtin/stash--helper.c  969) goto done;
9f630e7480 builtin/stash--helper.c  974) ret = -1;
9f630e7480 builtin/stash--helper.c  975) goto done;
9f630e7480 builtin/stash--helper.c 1001) ret = -1;
9f630e7480 builtin/stash--helper.c 1002) goto done;
9f630e7480 builtin/stash--helper.c 1013) ret = -1;
9f630e7480 builtin/stash--helper.c 1014) goto done;
9f630e7480 builtin/stash--helper.c 1020) ret = -1;
9f630e7480 builtin/stash--helper.c 1021) goto done;
9f630e7480 builtin/stash--helper.c 1028) ret = -1;
9f630e7480 builtin/stash--helper.c 1029) goto done;
9f630e7480 builtin/stash--helper.c 1054) ret = -1;
9f630e7480 builtin/stash--helper.c 1055) goto done;
9f630e7480 builtin/stash--helper.c 1067) ret = -1;
9f630e7480 builtin/stash--helper.c 1068) goto done;
9f630e7480 builtin/stash--helper.c 1074) ret = -1;
9f630e7480 builtin/stash--helper.c 1075) goto done;
9f630e7480 builtin/stash--helper.c 1086) ret = -1;
9f630e7480 builtin/stash--helper.c 1087) goto done;
9f630e7480 builtin/stash--helper.c 1092) ret = -1;
9f630e7480 builtin/stash--helper.c 1093) goto done;
c2cc69f192 builtin/stash--helper.c 1128) fprintf_ln(stderr, _("You do 
not have "
9f630e7480 builtin/stash--helper.c 1137) ret = 1;
9f630e7480 builtin/stash--helper.c 1138) goto done;
c2cc69f192 builtin/stash--helper.c 1154) if (!quiet)
c2cc69f192 builtin/stash--helper.c 1155) fprintf_ln(stderr, _("Cannot 
save the current "
9f630e7480 builtin/stash--helper.c 1157) ret = -1;
9f630e7480 builtin/stash--helper.c 1158) goto done;
c2cc69f192 builtin/stash--helper.c 1163) if (!quiet)
c2cc69f192 builtin/stash--helper.c 1164) fprintf_ln(stderr, _("Cannot save "
9f630e7480 builtin/stash--helper.c 1166) ret = -1;
9f630e7480 builtin/stash--helper.c 1167) goto done;
c2cc69f192 builtin/stash--helper.c 1174) if (!quiet)
c2cc69f192 builtin/stash--helper.c 1175) fprintf_ln(stderr, _("Cannot 
save the current "
9f630e7480 builtin/stash--helper.c 1177) goto done;
c2cc69f192 builtin/stash--helper.c 1183) if (!quiet)
c2cc69f192 builtin/stash--helper.c 1184) fprintf_ln(stderr, _("Cannot 
save the current "
9f630e7480 builtin/stash--helper.c 1186) ret = -1;
9f630e7480 builtin/stash--helper.c 1187) goto done;
c2cc69f192 builtin/stash--helper.c 1213) if (!quiet)
c2cc69f192 builtin/stash--helper.c 1214) fprintf_ln(stderr, _("Cannot 
record "
9f630e7480 builtin/stash--helper.c 1216) ret = -1;
9f630e7480 builtin/stash--helper.c 1217) goto done;
1a0f0409a7 builtin/stash--helper.c 1289) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1290) goto done;
1a0f0409a7 builtin/stash--helper.c 1300) ret = -1;
c2cc69f192 builtin/stash--helper.c 1301) if (!quiet)
c2cc69f192 builtin/stash--helper.c 1302) fprintf_ln(stderr, _("Cannot 
initialize stash"));
1a0f0409a7 builtin/stash--helper.c 1303) goto done;
1a0f0409a7 builtin/stash--helper.c 1313) ret = -1;
c2cc69f192 builtin/stash--helper.c 1314) if (!quiet)
c2cc69f192 builtin/stash--helper.c 1315) fprintf_ln(stderr, _("Cannot 
save the current status"));
1a0f0409a7 builtin/stash--helper.c 1316) goto done;
1a0f0409a7 builtin/stash--helper.c 1333) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1352) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1353) goto done;
1a0f0409a7 builtin/stash--helper.c 1362) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1363) goto done;
1a0f0409a7 builtin/stash--helper.c 1371) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1380) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1391) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1392) goto done;
1a0f0409a7 builtin/stash--helper.c 1401) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1402) goto done;
1a0f0409a7 builtin/stash--helper.c 1410) ret = -1;
1a0f0409a7 builtin/stash--helper.c 1436) ret = -1;
193c3e3516 builtin/stash.c         1568) 
usage_msg_opt(xstrfmt(_("unknown subcommand: %s"), argv[0]),
193c3e3516 builtin/stash.c         1596) continue;

packfile.c
1127a98cce  117) return error("index file %s is too small", path);
1127a98cce  119) return error("empty data");

prio-queue.c
2d181390f3 94) return queue->array[queue->nr - 1].data;

rebase-interactive.c
64a43cbd5d 62) return error_errno(_("could not read '%s'."), todo_file);
64a43cbd5d 66) strbuf_release(&buf);
64a43cbd5d 67) return -1;
a9f5476fbc 75) return error_errno(_("could not read '%s'."), todo_file);
a9f5476fbc 79) strbuf_release(&buf);
a9f5476fbc 80) return -1;
64a43cbd5d 86) return -1;

revision.c
4943d28849 2931) return;
4943d28849 2934) return;
4943d28849 2937) c->object.flags |= UNINTERESTING;
4943d28849 2940) return;
4943d28849 2943) mark_parents_uninteresting(c);
4943d28849 2966) return;
4943d28849 2969) return;
4943d28849 2974) return;
4943d28849 3022) init_author_date_slab(&info->author_date);
4943d28849 3023) info->topo_queue.compare = compare_commits_by_author_date;
4943d28849 3024) info->topo_queue.cb_data = &info->author_date;
4943d28849 3025) break;
4943d28849 3038) continue;
4943d28849 3048) record_author_date(&info->author_date, c);
6c04ff3001 3086) if (!revs->ignore_missing_links)
6c04ff3001 3087) die("Failed to traverse parents of commit %s",
4943d28849 3088) oid_to_hex(&commit->object.oid));
4943d28849 3096) continue;

sequencer.c
65850686cf 2278) return;
65850686cf 2375) write_file(rebase_path_quiet(), "%s\n", quiet);
2c58483a59 3373) return error(_("could not checkout %s"), commit);
4df66c40b0 3387) return error(_("%s: not a valid OID"), orig_head);
71f82465b1 3407) fprintf(stderr, _("Stopped at HEAD\n"));
b97e187364 4771) return -1;
b97e187364 4774) return -1;
b97e187364 4780) return error_errno(_("could not read '%s'."), todo_file);
b97e187364 4783) todo_list_release(&todo_list);
b97e187364 4784) return error(_("unusable todo list: '%s'"), todo_file);
b97e187364 4803) todo_list_release(&todo_list);
b97e187364 4804) return -1;
b97e187364 4808) return error(_("could not copy '%s' to '%s'."), todo_file,
b97e187364 4812) return error(_("could not transform the todo list"));
b97e187364 4841) return error(_("could not transform the todo list"));
b97e187364 4844) return error(_("could not skip unnecessary pick 
commands"));
b97e187364 4850) return -1;

split-index.c
e3d837989e 335) ce->ce_flags |= CE_UPDATE_IN_BASE;

strbuf.c
f95736288a  127) --sb->len;

Commits introducing uncovered code:
Alban Gruin      0af129b2e: rebase--interactive2: rewrite the submodes 
of interactive rebase in C
Alban Gruin      2c58483a5: rebase -i: rewrite setup_reflog_action() in C
Alban Gruin      34b47315d: rebase -i: move rebase--helper modes to 
rebase--interactive
Alban Gruin      4df66c40b: rebase -i: rewrite checkout_onto() in C
Alban Gruin      53bbcfbde: rebase -i: implement the main part of 
interactive rebase as a builtin
Alban Gruin      64a43cbd5: rebase -i: rewrite the edit-todo 
functionality in C
Alban Gruin      65850686c: rebase -i: rewrite write_basic_state() in C
Alban Gruin      a9f5476fb: sequencer: refactor append_todo_help() to 
write its message to a buffer
Alban Gruin      b97e18736: rebase -i: rewrite complete_action() in C
brian m. carlson      2f0c9e9a9: builtin/repack: replace hard-coded 
constants
brian m. carlson      eccb5a5f3: apply: rename new_sha1_prefix and 
old_sha1_prefix
Derrick Stolee      2d181390f: prio-queue: add 'peek' operation
Derrick Stolee      4943d2884: revision.c: refactor basic topo-order logic
Derrick Stolee      6c04ff300: revision.c: begin refactoring 
--topo-order logic
Joel Teichroeb      3d5ec65ce: stash: convert apply to builtin
Joel Teichroeb      5bf62a19c: stash: convert pop to builtin
Joel Teichroeb      700577117: stash: convert drop and clear to builtin
Johannes Schindelin      71f82465b: rebase -i: introduce the 'break' command
Johannes Schindelin      bc24382c2: builtin rebase: prepare for builtin 
rebase -i
Josh Steadmon      1127a98cc: fuzz: add fuzz testing for packfile indices.
Junio C Hamano      7bbc53a58: fetch: replace string-list used as a 
look-up table with a hashmap
Paul-Sebastian Ungureanu      104eb50d1: stash: convert show to builtin
Paul-Sebastian Ungureanu      193c3e351: stash: convert 
`stash--helper.c` into `stash.c`
Paul-Sebastian Ungureanu      1a0f0409a: stash: convert push to builtin
Paul-Sebastian Ungureanu      813904a0c: stash: convert store to builtin
Paul-Sebastian Ungureanu      9f630e748: stash: convert create to builtin
Paul-Sebastian Ungureanu      c2cc69f19: stash: make push -q quiet
Pratik Karki      002ee2fe6: builtin rebase: support `keep-empty` option
Pratik Karki      0eabf4b95: builtin rebase: stop if `git am` is in progress
Pratik Karki      12026a412: builtin rebase: support `--gpg-sign` option
Pratik Karki      122420c29: builtin rebase: support --skip
Pratik Karki      1ed9c14ff: builtin rebase: support --force-rebase
Pratik Karki      3c3588c7d: builtin rebase: support 
--rebase-merges[=[no-]rebase-cousins]
Pratik Karki      53f9e5be9: builtin rebase: support `ignore-date` option
Pratik Karki      55071ea24: rebase: start implementing it as a builtin
Pratik Karki      5a6149453: builtin rebase: support --quit
Pratik Karki      5e5d96197: builtin rebase: support --abort
Pratik Karki      6defce2b0: builtin rebase: support `--autostash` option
Pratik Karki      73d51ed0a: builtin rebase: support --signoff
Pratik Karki      7998dbe1e: builtin rebase: support `-C` and 
`--whitespace=<type>`
Pratik Karki      9a48a615b: builtin rebase: try to fast forward when 
possible
Pratik Karki      9dba809a6: builtin rebase: support --root
Pratik Karki      ac7f467fe: builtin/rebase: support running "git rebase 
<upstream>"
Pratik Karki      ba1905a5f: builtin rebase: add support for custom 
merge strategies
Pratik Karki      bff014dac: builtin rebase: support the `verbose` and 
`diffstat` options
Pratik Karki      c54dacb50: builtin rebase: start a new rebase only if 
none is in progress
Pratik Karki      cda614e48: builtin rebase: show progress when 
connected to a terminal
Pratik Karki      e0333e5c6: builtin rebase: require a clean worktree
Pratik Karki      e65123a71: builtin rebase: support `git rebase 
<upstream> <switch-to>`
Pratik Karki      ead98c111: builtin rebase: support --rerere-autoupdate
Pratik Karki      f28d40d3a: builtin rebase: support --onto
Pratik Karki      f95736288: builtin rebase: support --continue
SZEDER Gábor      e3d837989: split-index: don't compare cached data of 
entries already marked for split index


Uncovered code in 'next' (152ad8e) and not in 'master' (5a0cc8a)
----------------------------------------------------------------
blame.c
a470beea39  113)  !strcmp(r->index->cache[-1 - pos]->name, path))
a470beea39  272) int pos = index_name_pos(r->index, path, len);
a470beea39  274) mode = r->index->cache[pos]->ce_mode;

builtin/am.c
2abf350385 1414) repo_init_revisions(the_repository, &rev_info, NULL);

builtin/archive.c
e001fd3a50 builtin/archive.c  78) die(_("git archive: expected ACK/NAK, 
got a flush packet"));
e001fd3a50 builtin/archive.c  80) if (starts_with(reader.line, "NACK "))
e001fd3a50 builtin/archive.c  81) die(_("git archive: NACK %s"), 
reader.line + 5);
e001fd3a50 builtin/archive.c  82) if (starts_with(reader.line, "ERR "))
e001fd3a50 builtin/archive.c  83) die(_("remote error: %s"), reader.line 
+ 4);
e001fd3a50 builtin/archive.c  84) die(_("git archive: protocol error"));
e001fd3a50 builtin/archive.c  89) die(_("git archive: expected a flush"));
fb19d32f05 builtin/archive.c  99) if (version != discover_version(&reader))
fb19d32f05 builtin/archive.c 100) die(_("git archive: received different 
protocol versions in subsequent requests"));

builtin/gc.c
3029970275 builtin/gc.c 461) ret = error_errno(_("cannot stat '%s'"), 
gc_log_path);
3029970275 builtin/gc.c 470) ret = error_errno(_("cannot read '%s'"), 
gc_log_path);
fec2ed2187 builtin/gc.c 495) die(FAILED_RUN, pack_refs_cmd.argv[0]);
fec2ed2187 builtin/gc.c 498) die(FAILED_RUN, reflog.argv[0]);
3029970275 builtin/gc.c 585) exit(128);
fec2ed2187 builtin/gc.c 637) die(FAILED_RUN, repack.argv[0]);
fec2ed2187 builtin/gc.c 647) die(FAILED_RUN, prune.argv[0]);
fec2ed2187 builtin/gc.c 654) die(FAILED_RUN, prune_worktrees.argv[0]);
fec2ed2187 builtin/gc.c 658) die(FAILED_RUN, rerere.argv[0]);

builtin/pack-objects.c
2fa233a554 builtin/pack-objects.c 1512) hashcpy(base_oid.hash, base_sha1);
2fa233a554 builtin/pack-objects.c 1513) if 
(!in_same_island(&delta->idx.oid, &base_oid))
2fa233a554 builtin/pack-objects.c 1514) return 0;

builtin/rev-list.c
7c0fe330d5 builtin/rev-list.c 227) die("unexpected missing %s object '%s'",
7c0fe330d5 builtin/rev-list.c 228)     type_name(obj->type), 
oid_to_hex(&obj->oid));

builtin/upload-archive.c
e001fd3a50 builtin/upload-archive.c 111) if (version == protocol_v0 || 
version == protocol_v1)
e001fd3a50 builtin/upload-archive.c 112) packet_write_fmt(1, "NACK 
unable to spawn subprocess\n");
e001fd3a50 builtin/upload-archive.c 113) else if (version == protocol_v2)
e001fd3a50 builtin/upload-archive.c 114) error_clnt("unable to spawn 
subprocess\n");

commit-graph.c
5cef295f28   67) return 0;
20fd6d5799   79) return 0;

config.c
c780b9cfe8 2298) return val;
c780b9cfe8 2301) if (is_bool)
c780b9cfe8 2302) return val ? 0 : 1;
c780b9cfe8 2304) return val;

diff.c
b78ea5fc35 4130) add_external_diff_name(o->repo, &argv, other, two);

help.c
26c7d06783 help.c         500) static int get_alias(const char *var, 
const char *value, void *data)
26c7d06783 help.c         502) struct string_list *list = data;
26c7d06783 help.c         504) if (skip_prefix(var, "alias.", &var))
26c7d06783 help.c         505) string_list_append(list, var)->util = 
xstrdup(value);
26c7d06783 help.c         507) return 0;
26c7d06783 help.c         530) printf("\n%s\n", _("Command aliases"));
26c7d06783 help.c         531) ALLOC_ARRAY(aliases, alias_list.nr + 1);
26c7d06783 help.c         532) for (i = 0; i < alias_list.nr; i++) {
26c7d06783 help.c         533) aliases[i].name = alias_list.items[i].string;
26c7d06783 help.c         534) aliases[i].help = alias_list.items[i].util;
26c7d06783 help.c         535) aliases[i].category = 1;
26c7d06783 help.c         537) aliases[alias_list.nr].name = NULL;
26c7d06783 help.c         538) print_command_list(aliases, 1, longest);
26c7d06783 help.c         539) free(aliases);

http-backend.c
fb19d32f05 646) argv[1] = ".";
fb19d32f05 647) argv[2] = NULL;

list-objects-filter-options.c
bc5975d24f  55) if (errbuf) {
bc5975d24f  56) strbuf_addstr(
bc5975d24f  60) return 1;
cc0b05a4cc  86) if (errbuf)

list-objects-filter.c
list-objects.c
f447a499db 197) ctx->show_object(obj, base->buf, ctx->show_data);

midx.c
e43d2dcce1  285) if (hasheq(oid.hash,
e43d2dcce1  286)    p->bad_object_sha1 + the_hash_algo->rawsz * i))

oidset.c
8b2f8cbcb1 29) kh_del_oid(&set->set, pos);
8b2f8cbcb1 30) return 1;

preload-index.c
ae9af12287  73) struct progress_data *pd = p->progress;
ae9af12287  75) pthread_mutex_lock(&pd->mutex);
ae9af12287  76) pd->n += last_nr - nr;
ae9af12287  77) display_progress(pd->progress, pd->n);
ae9af12287  78) pthread_mutex_unlock(&pd->mutex);
ae9af12287  79) last_nr = nr;
ae9af12287  93) struct progress_data *pd = p->progress;
ae9af12287  95) pthread_mutex_lock(&pd->mutex);
ae9af12287  96) display_progress(pd->progress, pd->n + last_nr);
ae9af12287  97) pthread_mutex_unlock(&pd->mutex);
ae9af12287 128) pd.progress = start_delayed_progress(_("Refreshing 
index"), index->cache_nr);
ae9af12287 129) pthread_mutex_init(&pd.mutex, NULL);
ae9af12287 140) p->progress = &pd;

read-cache.c
ae9af12287 1490) progress = start_delayed_progress(_("Refresh index"),
ae9af12287 1491)   istate->cache_nr);
ae9af12287 1533) display_progress(progress, i);
ae9af12287 1566) display_progress(progress, istate->cache_nr);
ae9af12287 1567) stop_progress(&progress);
252d079cbd 1778) const unsigned char *cp = (const unsigned char *)name;
252d079cbd 1782) strip_len = decode_varint(&cp);
77ff1127a4 1783) if (previous_ce) {
77ff1127a4 1784) previous_len = previous_ce->ce_namelen;
77ff1127a4 1785) if (previous_len < strip_len)
252d079cbd 1786) die(_("malformed name field in the index, near path '%s'"),
77ff1127a4 1787) previous_ce->name);
77ff1127a4 1788) copy_len = previous_len - strip_len;
77ff1127a4 1790) copy_len = 0;
252d079cbd 1792) name = (const char *)cp;
252d079cbd 1798) len += copy_len;
252d079cbd 1819) if (copy_len)
252d079cbd 1820) memcpy(ce->name, previous_ce->name, copy_len);
252d079cbd 1821) memcpy(ce->name + copy_len, name, len + 1 - copy_len);
252d079cbd 1822) *ent_size = (name - ((char *)ondisk)) + len + 1 - copy_len;
abb4bb8384 1959) munmap((void *)p->mmap, p->mmap_size);
abb4bb8384 1960) die(_("index file corrupt"));
77ff1127a4 2001) mem_pool_init(&istate->ce_mem_pool,
77ff1127a4 2041) static void *load_cache_entries_thread(void *_data)
77ff1127a4 2043) struct load_cache_entries_thread_data *p = _data;
77ff1127a4 2047) for (i = p->ieot_start; i < p->ieot_start + 
p->ieot_blocks; i++) {
77ff1127a4 2048) p->consumed += load_cache_entry_block(p->istate, 
p->ce_mem_pool,
77ff1127a4 2049) p->offset, p->ieot->entries[i].nr, p->mmap, 
p->ieot->entries[i].offset, NULL);
77ff1127a4 2050) p->offset += p->ieot->entries[i].nr;
77ff1127a4 2052) return NULL;
77ff1127a4 2055) static unsigned long load_cache_entries_threaded(struct 
index_state *istate, const char *mmap, size_t mmap_size,
77ff1127a4 2060) unsigned long consumed = 0;
77ff1127a4 2063) if (istate->name_hash_initialized)
77ff1127a4 2066) mem_pool_init(&istate->ce_mem_pool, 0);
77ff1127a4 2069) if (nr_threads > ieot->nr)
77ff1127a4 2070) nr_threads = ieot->nr;
77ff1127a4 2071) data = xcalloc(nr_threads, sizeof(*data));
77ff1127a4 2073) offset = ieot_start = 0;
77ff1127a4 2074) ieot_blocks = DIV_ROUND_UP(ieot->nr, nr_threads);
77ff1127a4 2075) for (i = 0; i < nr_threads; i++) {
77ff1127a4 2076) struct load_cache_entries_thread_data *p = &data[i];
77ff1127a4 2079) if (ieot_start + ieot_blocks > ieot->nr)
77ff1127a4 2080) ieot_blocks = ieot->nr - ieot_start;
77ff1127a4 2082) p->istate = istate;
77ff1127a4 2083) p->offset = offset;
77ff1127a4 2084) p->mmap = mmap;
77ff1127a4 2085) p->ieot = ieot;
77ff1127a4 2086) p->ieot_start = ieot_start;
77ff1127a4 2087) p->ieot_blocks = ieot_blocks;
77ff1127a4 2090) nr = 0;
77ff1127a4 2091) for (j = p->ieot_start; j < p->ieot_start + 
p->ieot_blocks; j++)
77ff1127a4 2092) nr += p->ieot->entries[j].nr;
77ff1127a4 2093) if (istate->version == 4) {
77ff1127a4 2094) mem_pool_init(&p->ce_mem_pool,
77ff1127a4 2097) mem_pool_init(&p->ce_mem_pool,
77ff1127a4 2101) err = pthread_create(&p->pthread, NULL, 
load_cache_entries_thread, p);
77ff1127a4 2102) if (err)
77ff1127a4 2103) die(_("unable to create load_cache_entries thread: 
%s"), strerror(err));
77ff1127a4 2106) for (j = 0; j < ieot_blocks; j++)
77ff1127a4 2107) offset += ieot->entries[ieot_start + j].nr;
77ff1127a4 2108) ieot_start += ieot_blocks;
77ff1127a4 2111) for (i = 0; i < nr_threads; i++) {
77ff1127a4 2112) struct load_cache_entries_thread_data *p = &data[i];
77ff1127a4 2114) err = pthread_join(p->pthread, NULL);
77ff1127a4 2115) if (err)
77ff1127a4 2116) die(_("unable to join load_cache_entries thread: %s"), 
strerror(err));
77ff1127a4 2117) mem_pool_combine(istate->ce_mem_pool, p->ce_mem_pool);
77ff1127a4 2118) consumed += p->consumed;
77ff1127a4 2121) free(data);
77ff1127a4 2123) return consumed;
77ff1127a4 2192) nr_threads = cpus;
abb4bb8384 2196) extension_offset = read_eoie_extension(mmap, mmap_size);
abb4bb8384 2197) if (extension_offset) {
abb4bb8384 2200) p.src_offset = extension_offset;
abb4bb8384 2201) err = pthread_create(&p.pthread, NULL, 
load_index_extensions, &p);
abb4bb8384 2202) if (err)
abb4bb8384 2203) die(_("unable to create load_index_extensions thread: 
%s"), strerror(err));
abb4bb8384 2205) nr_threads--;
77ff1127a4 2214) ieot = read_ieot_extension(mmap, mmap_size, 
extension_offset);
77ff1127a4 2217) src_offset += load_cache_entries_threaded(istate, mmap, 
mmap_size, src_offset, nr_threads, ieot);
77ff1127a4 2218) free(ieot);
abb4bb8384 2232) int ret = pthread_join(p.pthread, NULL);
abb4bb8384 2233) if (ret)
abb4bb8384 2234) die(_("unable to join load_index_extensions thread: 
%s"), strerror(ret));
3255089ada 2775) ieot_blocks = nr_threads;
77ff1127a4 2776) if (ieot_blocks > istate->cache_nr)
77ff1127a4 2777) ieot_blocks = istate->cache_nr;
3255089ada 2785) ieot = xcalloc(1, sizeof(struct index_entry_offset_table)
3255089ada 2786) + (ieot_blocks * sizeof(struct index_entry_offset)));
77ff1127a4 2787) ieot_entries = DIV_ROUND_UP(entries, ieot_blocks);
3255089ada 2794) free(ieot);
3b1d9e045e 2795) return -1;
3255089ada 2821) ieot->entries[ieot->nr].nr = nr;
3255089ada 2822) ieot->entries[ieot->nr].offset = offset;
3255089ada 2823) ieot->nr++;
3255089ada 2829) if (previous_name)
3255089ada 2830) previous_name->buf[0] = 0;
3255089ada 2831) nr = 0;
3255089ada 2832) offset = lseek(newfd, 0, SEEK_CUR);
3255089ada 2833) if (offset < 0) {
3255089ada 2834) free(ieot);
3255089ada 2835) return -1;
3255089ada 2837) offset += write_buffer_len;
3255089ada 2847) ieot->entries[ieot->nr].nr = nr;
3255089ada 2848) ieot->entries[ieot->nr].offset = offset;
3255089ada 2849) ieot->nr++;
3255089ada 2861) free(ieot);
3b1d9e045e 2862) return -1;
3255089ada 2876) struct strbuf sb = STRBUF_INIT;
3255089ada 2878) write_ieot_extension(&sb, ieot);
3255089ada 2879) err = write_index_ext_header(&c, &eoie_c, newfd, 
CACHE_EXT_INDEXENTRYOFFSETTABLE, sb.len) < 0
3255089ada 2880) || ce_write(&c, newfd, sb.buf, sb.len) < 0;
3255089ada 2881) strbuf_release(&sb);
3255089ada 2882) free(ieot);
3255089ada 2883) if (err)
3255089ada 2884) return -1;
3b1d9e045e 3372) static size_t read_eoie_extension(const char *mmap, 
size_t mmap_size)
3b1d9e045e 3390) if (mmap_size < sizeof(struct cache_header) + 
EOIE_SIZE_WITH_HEADER + the_hash_algo->rawsz)
3b1d9e045e 3391) return 0;
3b1d9e045e 3394) index = eoie = mmap + mmap_size - EOIE_SIZE_WITH_HEADER 
- the_hash_algo->rawsz;
3b1d9e045e 3395) if (CACHE_EXT(index) != CACHE_EXT_ENDOFINDEXENTRIES)
3b1d9e045e 3396) return 0;
3b1d9e045e 3397) index += sizeof(uint32_t);
3b1d9e045e 3400) extsize = get_be32(index);
3b1d9e045e 3401) if (extsize != EOIE_SIZE)
3b1d9e045e 3402) return 0;
3b1d9e045e 3403) index += sizeof(uint32_t);
3b1d9e045e 3409) offset = get_be32(index);
3b1d9e045e 3410) if (mmap + offset < mmap + sizeof(struct cache_header))
3b1d9e045e 3411) return 0;
3b1d9e045e 3412) if (mmap + offset >= eoie)
3b1d9e045e 3413) return 0;
3b1d9e045e 3414) index += sizeof(uint32_t);
3b1d9e045e 3425) src_offset = offset;
3b1d9e045e 3426) the_hash_algo->init_fn(&c);
3b1d9e045e 3427) while (src_offset < mmap_size - the_hash_algo->rawsz - 
EOIE_SIZE_WITH_HEADER) {
3b1d9e045e 3435) memcpy(&extsize, mmap + src_offset + 4, 4);
3b1d9e045e 3436) extsize = ntohl(extsize);
3b1d9e045e 3439) if (src_offset + 8 + extsize < src_offset)
3b1d9e045e 3440) return 0;
3b1d9e045e 3442) the_hash_algo->update_fn(&c, mmap + src_offset, 8);
3b1d9e045e 3444) src_offset += 8;
3b1d9e045e 3445) src_offset += extsize;
3b1d9e045e 3447) the_hash_algo->final_fn(hash, &c);
3b1d9e045e 3448) if (!hasheq(hash, (const unsigned char *)index))
3b1d9e045e 3449) return 0;
3b1d9e045e 3452) if (src_offset != mmap_size - the_hash_algo->rawsz - 
EOIE_SIZE_WITH_HEADER)
3b1d9e045e 3453) return 0;
3b1d9e045e 3455) return offset;
3255089ada 3475) static struct index_entry_offset_table 
*read_ieot_extension(const char *mmap, size_t mmap_size, size_t offset)
3255089ada 3477)        const char *index = NULL;
3255089ada 3483)        if (!offset)
3255089ada 3484)        return NULL;
3255089ada 3485)        while (offset <= mmap_size - 
the_hash_algo->rawsz - 8) {
3255089ada 3486)        extsize = get_be32(mmap + offset + 4);
3255089ada 3487)        if (CACHE_EXT((mmap + offset)) == 
CACHE_EXT_INDEXENTRYOFFSETTABLE) {
3255089ada 3488)        index = mmap + offset + 4 + 4;
3255089ada 3489)        break;
3255089ada 3491)        offset += 8;
3255089ada 3492)        offset += extsize;
3255089ada 3494)        if (!index)
3255089ada 3495)        return NULL;
3255089ada 3498)        ext_version = get_be32(index);
3255089ada 3499)        if (ext_version != IEOT_VERSION) {
3255089ada 3500)        error("invalid IEOT version %d", ext_version);
3255089ada 3501)        return NULL;
3255089ada 3503)        index += sizeof(uint32_t);
3255089ada 3506)        nr = (extsize - sizeof(uint32_t)) / 
(sizeof(uint32_t) + sizeof(uint32_t));
3255089ada 3507)        if (!nr) {
3255089ada 3508)        error("invalid number of IEOT entries %d", nr);
3255089ada 3509)        return NULL;
3255089ada 3511)        ieot = xmalloc(sizeof(struct 
index_entry_offset_table)
3255089ada 3512)        + (nr * sizeof(struct index_entry_offset)));
3255089ada 3513)        ieot->nr = nr;
3255089ada 3514)        for (i = 0; i < nr; i++) {
3255089ada 3515)        ieot->entries[i].offset = get_be32(index);
3255089ada 3516)        index += sizeof(uint32_t);
3255089ada 3517)        ieot->entries[i].nr = get_be32(index);
3255089ada 3518)        index += sizeof(uint32_t);
3255089ada 3521)        return ieot;
3255089ada 3524) static void write_ieot_extension(struct strbuf *sb, 
struct index_entry_offset_table *ieot)
3255089ada 3530)        put_be32(&buffer, IEOT_VERSION);
3255089ada 3531)        strbuf_add(sb, &buffer, sizeof(uint32_t));
3255089ada 3534)        for (i = 0; i < ieot->nr; i++) {
3255089ada 3537)        put_be32(&buffer, ieot->entries[i].offset);
3255089ada 3538)        strbuf_add(sb, &buffer, sizeof(uint32_t));
3255089ada 3541)        put_be32(&buffer, ieot->entries[i].nr);
3255089ada 3542)        strbuf_add(sb, &buffer, sizeof(uint32_t));
3255089ada 3544) }

refs.c
4a6067cda5 1405) return 0;

revision.c
2abf350385 1525) if (ce_path_match(istate, ce, &revs->prune_data, NULL)) {
2abf350385 1531) while ((i+1 < istate->cache_nr) &&
2abf350385 1532)        ce_same_name(ce, istate->cache[i+1]))

transport-helper.c
fb19d32f05  643) if (!data->connect && !data->stateless_connect)

transport.c
wt-status.c
f3bd35fa0d  671) s->committable = 1;
73ba5d78b4 1953) if (s->state.rebase_in_progress ||
73ba5d78b4 1954)     s->state.rebase_interactive_in_progress)
73ba5d78b4 1955) branch_name = s->state.onto;
73ba5d78b4 1956) else if (s->state.detached_from)
73ba5d78b4 1957) branch_name = s->state.detached_from;

Commits introducing uncovered code:
Ben Peart      3255089ad: ieot: add Index Entry Offset Table (IEOT) 
extension
Ben Peart      3b1d9e045: eoie: add End of Index Entry (EOIE) extension
Ben Peart      77ff1127a: read-cache: load cache entries on worker threads
Ben Peart      abb4bb838: read-cache: load cache extensions on a worker 
thread
Ben Peart      c780b9cfe: config: add new index.threads config setting
Derrick Stolee      20fd6d579: commit-graph: not compatible with grafts
Derrick Stolee      5cef295f2: commit-graph: not compatible with 
uninitialized repo
Jeff King      2fa233a55: pack-objects: handle island check for 
"external" delta base
Jeff King      e43d2dcce: more oideq/hasheq conversions
Jonathan Nieder      302997027: gc: do not return error for prior errors 
in daemonized mode
Jonathan Nieder      fec2ed218: gc: exit with status 128 on failure
Josh Steadmon      e001fd3a5: archive: implement protocol v2 archive command
Josh Steadmon      fb19d32f0: archive: allow archive over HTTP(S) with 
proto v2
Matthew DeVore      7c0fe330d: rev-list: handle missing tree objects 
properly
Matthew DeVore      bc5975d24: list-objects-filter: implement filter tree:0
Matthew DeVore      cc0b05a4c: list-objects-filter-options: do not 
over-strbuf_init
Matthew DeVore      f447a499d: list-objects: store common func args in 
struct
Nguyễn Thái Ngọc Duy      252d079cb: read-cache.c: optimize reading 
index format v4
Nguyễn Thái Ngọc Duy      26c7d0678: help -a: improve and make --verbose 
default
Nguyễn Thái Ngọc Duy      2abf35038: revision.c: remove implicit 
dependency on the_index
Nguyễn Thái Ngọc Duy      a470beea3: blame.c: rename "repo" argument to "r"
Nguyễn Thái Ngọc Duy      ae9af1228: status: show progress bar if 
refreshing the index takes too long
Nguyễn Thái Ngọc Duy      b78ea5fc3: diff.c: reduce implicit dependency 
on the_index
René Scharfe      8b2f8cbcb: oidset: use khash
Stefan Beller      4a6067cda: refs.c: migrate internal ref iteration to 
pass thru repository argument
Stephen P. Smith      73ba5d78b: roll wt_status_state into wt_status and 
populate in the collect phase
Stephen P. Smith      f3bd35fa0: wt-status.c: set the committable flag 
in the collect phase


Uncovered code in 'master' (5a0cc8a) and not in (fe8321ec05)
-----------------------------------------------------------------
builtin/fsck.c
66ec0390e7 builtin/fsck.c 862) midx_argv[2] = "--object-dir";
66ec0390e7 builtin/fsck.c 863) midx_argv[3] = alt->path;
66ec0390e7 builtin/fsck.c 864) if (run_command(&midx_verify))
66ec0390e7 builtin/fsck.c 865) errors_found |= ERROR_COMMIT_GRAPH;

fsck.c
fb8952077d  214) die_errno("Could not read '%s'", path);

midx.c
56ee7ff156  949) return 0;
cc6af73c02  990) midx_report(_("failed to load pack-index for packfile %s"),
cc6af73c02  991)     e.p->pack_name);
cc6af73c02  992) break;

Commits introducing uncovered code:
Derrick Stolee      56ee7ff15: multi-pack-index: add 'verify' verb
Derrick Stolee      66ec0390e: fsck: verify multi-pack-index
Derrick Stolee      cc6af73c0: multi-pack-index: verify object offsets
René Scharfe      fb8952077: fsck: use strbuf_getline() to read skiplist 
file


[1] https://dev.azure.com/git/git/_build/results?buildId=184&view=results
     Build running coverage-test for 'pu' and coverage-diff.sh against 'jch'

[2] https://dev.azure.com/git/git/_build/results?buildId=185&view=results
     Build running coverage-test for 'jch' and coverage-diff.sh against 
'next'

[3] https://dev.azure.com/git/git/_build/results?buildId=186&view=results
     Build running coverage-test for 'next' and coverage-diff.sh against 
'master'

[4] https://dev.azure.com/git/git/_build/results?buildId=187&view=logs
     Build running coverage-test for 'master' and coverage-diff.sh 
against 'master@{1}'

^ permalink raw reply	[relevance 1%]

* Re: What's cooking in git.git (Oct 2018, #02; Sat, 13)
  2018-10-13  1:03   ` Junio C Hamano
@ 2018-10-15 17:32     ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-15 17:32 UTC (permalink / raw)
  To: Junio C Hamano; +Cc: git

On Fri, Oct 12, 2018 at 6:03 PM Junio C Hamano <gitster@pobox.com> wrote:
>
> Stefan Beller <sbeller@google.com> writes:
>
> >> * sb/submodule-recursive-fetch-gets-the-tip (2018-10-11) 9 commits
> >>  . builtin/fetch: check for submodule updates for non branch fetches
> >>  . fetch: retry fetching submodules if needed objects were not fetched
> >>  . submodule: fetch in submodules git directory instead of in worktree
> >>  . repository: repo_submodule_init to take a submodule struct
> >>  . submodule.c: do not copy around submodule list
> >>  . submodule.c: move global changed_submodule_names into fetch submodule struct
> >>  . submodule.c: sort changed_submodule_names before searching it
> >>  . submodule.c: fix indentation
> >>  . sha1-array: provide oid_array_filter
> >>
> >>  "git fetch --recurse-submodules" may not fetch the necessary commit
> >>  that is bound to the superproject, which is getting corrected.
> >>
> >>  Ejected for now, as it has fallouts in places like t/helper/.
> >
> > This is the first time I hear about that, I'll look into that.
> > The tipmost commit there is also shoddy, I'll redo that.
>
> This is the first time I saw the breakage with this series, but I
> would not be suprised, as this was rerolled recently.  Who knows
> what got changed in this series and in other topics---any new
> interaction can arise and that is a normal part of distributed
> development.

I performed the same merge last week, and the range-diff indicates,
that your version of next was further ahead than mine.

The breakage itself comes from

t/helper/test-submodule-nested-repo-config.c: In function
‘cmd__submodule_nested_repo_config’:
t/helper/test-submodule-nested-repo-config.c:20:54: warning: passing
argument 3 of ‘repo_submodule_init’ from incompatible pointer type
[-Wincompatible-pointer-types]
  if (repo_submodule_init(&submodule, the_repository, argv[1])) {
                                                      ^~~~
In file included from ./cache.h:17:0,
                 from ./submodule-config.h:4,
                 from t/helper/test-submodule-nested-repo-config.c:2:
./repository.h:126:5: note: expected ‘const struct submodule *’ but
argument is of type ‘const char *’
 int repo_submodule_init(struct repository *subrepo,
     ^~~~~~~~~~~~~~~~~~~

which is introduced by
commit c369da44610acc5e56213b8784d4250ae619fdb9
  (origin/ao/submodule-wo-gitmodules-checked-out)
Author: Antonio Ospite <ao2@ao2.it>
Date:   2018-10-05 15:06

    t/helper: add test-submodule-nested-repo-config

    Add a test tool to exercise config_from_gitmodules(), in particular for
    the case of nested submodules.

    Add also a test to document that reading the submoudles config of nested
    submodules does not work yet when the .gitmodules file is not in the
    working tree but it still in the index.

    This is because the git API does not always make it possible access the
    object store  of an arbitrary repository (see get_oid() usage in
    config_from_gitmodules()).

    When this git limitation gets fixed the aforementioned use case will be
    supported too.

    Signed-off-by: Antonio Ospite <ao2@ao2.it>
    Signed-off-by: Junio C Hamano <gitster@pobox.com>

My resend will take that into account, building on Antonios series.

Thanks,
Stefan

^ permalink raw reply	[relevance 5%]

* [PATCH] submodule helper: convert relative URL to absolute URL if needed
  2018-10-12 21:53 [PATCH] submodule helper: convert relative URL to absolute URL if needed Stefan Beller
  2018-10-12 22:27 ` Jonathan Nieder
@ 2018-10-16  0:19 ` Stefan Beller
  2018-10-16  0:33   ` Jonathan Nieder
  1 sibling, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-16  0:19 UTC (permalink / raw)
  To: sbeller, gitster; +Cc: git, jrnieder

The submodule helper update_clone called by "git submodule update",
clones submodules if needed. As submodules used to have the URL indicating
if they were active, the step to resolve relative URLs was done in the
"submodule init" step. Nowadays submodules can be configured active without
calling an explicit init, e.g. via configuring submodule.active.

When trying to obtain submodules that are set active this way, we'll
fallback to the URL found in the .gitmodules, which may be relative to the
superproject, but we do not resolve it, yet:

    git clone https://gerrit.googlesource.com/gerrit
    cd gerrit && grep url .gitmodules
	url = ../plugins/codemirror-editor
	...
    git config submodule.active .
    git submodule update
fatal: repository '../plugins/codemirror-editor' does not exist
fatal: clone of '../plugins/codemirror-editor' into submodule path '/tmp/gerrit/plugins/codemirror-editor' failed
Failed to clone 'plugins/codemirror-editor'. Retry scheduled
[...]
fatal: clone of '../plugins/codemirror-editor' into submodule path '/tmp/gerrit/plugins/codemirror-editor' failed
Failed to clone 'plugins/codemirror-editor' a second time, aborting
[...]

To resolve the issue, factor out the function that resolves the relative
URLs in "git submodule init" (in the submodule helper in the init_submodule
function) and call it at the appropriate place in the update_clone helper.

Signed-off-by: Stefan Beller <sbeller@google.com>
---

Jonathan wrote:
> Ah, this is moved code. I should have used --color-moved. ;-)

Yes, any cleanup should go on top.

I extended the commit message and made sure not to leak memory.

When rerolling origin/xxx/sb-submodule-recursive-fetch-gets-the-tip-in-pu,
there will be conflicts with this series, but I can work with that.

 builtin/submodule--helper.c | 48 ++++++++++++++++++++++++-------------
 t/t7400-submodule-basic.sh  | 24 +++++++++++++++++++
 2 files changed, 55 insertions(+), 17 deletions(-)

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index f6fb8991f3..03f5e0d03e 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -584,6 +584,27 @@ static int module_foreach(int argc, const char **argv, const char *prefix)
 	return 0;
 }
 
+
+static char *compute_submodule_clone_url(const char *rel_url)
+{
+	char *remoteurl, *relurl;
+	char *remote = get_default_remote();
+	struct strbuf remotesb = STRBUF_INIT;
+
+	strbuf_addf(&remotesb, "remote.%s.url", remote);
+	free(remote);
+
+	if (git_config_get_string(remotesb.buf, &remoteurl)) {
+		warning(_("could not lookup configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);
+		remoteurl = xgetcwd();
+	}
+	relurl = relative_url(remoteurl, rel_url, NULL);
+	strbuf_release(&remotesb);
+	free(remoteurl);
+
+	return relurl;
+}
+
 struct init_cb {
 	const char *prefix;
 	unsigned int flags;
@@ -634,21 +655,9 @@ static void init_submodule(const char *path, const char *prefix,
 		/* Possibly a url relative to parent */
 		if (starts_with_dot_dot_slash(url) ||
 		    starts_with_dot_slash(url)) {
-			char *remoteurl, *relurl;
-			char *remote = get_default_remote();
-			struct strbuf remotesb = STRBUF_INIT;
-			strbuf_addf(&remotesb, "remote.%s.url", remote);
-			free(remote);
-
-			if (git_config_get_string(remotesb.buf, &remoteurl)) {
-				warning(_("could not lookup configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);
-				remoteurl = xgetcwd();
-			}
-			relurl = relative_url(remoteurl, url, NULL);
-			strbuf_release(&remotesb);
-			free(remoteurl);
-			free(url);
-			url = relurl;
+			char *to_free = url;
+			url = compute_submodule_clone_url(url);
+			free(to_free);
 		}
 
 		if (git_config_set_gently(sb.buf, url))
@@ -1562,8 +1571,13 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce,
 
 	strbuf_reset(&sb);
 	strbuf_addf(&sb, "submodule.%s.url", sub->name);
-	if (repo_config_get_string_const(the_repository, sb.buf, &url))
-		url = sub->url;
+	if (repo_config_get_string_const(the_repository, sb.buf, &url)) {
+		if (starts_with_dot_slash(sub->url) ||
+		    starts_with_dot_dot_slash(sub->url))
+			url = compute_submodule_clone_url(sub->url);
+		else
+			url = sub->url;
+	}
 
 	strbuf_reset(&sb);
 	strbuf_addf(&sb, "%s/.git", ce->name);
diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh
index c0ffc1022a..76a7cb0af7 100755
--- a/t/t7400-submodule-basic.sh
+++ b/t/t7400-submodule-basic.sh
@@ -1224,6 +1224,30 @@ test_expect_success 'submodule update and setting submodule.<name>.active' '
 	test_cmp expect actual
 '
 
+test_expect_success 'clone active submodule without submodule url set' '
+	test_when_finished "rm -rf test/test" &&
+	mkdir test &&
+	# another dir breaks accidental relative paths still being correct
+	git clone file://"$pwd"/multisuper test/test &&
+	(
+		cd test/test &&
+		git config submodule.active "." &&
+
+		# do not pass --init flag, as the submodule is already active:
+		git submodule update &&
+		git submodule status >actual_raw &&
+
+		cut -c 1,43- actual_raw >actual &&
+		cat >expect <<-\EOF &&
+		 sub0 (test2)
+		 sub1 (test2)
+		 sub2 (test2)
+		 sub3 (test2)
+		EOF
+		test_cmp expect actual
+	)
+'
+
 test_expect_success 'clone --recurse-submodules with a pathspec works' '
 	test_when_finished "rm -rf multisuper_clone" &&
 	cat >expected <<-\EOF &&
-- 
2.19.0


^ permalink raw reply	[relevance 22%]

* Re: [PATCH] submodule helper: convert relative URL to absolute URL if needed
  2018-10-16  0:19 ` Stefan Beller
@ 2018-10-16  0:33   ` Jonathan Nieder
  2018-10-16  5:15     ` Junio C Hamano
  0 siblings, 1 reply; 200+ results
From: Jonathan Nieder @ 2018-10-16  0:33 UTC (permalink / raw)
  To: Stefan Beller; +Cc: gitster, git, Jaewoong Jung

Hi,

Stefan Beller wrote:

> The submodule helper update_clone called by "git submodule update",
> clones submodules if needed. As submodules used to have the URL indicating
> if they were active, the step to resolve relative URLs was done in the
> "submodule init" step. Nowadays submodules can be configured active without
> calling an explicit init, e.g. via configuring submodule.active.
>
> When trying to obtain submodules that are set active this way, we'll
> fallback to the URL found in the .gitmodules, which may be relative to the
> superproject, but we do not resolve it, yet:
> 
>     git clone https://gerrit.googlesource.com/gerrit
>     cd gerrit && grep url .gitmodules
> 	url = ../plugins/codemirror-editor
> 	...
>     git config submodule.active .
>     git submodule update
> fatal: repository '../plugins/codemirror-editor' does not exist
> fatal: clone of '../plugins/codemirror-editor' into submodule path '/tmp/gerrit/plugins/codemirror-editor' failed
> Failed to clone 'plugins/codemirror-editor'. Retry scheduled
> [...]
> fatal: clone of '../plugins/codemirror-editor' into submodule path '/tmp/gerrit/plugins/codemirror-editor' failed
> Failed to clone 'plugins/codemirror-editor' a second time, aborting
> [...]

Thanks.

"git log" and other tools have the ability to rewrap lines and will get
confused by this transcript.  Can you indent it to unconfuse them?

> Signed-off-by: Stefan Beller <sbeller@google.com>

Please also credit the bug reporter:

Reported-by: Jaewoong Jung <jungjw@google.com>

[...]
> --- a/builtin/submodule--helper.c
> +++ b/builtin/submodule--helper.c
> @@ -584,6 +584,27 @@ static int module_foreach(int argc, const char **argv, const char *prefix)
>  	return 0;
>  }
>  
> +

nit: inconsistent vertical whitespace (extra blank line?)

> +static char *compute_submodule_clone_url(const char *rel_url)
> +{
> +	char *remoteurl, *relurl;
> +	char *remote = get_default_remote();
> +	struct strbuf remotesb = STRBUF_INIT;
> +
> +	strbuf_addf(&remotesb, "remote.%s.url", remote);
> +	free(remote);
> +
> +	if (git_config_get_string(remotesb.buf, &remoteurl)) {
> +		warning(_("could not lookup configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);
> +		remoteurl = xgetcwd();
> +	}
> +	relurl = relative_url(remoteurl, rel_url, NULL);
> +	strbuf_release(&remotesb);
> +	free(remoteurl);
> +
> +	return relurl;
> +}

I think this would be easier to read with all the release commands
together:

	...

	free(remote);
	free(remoteurl);
	strbuf_release(&remotesb);
	return relurl;

[...]
> @@ -634,21 +655,9 @@ static void init_submodule(const char *path, const char *prefix,
[...]
> -			relurl = relative_url(remoteurl, url, NULL);
> -			strbuf_release(&remotesb);
> -			free(remoteurl);
> -			free(url);
> -			url = relurl;
> +			char *to_free = url;
> +			url = compute_submodule_clone_url(url);
> +			free(to_free);

I still think this would be easier to read with a style that makes
the ownership passing clearer:

			char *oldurl = url;
			url = compute_submodule_clone_url(oldurl);
			free(oldurl);

With whatever subset of the above tweaks makes sense,
Reviewed-by: Jonathan Nieder <jrnieder@gmail.com>

Thanks.

^ permalink raw reply	[relevance 4%]

* Re: [PATCH] submodule helper: convert relative URL to absolute URL if needed
  2018-10-16  0:33   ` Jonathan Nieder
@ 2018-10-16  5:15     ` Junio C Hamano
  2018-10-16 17:27       ` Stefan Beller
  0 siblings, 1 reply; 200+ results
From: Junio C Hamano @ 2018-10-16  5:15 UTC (permalink / raw)
  To: Jonathan Nieder; +Cc: Stefan Beller, git, Jaewoong Jung

Jonathan Nieder <jrnieder@gmail.com> writes:

>>     git config submodule.active .
>>     git submodule update
>> fatal: repository '../plugins/codemirror-editor' does not exist
>> fatal: clone of '../plugins/codemirror-editor' into submodule path '/tmp/gerrit/plugins/codemirror-editor' failed
>> Failed to clone 'plugins/codemirror-editor'. Retry scheduled
>> [...]
>> fatal: clone of '../plugins/codemirror-editor' into submodule path '/tmp/gerrit/plugins/codemirror-editor' failed
>> Failed to clone 'plugins/codemirror-editor' a second time, aborting
>> [...]
>
> Thanks.
>
> "git log" and other tools have the ability to rewrap lines and will get
> confused by this transcript.  Can you indent it to unconfuse them?
>
>> Signed-off-by: Stefan Beller <sbeller@google.com>
>
> Please also credit the bug reporter:
>
> Reported-by: Jaewoong Jung <jungjw@google.com>
>
> ...
> nit: inconsistent vertical whitespace (extra blank line?)
> ...
> I think this would be easier to read with all the release commands
> together:
> ...

All good points.

^ permalink raw reply	[relevance 2%]

* [PATCH] submodule helper: convert relative URL to absolute URL if needed
  2018-10-16  5:15     ` Junio C Hamano
@ 2018-10-16 17:27       ` Stefan Beller
  2018-10-16 21:05         ` Jonathan Nieder
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-16 17:27 UTC (permalink / raw)
  To: gitster; +Cc: git, jrnieder, jungjw, sbeller

The submodule helper update_clone called by "git submodule update",
clones submodules if needed. As submodules used to have the URL indicating
if they were active, the step to resolve relative URLs was done in the
"submodule init" step. Nowadays submodules can be configured active without
calling an explicit init, e.g. via configuring submodule.active.

When trying to obtain submodules that are set active this way, we'll
fallback to the URL found in the .gitmodules, which may be relative to the
superproject, but we do not resolve it, yet:

    git clone https://gerrit.googlesource.com/gerrit
    cd gerrit && grep url .gitmodules
	url = ../plugins/codemirror-editor
	...
    git config submodule.active .
    git submodule update
    fatal: repository '../plugins/codemirror-editor' does not exist
    fatal: clone of '../plugins/codemirror-editor' into submodule path '/tmp/gerrit/plugins/codemirror-editor' failed
    Failed to clone 'plugins/codemirror-editor'. Retry scheduled
    [...]
    fatal: clone of '../plugins/codemirror-editor' into submodule path '/tmp/gerrit/plugins/codemirror-editor' failed
    Failed to clone 'plugins/codemirror-editor' a second time, aborting
    [...]

To resolve the issue, factor out the function that resolves the relative
URLs in "git submodule init" (in the submodule helper in the init_submodule
function) and call it at the appropriate place in the update_clone helper.

Reported-by: Jaewoong Jung <jungjw@google.com>
Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/submodule--helper.c | 51 ++++++++++++++++++++++++-------------
 t/t7400-submodule-basic.sh  | 24 +++++++++++++++++
 2 files changed, 58 insertions(+), 17 deletions(-)

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index f6fb8991f3..13c2e4b556 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -584,6 +584,26 @@ static int module_foreach(int argc, const char **argv, const char *prefix)
 	return 0;
 }
 
+static char *compute_submodule_clone_url(const char *rel_url)
+{
+	char *remoteurl, *relurl;
+	char *remote = get_default_remote();
+	struct strbuf remotesb = STRBUF_INIT;
+
+	strbuf_addf(&remotesb, "remote.%s.url", remote);
+	if (git_config_get_string(remotesb.buf, &remoteurl)) {
+		warning(_("could not look up configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);
+		remoteurl = xgetcwd();
+	}
+	relurl = relative_url(remoteurl, rel_url, NULL);
+
+	free(remote);
+	free(remoteurl);
+	strbuf_release(&remotesb);
+
+	return relurl;
+}
+
 struct init_cb {
 	const char *prefix;
 	unsigned int flags;
@@ -634,21 +654,9 @@ static void init_submodule(const char *path, const char *prefix,
 		/* Possibly a url relative to parent */
 		if (starts_with_dot_dot_slash(url) ||
 		    starts_with_dot_slash(url)) {
-			char *remoteurl, *relurl;
-			char *remote = get_default_remote();
-			struct strbuf remotesb = STRBUF_INIT;
-			strbuf_addf(&remotesb, "remote.%s.url", remote);
-			free(remote);
-
-			if (git_config_get_string(remotesb.buf, &remoteurl)) {
-				warning(_("could not lookup configuration '%s'. Assuming this repository is its own authoritative upstream."), remotesb.buf);
-				remoteurl = xgetcwd();
-			}
-			relurl = relative_url(remoteurl, url, NULL);
-			strbuf_release(&remotesb);
-			free(remoteurl);
-			free(url);
-			url = relurl;
+			char *oldurl = url;
+			url = compute_submodule_clone_url(oldurl);
+			free(oldurl);
 		}
 
 		if (git_config_set_gently(sb.buf, url))
@@ -1514,6 +1522,7 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce,
 	struct strbuf sb = STRBUF_INIT;
 	const char *displaypath = NULL;
 	int needs_cloning = 0;
+	int need_free_url = 0;
 
 	if (ce_stage(ce)) {
 		if (suc->recursive_prefix)
@@ -1562,8 +1571,14 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce,
 
 	strbuf_reset(&sb);
 	strbuf_addf(&sb, "submodule.%s.url", sub->name);
-	if (repo_config_get_string_const(the_repository, sb.buf, &url))
-		url = sub->url;
+	if (repo_config_get_string_const(the_repository, sb.buf, &url)) {
+		if (starts_with_dot_slash(sub->url) ||
+		    starts_with_dot_dot_slash(sub->url)) {
+			url = compute_submodule_clone_url(sub->url);
+			need_free_url = 1;
+		} else
+			url = sub->url;
+	}
 
 	strbuf_reset(&sb);
 	strbuf_addf(&sb, "%s/.git", ce->name);
@@ -1608,6 +1623,8 @@ static int prepare_to_clone_next_submodule(const struct cache_entry *ce,
 cleanup:
 	strbuf_reset(&displaypath_sb);
 	strbuf_reset(&sb);
+	if (need_free_url)
+		free((void*)url);
 
 	return needs_cloning;
 }
diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh
index c0ffc1022a..76a7cb0af7 100755
--- a/t/t7400-submodule-basic.sh
+++ b/t/t7400-submodule-basic.sh
@@ -1224,6 +1224,30 @@ test_expect_success 'submodule update and setting submodule.<name>.active' '
 	test_cmp expect actual
 '
 
+test_expect_success 'clone active submodule without submodule url set' '
+	test_when_finished "rm -rf test/test" &&
+	mkdir test &&
+	# another dir breaks accidental relative paths still being correct
+	git clone file://"$pwd"/multisuper test/test &&
+	(
+		cd test/test &&
+		git config submodule.active "." &&
+
+		# do not pass --init flag, as the submodule is already active:
+		git submodule update &&
+		git submodule status >actual_raw &&
+
+		cut -c 1,43- actual_raw >actual &&
+		cat >expect <<-\EOF &&
+		 sub0 (test2)
+		 sub1 (test2)
+		 sub2 (test2)
+		 sub3 (test2)
+		EOF
+		test_cmp expect actual
+	)
+'
+
 test_expect_success 'clone --recurse-submodules with a pathspec works' '
 	test_when_finished "rm -rf multisuper_clone" &&
 	cat >expected <<-\EOF &&
-- 
2.19.0


^ permalink raw reply	[relevance 22%]

* [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip
@ 2018-10-16 18:13 Stefan Beller
  2018-10-16 18:13 ` [PATCH 2/9] submodule.c: fix indentation Stefan Beller
                   ` (9 more replies)
  0 siblings, 10 replies; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

This is based on ao/submodule-wo-gitmodules-checked-out.

This resends origin/sb/submodule-recursive-fetch-gets-the-tip, resolving
the issues pointed out via origin/xxx/sb-submodule-recursive-fetch-gets-the-tip-in-pu
by basing this series on origin/ao/submodule-wo-gitmodules-checked-out

A range-diff below shows how picking a different base changed the patches,
apart from that no further adjustments have been made.

Thanks,
Stefan

Stefan Beller (9):
  sha1-array: provide oid_array_filter
  submodule.c: fix indentation
  submodule.c: sort changed_submodule_names before searching it
  submodule.c: move global changed_submodule_names into fetch submodule
    struct
  submodule.c: do not copy around submodule list
  repository: repo_submodule_init to take a submodule struct
  submodule: fetch in submodules git directory instead of in worktree
  fetch: retry fetching submodules if needed objects were not fetched
  builtin/fetch: check for submodule updates for non branch fetches

 Documentation/technical/api-oid-array.txt    |   5 +
 builtin/fetch.c                              |  14 +-
 builtin/grep.c                               |  17 +-
 builtin/ls-files.c                           |  12 +-
 repository.c                                 |  27 +-
 repository.h                                 |  11 +-
 sha1-array.c                                 |  17 ++
 sha1-array.h                                 |   3 +
 submodule.c                                  | 275 +++++++++++++++----
 t/helper/test-submodule-nested-repo-config.c |   8 +-
 t/t5526-fetch-submodules.sh                  |  23 +-
 11 files changed, 315 insertions(+), 97 deletions(-)

git range-diff origin/xxx/sb-submodule-recursive-fetch-gets-the-tip-in-pu...
[...]
585:  ac1f98a0df <   -:  ---------- doc: move git-rev-parse from porcelain to plumbing
586:  7cf1a0fbef =   1:  a035323c49 sha1-array: provide oid_array_filter
587:  01077381d0 =   2:  30ed20b4f0 submodule.c: fix indentation
588:  4b0cdf5899 =   3:  cd590ea88d submodule.c: sort changed_submodule_names before searching it
589:  78e5099ecc !   4:  ce959811ba submodule.c: move global changed_submodule_names into fetch submodule struct
    @@ -12,7 +12,7 @@
     --- a/submodule.c
     +++ b/submodule.c
     @@
    - #include "commit-reach.h"
    + #include "object-store.h"
      
      static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
     -static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
590:  d813f18bb3 =   5:  151f9a8ad4 submodule.c: do not copy around submodule list
591:  a077d63af7 !   6:  3a97743fa2 repository: repo_submodule_init to take a submodule struct
    @@ -15,7 +15,6 @@
         Also move its documentation into the header file.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/builtin/grep.c b/builtin/grep.c
     --- a/builtin/grep.c
    @@ -31,12 +30,16 @@
     +
      	int hit;
      
    - 	if (!is_submodule_active(superproject, path))
    + 	/*
    +@@
      		return 0;
    + 	}
      
    --	if (repo_submodule_init(&submodule, superproject, path))
    -+	if (repo_submodule_init(&subrepo, superproject, sub))
    +-	if (repo_submodule_init(&submodule, superproject, path)) {
    ++	if (repo_submodule_init(&subrepo, superproject, sub)) {
    + 		grep_read_unlock();
      		return 0;
    + 	}
      
     -	repo_read_gitmodules(&submodule);
     +	repo_read_gitmodules(&subrepo);
    @@ -44,9 +47,9 @@
      	/*
      	 * NEEDSWORK: This adds the submodule's object directory to the list of
     @@
    + 	 * store is no longer global and instead is a member of the repository
      	 * object.
      	 */
    - 	grep_read_lock();
     -	add_to_alternates_memory(submodule.objects->objectdir);
     +	add_to_alternates_memory(subrepo.objects->objectdir);
      	grep_read_unlock();
    @@ -100,19 +103,6 @@
      
      static void show_ce(struct repository *repo, struct dir_struct *dir,
     
    -diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
    ---- a/builtin/submodule--helper.c
    -+++ b/builtin/submodule--helper.c
    -@@
    - 	if (!sub)
    - 		BUG("We could get the submodule handle before?");
    - 
    --	if (repo_submodule_init(&subrepo, the_repository, path))
    -+	if (repo_submodule_init(&subrepo, the_repository, sub))
    - 		die(_("could not get a repository handle for submodule '%s'"), path);
    - 
    - 	if (!repo_config_get_string(&subrepo, "core.worktree", &cw)) {
    -
     diff --git a/repository.c b/repository.c
     --- a/repository.c
     +++ b/repository.c
    @@ -197,3 +187,32 @@
      void repo_clear(struct repository *repo);
      
      /*
    +
    +diff --git a/t/helper/test-submodule-nested-repo-config.c b/t/helper/test-submodule-nested-repo-config.c
    +--- a/t/helper/test-submodule-nested-repo-config.c
    ++++ b/t/helper/test-submodule-nested-repo-config.c
    +@@
    + 
    + int cmd__submodule_nested_repo_config(int argc, const char **argv)
    + {
    +-	struct repository submodule;
    ++	struct repository subrepo;
    ++	const struct submodule *sub;
    + 
    + 	if (argc < 3)
    + 		die_usage(argc, argv, "Wrong number of arguments.");
    + 
    + 	setup_git_directory();
    + 
    +-	if (repo_submodule_init(&submodule, the_repository, argv[1])) {
    ++	sub = submodule_from_path(the_repository, &null_oid, argv[1]);
    ++	if (repo_submodule_init(&subrepo, the_repository, sub)) {
    + 		die_usage(argc, argv, "Submodule not found.");
    + 	}
    + 
    + 	/* Read the config of _child_ submodules. */
    +-	print_config_from_gitmodules(&submodule, argv[2]);
    ++	print_config_from_gitmodules(&subrepo, argv[2]);
    + 
    + 	submodule_free(the_repository);
    + 
592:  780f6c1a92 =   7:  4e8ad61f8d submodule: fetch in submodules git directory instead of in worktree
593:  a530535912 =   8:  24bac00db7 fetch: retry fetching submodules if needed objects were not fetched
594:  a72bde3a8a =   9:  e031182e44 builtin/fetch: check for submodule updates for non branch fetches
[...]

^ permalink raw reply	[relevance 9%]

* [PATCH 2/9] submodule.c: fix indentation
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
@ 2018-10-16 18:13 ` Stefan Beller
  2018-10-16 18:13 ` [PATCH 3/9] submodule.c: sort changed_submodule_names before searching it Stefan Beller
                   ` (8 subsequent siblings)
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

The submodule subsystem is really bad at staying within 80 characters.
Fix it while we are here.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 submodule.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/submodule.c b/submodule.c
index 2b7082b2db..e145ebbb16 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1258,7 +1258,8 @@ static int get_next_submodule(struct child_process *cp,
 		if (!submodule) {
 			const char *name = default_name_or_path(ce->name);
 			if (name) {
-				default_submodule.path = default_submodule.name = name;
+				default_submodule.path = name;
+				default_submodule.name = name;
 				submodule = &default_submodule;
 			}
 		}
@@ -1268,8 +1269,10 @@ static int get_next_submodule(struct child_process *cp,
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names,
-							 submodule->name))
+			if (!submodule ||
+			    !unsorted_string_list_lookup(
+					&changed_submodule_names,
+					submodule->name))
 				continue;
 			default_argv = "on-demand";
 			break;
-- 
2.19.0


^ permalink raw reply	[relevance 26%]

* [PATCH 3/9] submodule.c: sort changed_submodule_names before searching it
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
  2018-10-16 18:13 ` [PATCH 2/9] submodule.c: fix indentation Stefan Beller
@ 2018-10-16 18:13 ` Stefan Beller
  2018-10-16 18:13 ` [PATCH 4/9] submodule.c: move global changed_submodule_names into fetch submodule struct Stefan Beller
                   ` (7 subsequent siblings)
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

We can string_list_insert() to maintain sorted-ness of the
list as we find new items, or we can string_list_append() to
build an unsorted list and sort it at the end just once.

As we do not rely on the sortedness while building the
list, we pick the "append and sort at the end" as it
has better worst case execution times.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 submodule.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/submodule.c b/submodule.c
index e145ebbb16..9fbfcfcfe1 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1270,7 +1270,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
-			    !unsorted_string_list_lookup(
+			    !string_list_lookup(
 					&changed_submodule_names,
 					submodule->name))
 				continue;
@@ -1364,6 +1364,7 @@ int fetch_populated_submodules(struct repository *r,
 	/* default value, "--submodule-prefix" and its value are added later */
 
 	calculate_changed_submodule_paths();
+	string_list_sort(&changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* [PATCH 4/9] submodule.c: move global changed_submodule_names into fetch submodule struct
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
  2018-10-16 18:13 ` [PATCH 2/9] submodule.c: fix indentation Stefan Beller
  2018-10-16 18:13 ` [PATCH 3/9] submodule.c: sort changed_submodule_names before searching it Stefan Beller
@ 2018-10-16 18:13 ` Stefan Beller
      [irrelevant]   ` <20181017212624.196598-1-jonathantanmy@google.com>
  2018-10-16 18:13 ` [PATCH 5/9] submodule.c: do not copy around submodule list Stefan Beller
                   ` (6 subsequent siblings)
  9 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

The `changed_submodule_names` are only used for fetching, so let's make it
part of the struct that is passed around for fetching submodules.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 submodule.c | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/submodule.c b/submodule.c
index 9fbfcfcfe1..6b4cee82bf 100644
--- a/submodule.c
+++ b/submodule.c
@@ -24,7 +24,7 @@
 #include "object-store.h"
 
 static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
-static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
+
 static int initialized_fetch_ref_tips;
 static struct oid_array ref_tips_before_fetch;
 static struct oid_array ref_tips_after_fetch;
@@ -1124,7 +1124,22 @@ void check_for_new_submodule_commits(struct object_id *oid)
 	oid_array_append(&ref_tips_after_fetch, oid);
 }
 
-static void calculate_changed_submodule_paths(void)
+struct submodule_parallel_fetch {
+	int count;
+	struct argv_array args;
+	struct repository *r;
+	const char *prefix;
+	int command_line_option;
+	int default_option;
+	int quiet;
+	int result;
+
+	struct string_list changed_submodule_names;
+};
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+
+static void calculate_changed_submodule_paths(
+	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
 	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
@@ -1162,7 +1177,8 @@ static void calculate_changed_submodule_paths(void)
 			continue;
 
 		if (!submodule_has_commits(path, commits))
-			string_list_append(&changed_submodule_names, name->string);
+			string_list_append(&spf->changed_submodule_names,
+					   name->string);
 	}
 
 	free_submodules_oids(&changed_submodules);
@@ -1199,18 +1215,6 @@ int submodule_touches_in_range(struct object_id *excl_oid,
 	return ret;
 }
 
-struct submodule_parallel_fetch {
-	int count;
-	struct argv_array args;
-	struct repository *r;
-	const char *prefix;
-	int command_line_option;
-	int default_option;
-	int quiet;
-	int result;
-};
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
-
 static int get_fetch_recurse_config(const struct submodule *submodule,
 				    struct submodule_parallel_fetch *spf)
 {
@@ -1271,7 +1275,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
 			    !string_list_lookup(
-					&changed_submodule_names,
+					&spf->changed_submodule_names,
 					submodule->name))
 				continue;
 			default_argv = "on-demand";
@@ -1363,8 +1367,8 @@ int fetch_populated_submodules(struct repository *r,
 	argv_array_push(&spf.args, "--recurse-submodules-default");
 	/* default value, "--submodule-prefix" and its value are added later */
 
-	calculate_changed_submodule_paths();
-	string_list_sort(&changed_submodule_names);
+	calculate_changed_submodule_paths(&spf);
+	string_list_sort(&spf.changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
@@ -1373,7 +1377,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&changed_submodule_names, 1);
+	string_list_clear(&spf.changed_submodule_names, 1);
 	return spf.result;
 }
 
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* [PATCH 5/9] submodule.c: do not copy around submodule list
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
                   ` (2 preceding siblings ...)
  2018-10-16 18:13 ` [PATCH 4/9] submodule.c: move global changed_submodule_names into fetch submodule struct Stefan Beller
@ 2018-10-16 18:13 ` Stefan Beller
  2018-10-16 18:13 ` [PATCH 6/9] repository: repo_submodule_init to take a submodule struct Stefan Beller
                   ` (5 subsequent siblings)
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

'calculate_changed_submodule_paths' uses a local list to compute the
changed submodules, and then produces the result by copying appropriate
items into the result list.

Instead use the result list directly and prune items afterwards
using string_list_remove_empty_items.

By doing so we'll have access to the util pointer for longer that
contains the commits that we need to fetch, which will be
useful in a later patch.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 submodule.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/submodule.c b/submodule.c
index 6b4cee82bf..cbefe5f54d 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1142,8 +1142,7 @@ static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
-	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
-	const struct string_list_item *name;
+	struct string_list_item *name;
 
 	/* No need to check if there are no submodules configured */
 	if (!submodule_from_path(the_repository, NULL, NULL))
@@ -1160,9 +1159,9 @@ static void calculate_changed_submodule_paths(
 	 * Collect all submodules (whether checked out or not) for which new
 	 * commits have been recorded upstream in "changed_submodule_names".
 	 */
-	collect_changed_submodules(&changed_submodules, &argv);
+	collect_changed_submodules(&spf->changed_submodule_names, &argv);
 
-	for_each_string_list_item(name, &changed_submodules) {
+	for_each_string_list_item(name, &spf->changed_submodule_names) {
 		struct oid_array *commits = name->util;
 		const struct submodule *submodule;
 		const char *path = NULL;
@@ -1176,12 +1175,14 @@ static void calculate_changed_submodule_paths(
 		if (!path)
 			continue;
 
-		if (!submodule_has_commits(path, commits))
-			string_list_append(&spf->changed_submodule_names,
-					   name->string);
+		if (submodule_has_commits(path, commits)) {
+			oid_array_clear(commits);
+			*name->string = '\0';
+		}
 	}
 
-	free_submodules_oids(&changed_submodules);
+	string_list_remove_empty_items(&spf->changed_submodule_names, 1);
+
 	argv_array_clear(&argv);
 	oid_array_clear(&ref_tips_before_fetch);
 	oid_array_clear(&ref_tips_after_fetch);
@@ -1377,7 +1378,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&spf.changed_submodule_names, 1);
+	free_submodules_oids(&spf.changed_submodule_names);
 	return spf.result;
 }
 
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* [PATCH 6/9] repository: repo_submodule_init to take a submodule struct
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
                   ` (3 preceding siblings ...)
  2018-10-16 18:13 ` [PATCH 5/9] submodule.c: do not copy around submodule list Stefan Beller
@ 2018-10-16 18:13 ` Stefan Beller
  2018-10-16 18:13 ` [PATCH 7/9] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
                   ` (4 subsequent siblings)
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

When constructing a struct repository for a submodule for some revision
of the superproject where the submodule is not contained in the index,
it may not be present in the working tree currently either. In that
siutation giving a 'path' argument is not useful. Upgrade the
repo_submodule_init function to take a struct submodule instead.

While we are at it, overhaul the repo_submodule_init function by renaming
the submodule repository struct, which is to be initialized, to a name
that is not confused with the struct submodule as easily.

Also move its documentation into the header file.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/grep.c                               | 17 +++++++-----
 builtin/ls-files.c                           | 12 +++++----
 repository.c                                 | 27 ++++++++------------
 repository.h                                 | 11 ++++++--
 t/helper/test-submodule-nested-repo-config.c |  8 +++---
 5 files changed, 41 insertions(+), 34 deletions(-)

diff --git a/builtin/grep.c b/builtin/grep.c
index 7da8fef31a..ba7634258a 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -418,7 +418,10 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 			  const struct object_id *oid,
 			  const char *filename, const char *path)
 {
-	struct repository submodule;
+	struct repository subrepo;
+	const struct submodule *sub = submodule_from_path(superproject,
+							  &null_oid, path);
+
 	int hit;
 
 	/*
@@ -434,12 +437,12 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 		return 0;
 	}
 
-	if (repo_submodule_init(&submodule, superproject, path)) {
+	if (repo_submodule_init(&subrepo, superproject, sub)) {
 		grep_read_unlock();
 		return 0;
 	}
 
-	repo_read_gitmodules(&submodule);
+	repo_read_gitmodules(&subrepo);
 
 	/*
 	 * NEEDSWORK: This adds the submodule's object directory to the list of
@@ -451,7 +454,7 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 	 * store is no longer global and instead is a member of the repository
 	 * object.
 	 */
-	add_to_alternates_memory(submodule.objects->objectdir);
+	add_to_alternates_memory(subrepo.objects->objectdir);
 	grep_read_unlock();
 
 	if (oid) {
@@ -476,14 +479,14 @@ static int grep_submodule(struct grep_opt *opt, struct repository *superproject,
 
 		init_tree_desc(&tree, data, size);
 		hit = grep_tree(opt, pathspec, &tree, &base, base.len,
-				object->type == OBJ_COMMIT, &submodule);
+				object->type == OBJ_COMMIT, &subrepo);
 		strbuf_release(&base);
 		free(data);
 	} else {
-		hit = grep_cache(opt, &submodule, pathspec, 1);
+		hit = grep_cache(opt, &subrepo, pathspec, 1);
 	}
 
-	repo_clear(&submodule);
+	repo_clear(&subrepo);
 	return hit;
 }
 
diff --git a/builtin/ls-files.c b/builtin/ls-files.c
index 7f9919a362..4d1649c1b3 100644
--- a/builtin/ls-files.c
+++ b/builtin/ls-files.c
@@ -206,17 +206,19 @@ static void show_files(struct repository *repo, struct dir_struct *dir);
 static void show_submodule(struct repository *superproject,
 			   struct dir_struct *dir, const char *path)
 {
-	struct repository submodule;
+	struct repository subrepo;
+	const struct submodule *sub = submodule_from_path(superproject,
+							  &null_oid, path);
 
-	if (repo_submodule_init(&submodule, superproject, path))
+	if (repo_submodule_init(&subrepo, superproject, sub))
 		return;
 
-	if (repo_read_index(&submodule) < 0)
+	if (repo_read_index(&subrepo) < 0)
 		die("index file corrupt");
 
-	show_files(&submodule, dir);
+	show_files(&subrepo, dir);
 
-	repo_clear(&submodule);
+	repo_clear(&subrepo);
 }
 
 static void show_ce(struct repository *repo, struct dir_struct *dir,
diff --git a/repository.c b/repository.c
index 5dd1486718..aabe64ee5d 100644
--- a/repository.c
+++ b/repository.c
@@ -166,30 +166,23 @@ int repo_init(struct repository *repo,
 	return -1;
 }
 
-/*
- * Initialize 'submodule' as the submodule given by 'path' in parent repository
- * 'superproject'.
- * Return 0 upon success and a non-zero value upon failure.
- */
-int repo_submodule_init(struct repository *submodule,
+int repo_submodule_init(struct repository *subrepo,
 			struct repository *superproject,
-			const char *path)
+			const struct submodule *sub)
 {
-	const struct submodule *sub;
 	struct strbuf gitdir = STRBUF_INIT;
 	struct strbuf worktree = STRBUF_INIT;
 	int ret = 0;
 
-	sub = submodule_from_path(superproject, &null_oid, path);
 	if (!sub) {
 		ret = -1;
 		goto out;
 	}
 
-	strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", path);
-	strbuf_repo_worktree_path(&worktree, superproject, "%s", path);
+	strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", sub->path);
+	strbuf_repo_worktree_path(&worktree, superproject, "%s", sub->path);
 
-	if (repo_init(submodule, gitdir.buf, worktree.buf)) {
+	if (repo_init(subrepo, gitdir.buf, worktree.buf)) {
 		/*
 		 * If initilization fails then it may be due to the submodule
 		 * not being populated in the superproject's worktree.  Instead
@@ -201,16 +194,16 @@ int repo_submodule_init(struct repository *submodule,
 		strbuf_repo_git_path(&gitdir, superproject,
 				     "modules/%s", sub->name);
 
-		if (repo_init(submodule, gitdir.buf, NULL)) {
+		if (repo_init(subrepo, gitdir.buf, NULL)) {
 			ret = -1;
 			goto out;
 		}
 	}
 
-	submodule->submodule_prefix = xstrfmt("%s%s/",
-					      superproject->submodule_prefix ?
-					      superproject->submodule_prefix :
-					      "", path);
+	subrepo->submodule_prefix = xstrfmt("%s%s/",
+					    superproject->submodule_prefix ?
+					    superproject->submodule_prefix :
+					    "", sub->path);
 
 out:
 	strbuf_release(&gitdir);
diff --git a/repository.h b/repository.h
index 9f16c42c1e..a9c7a5baa5 100644
--- a/repository.h
+++ b/repository.h
@@ -116,9 +116,16 @@ void repo_set_worktree(struct repository *repo, const char *path);
 void repo_set_hash_algo(struct repository *repo, int algo);
 void initialize_the_repository(void);
 int repo_init(struct repository *r, const char *gitdir, const char *worktree);
-int repo_submodule_init(struct repository *submodule,
+
+/*
+ * Initialize the repository 'subrepo' as the submodule given by the
+ * struct submodule 'sub' in parent repository 'superproject'.
+ * Return 0 upon success and a non-zero value upon failure.
+ */
+struct submodule;
+int repo_submodule_init(struct repository *subrepo,
 			struct repository *superproject,
-			const char *path);
+			const struct submodule *sub);
 void repo_clear(struct repository *repo);
 
 /*
diff --git a/t/helper/test-submodule-nested-repo-config.c b/t/helper/test-submodule-nested-repo-config.c
index a31e2a9bea..bc97929bbc 100644
--- a/t/helper/test-submodule-nested-repo-config.c
+++ b/t/helper/test-submodule-nested-repo-config.c
@@ -10,19 +10,21 @@ static void die_usage(int argc, const char **argv, const char *msg)
 
 int cmd__submodule_nested_repo_config(int argc, const char **argv)
 {
-	struct repository submodule;
+	struct repository subrepo;
+	const struct submodule *sub;
 
 	if (argc < 3)
 		die_usage(argc, argv, "Wrong number of arguments.");
 
 	setup_git_directory();
 
-	if (repo_submodule_init(&submodule, the_repository, argv[1])) {
+	sub = submodule_from_path(the_repository, &null_oid, argv[1]);
+	if (repo_submodule_init(&subrepo, the_repository, sub)) {
 		die_usage(argc, argv, "Submodule not found.");
 	}
 
 	/* Read the config of _child_ submodules. */
-	print_config_from_gitmodules(&submodule, argv[2]);
+	print_config_from_gitmodules(&subrepo, argv[2]);
 
 	submodule_free(the_repository);
 
-- 
2.19.0


^ permalink raw reply	[relevance 26%]

* [PATCH 7/9] submodule: fetch in submodules git directory instead of in worktree
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
                   ` (4 preceding siblings ...)
  2018-10-16 18:13 ` [PATCH 6/9] repository: repo_submodule_init to take a submodule struct Stefan Beller
@ 2018-10-16 18:13 ` Stefan Beller
      [irrelevant]   ` <20181017225811.66554-1-jonathantanmy@google.com>
  2018-10-16 18:13 ` [PATCH 8/9] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
                   ` (3 subsequent siblings)
  9 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

This patch started as a refactoring to make 'get_next_submodule' more
readable, but upon doing so, I realized that "git fetch" of the submodule
actually doesn't need to be run in the submodules worktree. So let's run
it in its git dir instead.

That should pave the way towards fetching submodules that are currently
not checked out.

This patch leaks the cp->dir in get_next_submodule, as any further
callback in run_processes_parallel doesn't have access to the child
process any more. In an early iteration of this patch, the function
get_submodule_repo_for directly returned the string containing the
git directory, which would be a better design choice for this patch.

However the next patch both fixes the memory leak of cp->dir and also has
a use case for using the full repository handle of the submodule, so
it makes sense to introduce the get_submodule_repo_for here already.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 submodule.c                 | 51 +++++++++++++++++++++++++++----------
 t/t5526-fetch-submodules.sh |  7 ++++-
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/submodule.c b/submodule.c
index cbefe5f54d..30c06507e3 100644
--- a/submodule.c
+++ b/submodule.c
@@ -495,6 +495,12 @@ void prepare_submodule_repo_env(struct argv_array *out)
 			 DEFAULT_GIT_DIR_ENVIRONMENT);
 }
 
+static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out)
+{
+	prepare_submodule_repo_env_no_git_dir(out);
+	argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
+}
+
 /* Helper function to display the submodule header line prior to the full
  * summary output. If it can locate the submodule objects directory it will
  * attempt to lookup both the left and right commits and put them into the
@@ -1241,6 +1247,29 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+static struct repository *get_submodule_repo_for(struct repository *r,
+						 const struct submodule *sub)
+{
+	struct repository *ret = xmalloc(sizeof(*ret));
+
+	if (repo_submodule_init(ret, r, sub)) {
+		/*
+		 * No entry in .gitmodules? Technically not a submodule,
+		 * but historically we supported repositories that happen to be
+		 * in-place where a gitlink is. Keep supporting them.
+		 */
+		struct strbuf gitdir = STRBUF_INIT;
+		strbuf_repo_worktree_path(&gitdir, r, "%s/.git", sub->path);
+		if (repo_init(ret, gitdir.buf, NULL)) {
+			strbuf_release(&gitdir);
+			return NULL;
+		}
+		strbuf_release(&gitdir);
+	}
+
+	return ret;
+}
+
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
@@ -1248,12 +1277,11 @@ static int get_next_submodule(struct child_process *cp,
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_path = STRBUF_INIT;
-		struct strbuf submodule_git_dir = STRBUF_INIT;
 		struct strbuf submodule_prefix = STRBUF_INIT;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
-		const char *git_dir, *default_argv;
+		const char *default_argv;
 		const struct submodule *submodule;
+		struct repository *repo;
 		struct submodule default_submodule = SUBMODULE_INIT;
 
 		if (!S_ISGITLINK(ce->ce_mode))
@@ -1288,16 +1316,12 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_repo_worktree_path(&submodule_path, spf->r, "%s", ce->name);
-		strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
 		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		git_dir = read_gitfile(submodule_git_dir.buf);
-		if (!git_dir)
-			git_dir = submodule_git_dir.buf;
-		if (is_directory(git_dir)) {
+		repo = get_submodule_repo_for(spf->r, submodule);
+		if (repo) {
 			child_process_init(cp);
-			cp->dir = strbuf_detach(&submodule_path, NULL);
-			prepare_submodule_repo_env(&cp->env_array);
+			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+			cp->dir = xstrdup(repo->gitdir);
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1307,10 +1331,11 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
 			argv_array_push(&cp->args, submodule_prefix.buf);
+
+			repo_clear(repo);
+			free(repo);
 			ret = 1;
 		}
-		strbuf_release(&submodule_path);
-		strbuf_release(&submodule_git_dir);
 		strbuf_release(&submodule_prefix);
 		if (ret) {
 			spf->count++;
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 6c2f9b2ba2..42692219a1 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -566,7 +566,12 @@ test_expect_success 'fetching submodule into a broken repository' '
 
 	test_must_fail git -C dst status &&
 	test_must_fail git -C dst diff &&
-	test_must_fail git -C dst fetch --recurse-submodules
+
+	# git-fetch cannot find the git directory of the submodule,
+	# so it will do nothing, successfully, as it cannot distinguish between
+	# this broken submodule and a submodule that was just set active but
+	# not cloned yet
+	git -C dst fetch --recurse-submodules
 '
 
 test_expect_success "fetch new commits when submodule got renamed" '
-- 
2.19.0


^ permalink raw reply	[relevance 28%]

* [PATCH 8/9] fetch: retry fetching submodules if needed objects were not fetched
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
                   ` (5 preceding siblings ...)
  2018-10-16 18:13 ` [PATCH 7/9] submodule: fetch in submodules git directory instead of in worktree Stefan Beller
@ 2018-10-16 18:13 ` Stefan Beller
      [irrelevant]   ` <20181018003954.139498-1-jonathantanmy@google.com>
  2018-10-16 18:13 ` [PATCH 9/9] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
                   ` (2 subsequent siblings)
  9 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

Currently when git-fetch is asked to recurse into submodules, it dispatches
a plain "git-fetch -C <submodule-dir>" (with some submodule related options
such as prefix and recusing strategy, but) without any information of the
remote or the tip that should be fetched.

This works surprisingly well in some workflows (such as using submodules
as a third party library), while not so well in other scenarios, such
as in a Gerrit topic-based workflow, that can tie together changes
(potentially across repositories) on the server side. One of the parts
of such a Gerrit workflow is to download a change when wanting to examine
it, and you'd want to have its submodule changes that are in the same
topic downloaded as well. However these submodule changes reside in their
own repository in their own ref (refs/changes/<int>).

Retry fetching a submodule by object name if the object id that the
superproject points to, cannot be found.

This retrying does not happen when the "git fetch" done at the
superproject is not storing the fetched results in remote
tracking branches (i.e. instead just recording them to
FETCH_HEAD) in this step. A later patch will fix this.

builtin/fetch used to only inspect submodules when they were fetched
"on-demand", as in either on/off case it was clear whether the submodule
needs to be fetched. However to know whether we need to try fetching the
object ids, we need to identify the object names, which is done in this
function check_for_new_submodule_commits(), so we'll also run that code
in case the submodule recursion is set to "on".

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 builtin/fetch.c             |   9 +-
 submodule.c                 | 185 ++++++++++++++++++++++++++++++------
 t/t5526-fetch-submodules.sh |  16 ++++
 3 files changed, 177 insertions(+), 33 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 61bec5d213..95c44bf6ff 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -700,8 +700,7 @@ static int update_local_ref(struct ref *ref,
 			what = _("[new ref]");
 		}
 
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref(msg, ref, 0);
 		format_display(display, r ? '!' : '*', what,
@@ -716,8 +715,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "..");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("fast-forward", ref, 1);
 		format_display(display, r ? '!' : ' ', quickref.buf,
@@ -731,8 +729,7 @@ static int update_local_ref(struct ref *ref,
 		strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
 		strbuf_addstr(&quickref, "...");
 		strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-		if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-		    (recurse_submodules != RECURSE_SUBMODULES_ON))
+		if (recurse_submodules != RECURSE_SUBMODULES_OFF)
 			check_for_new_submodule_commits(&ref->new_oid);
 		r = s_update_ref("forced-update", ref, 1);
 		format_display(display, r ? '!' : '+', quickref.buf,
diff --git a/submodule.c b/submodule.c
index 30c06507e3..7246b776f3 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1141,8 +1141,12 @@ struct submodule_parallel_fetch {
 	int result;
 
 	struct string_list changed_submodule_names;
+	struct get_next_submodule_task **retry;
+	int retry_nr, retry_alloc;
 };
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \
+		  STRING_LIST_INIT_DUP, \
+		  NULL, 0, 0}
 
 static void calculate_changed_submodule_paths(
 	struct submodule_parallel_fetch *spf)
@@ -1247,6 +1251,56 @@ static int get_fetch_recurse_config(const struct submodule *submodule,
 	return spf->default_option;
 }
 
+struct get_next_submodule_task {
+	struct repository *repo;
+	const struct submodule *sub;
+	unsigned free_sub : 1; /* Do we need to free the submodule? */
+	struct oid_array *commits;
+};
+
+static const struct submodule *get_default_submodule(const char *path)
+{
+	struct submodule *ret = NULL;
+	const char *name = default_name_or_path(path);
+
+	if (!name)
+		return NULL;
+
+	ret = xmalloc(sizeof(*ret));
+	memset(ret, 0, sizeof(*ret));
+	ret->path = name;
+	ret->name = name;
+
+	return (const struct submodule *) ret;
+}
+
+static struct get_next_submodule_task *get_next_submodule_task_create(
+	struct repository *r, const char *path)
+{
+	struct get_next_submodule_task *task = xmalloc(sizeof(*task));
+	memset(task, 0, sizeof(*task));
+
+	task->sub = submodule_from_path(r, &null_oid, path);
+	if (!task->sub) {
+		task->sub = get_default_submodule(path);
+		task->free_sub = 1;
+	}
+
+	return task;
+}
+
+static void get_next_submodule_task_release(struct get_next_submodule_task *p)
+{
+	if (p->free_sub)
+		free((void*)p->sub);
+	p->free_sub = 0;
+	p->sub = NULL;
+
+	if (p->repo)
+		repo_clear(p->repo);
+	FREE_AND_NULL(p->repo);
+}
+
 static struct repository *get_submodule_repo_for(struct repository *r,
 						 const struct submodule *sub)
 {
@@ -1273,39 +1327,35 @@ static struct repository *get_submodule_repo_for(struct repository *r,
 static int get_next_submodule(struct child_process *cp,
 			      struct strbuf *err, void *data, void **task_cb)
 {
-	int ret = 0;
 	struct submodule_parallel_fetch *spf = data;
 
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-		struct strbuf submodule_prefix = STRBUF_INIT;
+		int recurse_config;
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
 		const char *default_argv;
-		const struct submodule *submodule;
-		struct repository *repo;
-		struct submodule default_submodule = SUBMODULE_INIT;
+		struct get_next_submodule_task *task;
 
 		if (!S_ISGITLINK(ce->ce_mode))
 			continue;
 
-		submodule = submodule_from_path(spf->r, &null_oid, ce->name);
-		if (!submodule) {
-			const char *name = default_name_or_path(ce->name);
-			if (name) {
-				default_submodule.path = name;
-				default_submodule.name = name;
-				submodule = &default_submodule;
-			}
+		task = get_next_submodule_task_create(spf->r, ce->name);
+
+		if (!task->sub) {
+			free(task);
+			continue;
 		}
 
-		switch (get_fetch_recurse_config(submodule, spf))
+		recurse_config = get_fetch_recurse_config(task->sub, spf);
+
+		switch (recurse_config)
 		{
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule ||
+			if (!task->sub ||
 			    !string_list_lookup(
 					&spf->changed_submodule_names,
-					submodule->name))
+					task->sub->name))
 				continue;
 			default_argv = "on-demand";
 			break;
@@ -1316,12 +1366,12 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-		repo = get_submodule_repo_for(spf->r, submodule);
-		if (repo) {
+		task->repo = get_submodule_repo_for(spf->r, task->sub);
+		if (task->repo) {
+			struct strbuf submodule_prefix = STRBUF_INIT;
 			child_process_init(cp);
 			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
-			cp->dir = xstrdup(repo->gitdir);
+			cp->dir = task->repo->gitdir;
 			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, "Fetching submodule %s%s\n",
@@ -1330,18 +1380,51 @@ static int get_next_submodule(struct child_process *cp,
 			argv_array_pushv(&cp->args, spf->args.argv);
 			argv_array_push(&cp->args, default_argv);
 			argv_array_push(&cp->args, "--submodule-prefix");
+
+			strbuf_addf(&submodule_prefix, "%s%s/",
+						       spf->prefix,
+						       task->sub->path);
 			argv_array_push(&cp->args, submodule_prefix.buf);
 
-			repo_clear(repo);
-			free(repo);
-			ret = 1;
-		}
-		strbuf_release(&submodule_prefix);
-		if (ret) {
 			spf->count++;
+			*task_cb = task;
+
+			strbuf_release(&submodule_prefix);
 			return 1;
+		} else {
+			get_next_submodule_task_release(task);
+			free(task);
 		}
 	}
+
+	if (spf->retry_nr) {
+		struct get_next_submodule_task *task = spf->retry[spf->retry_nr - 1];
+		struct strbuf submodule_prefix = STRBUF_INIT;
+		spf->retry_nr--;
+
+		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, task->sub->path);
+
+		child_process_init(cp);
+		prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+		cp->git_cmd = 1;
+		cp->dir = task->repo->gitdir;
+
+		argv_array_init(&cp->args);
+		argv_array_pushv(&cp->args, spf->args.argv);
+		argv_array_push(&cp->args, "on-demand");
+		argv_array_push(&cp->args, "--submodule-prefix");
+		argv_array_push(&cp->args, submodule_prefix.buf);
+
+		/* NEEDSWORK: have get_default_remote from s--h */
+		argv_array_push(&cp->args, "origin");
+		oid_array_for_each_unique(task->commits,
+					  append_oid_to_argv, &cp->args);
+
+		*task_cb = task;
+		strbuf_release(&submodule_prefix);
+		return 1;
+	}
+
 	return 0;
 }
 
@@ -1349,20 +1432,68 @@ static int fetch_start_failure(struct strbuf *err,
 			       void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct get_next_submodule_task *task = task_cb;
 
 	spf->result = 1;
 
+	get_next_submodule_task_release(task);
 	return 0;
 }
 
+static int commit_exists_in_sub(const struct object_id *oid, void *data)
+{
+	struct repository *subrepo = data;
+
+	enum object_type type = oid_object_info(subrepo, oid, NULL);
+
+	return type != OBJ_COMMIT;
+}
+
 static int fetch_finish(int retvalue, struct strbuf *err,
 			void *cb, void *task_cb)
 {
 	struct submodule_parallel_fetch *spf = cb;
+	struct get_next_submodule_task *task = task_cb;
+	const struct submodule *sub;
+
+	struct string_list_item *it;
+	struct oid_array *commits;
 
 	if (retvalue)
 		spf->result = 1;
 
+	if (!task)
+		return 0;
+
+	sub = task->sub;
+	if (!sub)
+		goto out;
+
+	it = string_list_lookup(&spf->changed_submodule_names, sub->name);
+	if (!it)
+		goto out;
+
+	commits = it->util;
+	oid_array_filter(commits,
+			 commit_exists_in_sub,
+			 task->repo);
+
+	/* Are there commits that do not exist? */
+	if (commits->nr) {
+		/* We already tried fetching them, do not try again. */
+		if (task->commits)
+			return 0;
+
+		task->commits = commits;
+		ALLOC_GROW(spf->retry, spf->retry_nr + 1, spf->retry_alloc);
+		spf->retry[spf->retry_nr] = task;
+		spf->retry_nr++;
+		return 0;
+	}
+
+out:
+	get_next_submodule_task_release(task);
+
 	return 0;
 }
 
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 42692219a1..af12c50e7d 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -605,4 +605,20 @@ test_expect_success "fetch new commits when submodule got renamed" '
 	test_cmp expect actual
 '
 
+test_expect_success "fetch new commits on-demand when they are not reachable" '
+	git checkout --detach &&
+	C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
+	git -C submodule update-ref refs/changes/1 $C &&
+	git update-index --cacheinfo 160000 $C submodule &&
+	git commit -m "updated submodule outside of refs/heads" &&
+	D=$(git rev-parse HEAD) &&
+	git update-ref refs/changes/2 $D &&
+	(
+		cd downstream &&
+		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git -C submodule cat-file -t $C &&
+		git checkout --recurse-submodules FETCH_HEAD
+	)
+'
+
 test_done
-- 
2.19.0


^ permalink raw reply	[relevance 25%]

* [PATCH 9/9] builtin/fetch: check for submodule updates for non branch fetches
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
                   ` (6 preceding siblings ...)
  2018-10-16 18:13 ` [PATCH 8/9] fetch: retry fetching submodules if needed objects were not fetched Stefan Beller
@ 2018-10-16 18:13 ` Stefan Beller
  2018-10-18  2:30 ` [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Junio C Hamano
  2018-10-18  7:30 ` Junio C Hamano
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-16 18:13 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

Gerrit, the code review tool, has a different workflow than our mailing
list based approach. Usually users upload changes to a Gerrit server and
continuous integration and testing happens by bots. Sometimes however a
user wants to checkout a change locally and look at it locally. For this
use case, Gerrit offers a command line snippet to copy and paste to your
terminal, which looks like

  git fetch https://<host>/gerrit refs/changes/<id> &&
  git checkout FETCH_HEAD

For Gerrit changes that contain changing submodule gitlinks, it would be
easy to extend both the fetch and checkout with the '--recurse-submodules'
flag, such that this command line snippet would produce the state of a
change locally.

However the functionality added in the previous patch, which would
ensure that we fetch the objects in the submodule that the gitlink pointed
at, only works for remote tracking branches so far, not for FETCH_HEAD.

Make sure that fetching a superproject to its FETCH_HEAD, also respects
the existence checks for objects in the submodule recursion.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 builtin/fetch.c             | 5 ++++-
 t/t5526-fetch-submodules.sh | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 95c44bf6ff..ea6ecd123e 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -887,11 +887,14 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
 				rc |= update_local_ref(ref, what, rm, &note,
 						       summary_width);
 				free(ref);
-			} else
+			} else {
+				check_for_new_submodule_commits(&rm->old_oid);
 				format_display(&note, '*',
 					       *kind ? kind : "branch", NULL,
 					       *what ? what : "HEAD",
 					       "FETCH_HEAD", summary_width);
+			}
+
 			if (note.len) {
 				if (verbosity >= 0 && !shown_url) {
 					fprintf(stderr, _("From %.*s\n"),
diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index af12c50e7d..a509eabb04 100755
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -615,7 +615,7 @@ test_expect_success "fetch new commits on-demand when they are not reachable" '
 	git update-ref refs/changes/2 $D &&
 	(
 		cd downstream &&
-		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
+		git fetch --recurse-submodules origin refs/changes/2 &&
 		git -C submodule cat-file -t $C &&
 		git checkout --recurse-submodules FETCH_HEAD
 	)
-- 
2.19.0


^ permalink raw reply	[relevance 22%]

* Re: [PATCH 17/19] submodule: use submodule repos for object lookup
      [irrelevant]   ` <20181011224052.191281-1-jonathantanmy@google.com>
  2018-10-13  0:20     ` Stefan Beller
@ 2018-10-16 19:30     ` Stefan Beller
      [irrelevant]       ` <20181016231343.189184-1-jonathantanmy@google.com>
  1 sibling, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-16 19:30 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: git

On Thu, Oct 11, 2018 at 3:41 PM Jonathan Tan <jonathantanmy@google.com> wrote:
>
> > +/*
> > + * Initialize 'out' based on the provided submodule path.
> > + *
> > + * Unlike repo_submodule_init, this tolerates submodules not present
> > + * in .gitmodules. NEEDSWORK: The repo_submodule_init behavior is
> > + * preferrable. This function exists only to preserve historical behavior.
>
> What do you mean by "The repo_submodule_init behavior is preferable"? If
> we need to preserve historical behavior, then it seems that the most
> preferable one is something that meets our needs (like open_submodule()
> in this patch).
>
> > +static int open_submodule(struct repository *out, const char *path)
> > +{
> > +     struct strbuf sb = STRBUF_INIT;
> > +
> > +     if (submodule_to_gitdir(&sb, path) || repo_init(out, sb.buf, NULL)) {
> > +             strbuf_release(&sb);
> > +             return -1;
> > +     }
> > +
> > +     out->submodule_prefix = xstrdup(path);
>
> Do we need to set submodule_prefix?
>
> > @@ -507,7 +533,7 @@ static void show_submodule_header(struct diff_options *o, const char *path,
> >       else if (is_null_oid(two))
> >               message = "(submodule deleted)";
> >
> > -     if (add_submodule_odb(path)) {
> > +     if (open_submodule(sub, path) < 0) {
>
> This function, as a side effect, writes the open repository to "sub" for
> its caller to use. I think it's better if its callers open "sub" and
> then pass it to show_submodule_header() if successful. If opening the
> submodule is expected to fail sometimes (like it seems here), then we
> can allow callers to also pass NULL, and document what happens when NULL
> is passed.

Thanks for the review of the whole series!

I have redone this series, addressing all your comments. I addressed
this comment differently than suggested, and put the submodule
repository argument at the end of the parameter list, such that it
goes with all the other arguments to be filled in.

I was about to resend the series, but test-merged with the other
submodule series in flight (origin/sb/submodule-recursive-fetch-gets-the-tip)
which had some conflicts that I can easily resolve by rebasing on top.

It conflicts a lot when merging to next, due to the latest patch
("Apply semantic patches from previous patches"), so I am not sure
how to proceed properly. Maybe we'd omit that patch and
run 'make coccicheck' on next to apply the semantic patches
there instead.

^ permalink raw reply	[relevance 8%]

* Re: [PATCH] submodule helper: convert relative URL to absolute URL if needed
  2018-10-16 17:27       ` Stefan Beller
@ 2018-10-16 21:05         ` Jonathan Nieder
  0 siblings, 0 replies; 200+ results
From: Jonathan Nieder @ 2018-10-16 21:05 UTC (permalink / raw)
  To: Stefan Beller; +Cc: gitster, git, jungjw

Stefan Beller wrote:

> Reported-by: Jaewoong Jung <jungjw@google.com>
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
>  builtin/submodule--helper.c | 51 ++++++++++++++++++++++++-------------
>  t/t7400-submodule-basic.sh  | 24 +++++++++++++++++
>  2 files changed, 58 insertions(+), 17 deletions(-)

Reviewed-by: Jonathan Nieder <jrnieder@gmail.com>

Thanks for your patient work.

^ permalink raw reply	[relevance 5%]

* Re: [PATCH 17/19] submodule: use submodule repos for object lookup
      [irrelevant]       ` <20181016231343.189184-1-jonathantanmy@google.com>
@ 2018-10-16 23:16         ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-16 23:16 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: git

On Tue, Oct 16, 2018 at 4:13 PM Jonathan Tan <jonathantanmy@google.com> wrote:
>
> > Thanks for the review of the whole series!
> >
> > I have redone this series, addressing all your comments. I addressed
> > this comment differently than suggested, and put the submodule
> > repository argument at the end of the parameter list, such that it
> > goes with all the other arguments to be filled in.
>
> Sounds good.

Actually I changed my mind on that after figuring out how to free
the submodule repository appropriately and went with your original suggestion.

>
> > I was about to resend the series, but test-merged with the other
> > submodule series in flight (origin/sb/submodule-recursive-fetch-gets-the-tip)
> > which had some conflicts that I can easily resolve by rebasing on top.
>
> I presume you are talking about [1]? Maybe consider rebasing that one on
> top of this instead, since this is just a refactoring whereas
> submodule-recursive-fetch-gets-the-tip changes functionality, from what
> I understand of patches 8 and 9.

From my understanding, that series is further along than this one,
so I would not want to mix up their order.

Currently I am rebasing this on top of select topics from next,
(ds/reachable) as that are the other conflicts that I'd have to deal with.

> [1] https://public-inbox.org/git/20181016181327.107186-1-sbeller@google.com/
>
> > It conflicts a lot when merging to next, due to the latest patch
> > ("Apply semantic patches from previous patches"), so I am not sure
> > how to proceed properly. Maybe we'd omit that patch and
> > run 'make coccicheck' on next to apply the semantic patches
> > there instead.
>
> Omitting the patch sounds good to me. For me, just stating that you have
> excluded any coccinelle-generated patches in order to ease merging into
> the various branches is sufficient, and people can test the coccinelle
> patches included by running "make coccicheck" then "patch -p1
> <contrib/coccinelle/the_repository.cocci.patch".

ok.

Thanks,
Stefan

^ permalink raw reply	[relevance 8%]

* [PATCH 00/19] Bring more repository handles into our code base
@ 2018-10-16 23:35 Stefan Beller
  2018-10-16 23:35 ` [PATCH 18/19] submodule: use submodule repos for object lookup Stefan Beller
                   ` (3 more replies)
  0 siblings, 4 replies; 200+ results
From: Stefan Beller @ 2018-10-16 23:35 UTC (permalink / raw)
  To: git; +Cc: jonathantanmy, Stefan Beller

This rerolls sb/more-repo-in-api.
It applies on nd/the-index merged with ds/reachable and is available via
    git fetch https://github.com/stefanbeller/git object-store-final-3

I addressed all of Jonathans comments, see range-diff below;
the last patch (applying the semantic patches) has been omitted as that
would produce a lot of merge conflicts. Without that patch, this merges
cleanly to next.

As for when to apply the semantic patches, I wondered if we would prefer a dirty merge
(created via "make coccicheck && git apply contrib/coccinelle/the_repository.cocci.patch")
of the semantic patches, or if we'd actually trickle in the changes over some time?

Thanks,
Stefan

An earlier RFC was sent out at
https://public-inbox.org/git/20181011211754.31369-1-sbeller@google.com/ and said:
    This applies on nd/the-index (b3c7eef9b05) and is the logical continuation of
    the object store series, which I sent over the last year.
    
    The previous series did take a very slow and pedantic approach,
    using a #define trick, see cfc62fc98c for details, but it turns out,
    that it doesn't work:
       When changing the signature of widely used functions, it burdens the
       maintainer in resolving the semantic conflicts.
       
       In the orginal approach this was called a feature, as then we can ensure
       that not bugs creep into the code base during the merge window (while such
       a refactoring series wanders from pu to master). It turns out this
       was not well received and was just burdensome.
       
       The #define trick doesn't buy us much to begin with when dealing with
       non-merge-conflicts.  For example, see deref_tag at tag.c:68, which got
       the repository argument in 286d258d4f (tag.c: allow deref_tag to handle
       arbitrary repositories, 2018-06-28) but lost its property of working on any
       repository while 8c4cc32689 (tag: don't warn if target is missing but
       promised, 2018-07-12) was in flight simultaneously.
       
       Another example of failure of this approach is seen in patch 5, which
       shows that the pedantry was missed.
            
    This series takes another approach as it doesn't change the signature of
    functions, but introduces new functions that can deal with arbitrary 
    repositories, keeping the old function signature around using a shallow wrapper.
    
    Additionally each patch adds a semantic patch, that would port from the old to
    the new function. These semantic patches are all applied in the very last patch,
    but we could omit applying the last patch if it causes too many merge conflicts
    and trickl in the semantic patches over time when there are no merge conflicts.
    
    
    The original goal of all these refactoring series was to remove add_submodule_odb 
    in submodule.c, which was partially reached with this series. I'll investigate the
    remaining calls in another series, but it shows we're close to be done with these
    large refactorings as far as I am concerned.
    
    Thanks,
    Stefan
    

Stefan Beller (19):
  sha1_file: allow read_object to read objects in arbitrary repositories
  packfile: allow has_packed_and_bad to handle arbitrary repositories
  object-store: allow read_object_file_extended to read from arbitrary
    repositories
  object-store: prepare read_object_file to deal with arbitrary
    repositories
  object-store: prepare has_{sha1, object}_file[_with_flags] to handle
    arbitrary repositories
  object: parse_object to honor its repository argument
  commit: allow parse_commit* to handle arbitrary repositories
  commit-reach.c: allow paint_down_to_common to handle arbitrary
    repositories
  commit-reach.c: allow merge_bases_many to handle arbitrary
    repositories
  commit-reach.c: allow remove_redundant to handle arbitrary
    repositories
  commit-reach.c: allow get_merge_bases_many_0 to handle arbitrary
    repositories
  commit-reach: prepare get_merge_bases to handle arbitrary repositories
  commit-reach: prepare in_merge_bases[_many] to handle arbitrary
    repositories
  commit: prepare get_commit_buffer to handle arbitrary repositories
  commit: prepare repo_unuse_commit_buffer to handle arbitrary
    repositories
  commit: prepare logmsg_reencode to handle arbitrary repositories
  pretty: prepare format_commit_message to handle arbitrary repositories
  submodule: use submodule repos for object lookup
  submodule: don't add submodule as odb for push

 commit-reach.c                          |  73 +++++++-----
 commit-reach.h                          |  38 +++++--
 commit.c                                |  32 ++++--
 commit.h                                |  39 ++++++-
 contrib/coccinelle/the_repository.cocci | 144 ++++++++++++++++++++++++
 object-store.h                          |  35 ++++--
 object.c                                |   6 +-
 packfile.c                              |   5 +-
 packfile.h                              |   2 +-
 pretty.c                                |  28 ++---
 pretty.h                                |   7 +-
 sha1-file.c                             |  34 +++---
 streaming.c                             |   2 +-
 submodule.c                             |  78 ++++++++++---
 14 files changed, 402 insertions(+), 121 deletions(-)
 create mode 100644 contrib/coccinelle/the_repository.cocci

 git range-diff origin/sb/more-repo-in-api...

 -:  ---------- >  1:  5227c38566 commit-reach: move walk methods from commit.c
 -:  ---------- >  2:  6404355657 commit.h: remove method declarations
 -:  ---------- >  3:  1d614d41e5 commit-reach: move ref_newer from remote.c
 -:  ---------- >  4:  920f93ca1c commit-reach: move commit_contains from ref-filter
 -:  ---------- >  5:  f044bb49ad upload-pack: make reachable() more generic
 -:  ---------- >  6:  921bf7734f upload-pack: refactor ok_to_give_up()
 -:  ---------- >  7:  118be5785e upload-pack: generalize commit date cutoff
 -:  ---------- >  8:  ba3ca1edce commit-reach: move can_all_from_reach_with_flags
 -:  ---------- >  9:  ab176ac4ae test-reach: create new test tool for ref_newer
 -:  ---------- > 10:  5cd52de326 test-reach: test in_merge_bases
 -:  ---------- > 11:  6255232ec1 test-reach: test is_descendant_of
 -:  ---------- > 12:  324dec0191 test-reach: test get_merge_bases_many
 -:  ---------- > 13:  0c89f715d0 test-reach: test reduce_heads
 -:  ---------- > 14:  1792bc1250 test-reach: test can_all_from_reach_with_flags
 -:  ---------- > 15:  1fee124257 test-reach: test commit_contains
 -:  ---------- > 16:  1e3497a24c commit-reach: replace ref_newer logic
 -:  ---------- > 17:  4fbcca4eff commit-reach: make can_all_from_reach... linear
 -:  ---------- > 18:  6cc017431c commit-reach: use can_all_from_reach
 -:  ---------- > 19:  6621c83874 commit-reach: correct accidental #include of C file
 -:  ---------- > 20:  b67f6b26e3 commit-reach: properly peel tags
 -:  ---------- > 21:  4067a64672 commit-reach: fix memory and flag leaks
 1:  2c13c6101f = 22:  14c12a5350 sha1_file: allow read_object to read objects in arbitrary repositories
 2:  82848c2571 = 23:  f978950604 packfile: allow has_packed_and_bad to handle arbitrary repositories
 3:  717023d801 = 24:  852f6696d1 object-store: allow read_object_file_extended to read from arbitrary repositories
 4:  fab6bdfad0 ! 25:  b185b88ad8 object-store: prepare read_object_file to deal with arbitrary repositories
    @@ -9,7 +9,7 @@
     
         Introduce repo_read_object_file which takes the repository argument, and
         hide the original read_object_file as a macro behind
    -    NO_THE_REPOSITORY_COMPATIBILITY_MACROS, which we planned for in
    +    NO_THE_REPOSITORY_COMPATIBILITY_MACROS, similar to
         e675765235 (diff.c: remove implicit dependency on the_index, 2018-09-21)
     
         Add a coccinelle patch to convert existing callers, but do not apply
 -:  ---------- > 26:  009d6f350b object-store: prepare has_{sha1, object}_file[_with_flags] to handle arbitrary repositories
 5:  9e34d76c53 ! 27:  df2feb2baf object: parse_object to honor its repository argument
    @@ -12,6 +12,17 @@
     diff --git a/object.c b/object.c
     --- a/object.c
     +++ b/object.c
    +@@
    + 	if (obj && obj->parsed)
    + 		return obj;
    + 
    +-	if ((obj && obj->type == OBJ_BLOB && has_object_file(oid)) ||
    +-	    (!obj && has_object_file(oid) &&
    ++	if ((obj && obj->type == OBJ_BLOB && repo_has_object_file(r, oid)) ||
    ++	    (!obj && repo_has_object_file(r, oid) &&
    + 	     oid_object_info(r, oid, NULL) == OBJ_BLOB)) {
    + 		if (check_object_signature(repl, NULL, 0, NULL) < 0) {
    + 			error(_("sha1 mismatch %s"), oid_to_hex(oid));
     @@
      		return lookup_object(r, oid->hash);
      	}
 6:  9cb7efc7b2 ! 28:  18b7073c25 commit: allow parse_commit* to handle arbitrary repositories
    @@ -97,9 +97,10 @@
     --- a/contrib/coccinelle/the_repository.cocci
     +++ b/contrib/coccinelle/the_repository.cocci
     @@
    - + repo_read_object_file(the_repository,
    -   E, F, G)
    - 
    + - has_object_file_with_flags(
    + + repo_has_object_file_with_flags(the_repository,
    +   E)
    ++
     +@@
     +expression E;
     +expression F;
    @@ -123,4 +124,3 @@
     +- parse_commit(
     ++ repo_parse_commit(the_repository,
     +  E)
    -+
 7:  fcab4a4000 ! 29:  48f8999e95 commit.c: allow paint_down_to_common to handle arbitrary repositories
    @@ -1,15 +1,14 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    commit.c: allow paint_down_to_common to handle arbitrary repositories
    +    commit-reach.c: allow paint_down_to_common to handle arbitrary repositories
     
         As the function is file local and not widely used, migrate it all at once.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
    -diff --git a/commit.c b/commit.c
    ---- a/commit.c
    -+++ b/commit.c
    +diff --git a/commit-reach.c b/commit-reach.c
    +--- a/commit-reach.c
    ++++ b/commit-reach.c
     @@
      }
      
 8:  324422f0d3 ! 30:  b84d32c972 commit.c: allow merge_bases_many to handle arbitrary repositories
    @@ -1,13 +1,12 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    commit.c: allow merge_bases_many to handle arbitrary repositories
    +    commit-reach.c: allow merge_bases_many to handle arbitrary repositories
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
    -diff --git a/commit.c b/commit.c
    ---- a/commit.c
    -+++ b/commit.c
    +diff --git a/commit-reach.c b/commit-reach.c
    +--- a/commit-reach.c
    ++++ b/commit-reach.c
     @@
      	return result;
      }
 9:  9c43e74d6c ! 31:  cd0e595b8d commit.c: allow remove_redundant to handle arbitrary repositories
    @@ -1,13 +1,12 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    commit.c: allow remove_redundant to handle arbitrary repositories
    +    commit-reach.c: allow remove_redundant to handle arbitrary repositories
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
    -diff --git a/commit.c b/commit.c
    ---- a/commit.c
    -+++ b/commit.c
    +diff --git a/commit-reach.c b/commit-reach.c
    +--- a/commit-reach.c
    ++++ b/commit-reach.c
     @@
      	return ret;
      }
10:  e5c2e2fa8e ! 32:  40ebd46acd commit: allow get_merge_bases_many_0 to handle arbitrary repositories
    @@ -1,13 +1,12 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    commit: allow get_merge_bases_many_0 to handle arbitrary repositories
    +    commit-reach.c: allow get_merge_bases_many_0 to handle arbitrary repositories
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
    -diff --git a/commit.c b/commit.c
    ---- a/commit.c
    -+++ b/commit.c
    +diff --git a/commit-reach.c b/commit-reach.c
    +--- a/commit-reach.c
    ++++ b/commit-reach.c
     @@
      	return filled;
      }
11:  27d4669ccb ! 33:  151cc8cd9b commit: prepare get_merge_bases to handle arbitrary repositories
    @@ -1,6 +1,6 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    commit: prepare get_merge_bases to handle arbitrary repositories
    +    commit-reach: prepare get_merge_bases to handle arbitrary repositories
     
         Similarly to previous patches, the get_merge_base functions are used
         often in the code base, which makes migrating them hard.
    @@ -9,11 +9,10 @@
         functions behind a wrapper macro.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
    -diff --git a/commit.c b/commit.c
    ---- a/commit.c
    -+++ b/commit.c
    +diff --git a/commit-reach.c b/commit-reach.c
    +--- a/commit-reach.c
    ++++ b/commit-reach.c
     @@
      	return result;
      }
    @@ -53,16 +52,21 @@
      
      /*
     
    -diff --git a/commit.h b/commit.h
    ---- a/commit.h
    -+++ b/commit.h
    +diff --git a/commit-reach.h b/commit-reach.h
    +--- a/commit-reach.h
    ++++ b/commit-reach.h
     @@
    - int register_commit_graft(struct repository *r, struct commit_graft *, int);
    - struct commit_graft *lookup_commit_graft(struct repository *r, const struct object_id *oid);
    + struct contains_cache;
    + struct ref_filter;
      
    --extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2);
    --extern struct commit_list *get_merge_bases_many(struct commit *one, int n, struct commit **twos);
    --extern struct commit_list *get_octopus_merge_bases(struct commit_list *in);
    +-struct commit_list *get_merge_bases_many(struct commit *one,
    +-					 int n,
    +-					 struct commit **twos);
    +-struct commit_list *get_merge_bases_many_dirty(struct commit *one,
    +-					       int n,
    +-					       struct commit **twos);
    +-struct commit_list *get_merge_bases(struct commit *one, struct commit *two);
    +-struct commit_list *get_octopus_merge_bases(struct commit_list *in);
     -
     +struct commit_list *repo_get_merge_bases(struct repository *r,
     +					 struct commit *rev1,
    @@ -71,7 +75,7 @@
     +					      struct commit *one, int n,
     +					      struct commit **twos);
      /* To be used only when object flags after this call no longer matter */
    --extern struct commit_list *get_merge_bases_many_dirty(struct commit *one, int n, struct commit **twos);
    +-struct commit_list *get_merge_bases_many_dirty(struct commit *one, int n, struct commit **twos);
     +struct commit_list *repo_get_merge_bases_many_dirty(struct repository *r,
     +						    struct commit *one, int n,
     +						    struct commit **twos);
    @@ -80,18 +84,20 @@
     +#define get_merge_bases_many(one, n, two) repo_get_merge_bases_many(the_repository, one, n, two)
     +#define get_merge_bases_many_dirty(one, n, twos) repo_get_merge_bases_many_dirty(the_repository, one, n, twos)
     +#endif
    -+extern struct commit_list *get_octopus_merge_bases(struct commit_list *in);
    ++
    ++struct commit_list *get_octopus_merge_bases(struct commit_list *in);
      
    - /* largest positive number a signed 32-bit integer can contain */
    - #define INFINITE_DEPTH 0x7fffffff
    + int is_descendant_of(struct commit *commit, struct commit_list *with_commit);
    + int in_merge_bases_many(struct commit *commit, int nr_reference, struct commit **reference);
     
     diff --git a/contrib/coccinelle/the_repository.cocci b/contrib/coccinelle/the_repository.cocci
     --- a/contrib/coccinelle/the_repository.cocci
     +++ b/contrib/coccinelle/the_repository.cocci
     @@
    + - parse_commit(
      + repo_parse_commit(the_repository,
        E)
    - 
    ++
     +@@
     +expression E;
     +expression F;
13:  977359ec86 ! 34:  b508acf4fd commit: prepare in_merge_bases[_many] to handle arbitrary repositories
    @@ -1,13 +1,12 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    commit: prepare in_merge_bases[_many] to handle arbitrary repositories
    +    commit-reach: prepare in_merge_bases[_many] to handle arbitrary repositories
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
    -diff --git a/commit.c b/commit.c
    ---- a/commit.c
    -+++ b/commit.c
    +diff --git a/commit-reach.c b/commit-reach.c
    +--- a/commit-reach.c
    ++++ b/commit-reach.c
     @@
      /*
       * Is "commit" an ancestor of one of the "references"?
    @@ -53,33 +52,36 @@
      
      struct commit_list *reduce_heads(struct commit_list *heads)
     
    -diff --git a/commit.h b/commit.h
    ---- a/commit.h
    -+++ b/commit.h
    +diff --git a/commit-reach.h b/commit-reach.h
    +--- a/commit-reach.h
    ++++ b/commit-reach.h
     @@
    - extern struct trace_key trace_shallow;
    + struct commit_list *get_octopus_merge_bases(struct commit_list *in);
      
    - int is_descendant_of(struct commit *, struct commit_list *);
    --int in_merge_bases(struct commit *, struct commit *);
    --int in_merge_bases_many(struct commit *, int, struct commit **);
    -+int repo_in_merge_bases(struct repository *r, struct commit *, struct commit *);
    -+int repo_in_merge_bases_many(struct repository *r, struct commit *, int, struct commit **);
    + int is_descendant_of(struct commit *commit, struct commit_list *with_commit);
    +-int in_merge_bases_many(struct commit *commit, int nr_reference, struct commit **reference);
    +-int in_merge_bases(struct commit *commit, struct commit *reference);
    ++int repo_in_merge_bases(struct repository *r,
    ++			struct commit *commit,
    ++			struct commit *reference);
    ++int repo_in_merge_bases_many(struct repository *r,
    ++			     struct commit *commit,
    ++			     int nr_reference, struct commit **reference);
     +#ifndef NO_THE_REPOSITORY_COMPATIBILITY_MACROS
     +#define in_merge_bases(c1, c2) repo_in_merge_bases(the_repository, c1, c2)
     +#define in_merge_bases_many(c1, n, cs) repo_in_merge_bases_many(the_repository, c1, n, cs)
     +#endif
      
    - extern int interactive_add(int argc, const char **argv, const char *prefix, int patch);
    - extern int run_add_interactive(const char *revision, const char *patch_mode,
    + /*
    +  * Takes a list of commits and returns a new list where those
     
     diff --git a/contrib/coccinelle/the_repository.cocci b/contrib/coccinelle/the_repository.cocci
     --- a/contrib/coccinelle/the_repository.cocci
     +++ b/contrib/coccinelle/the_repository.cocci
     @@
    - - get_commit_buffer(
    - + repo_get_commit_buffer(the_repository,
    -   E, F);
    -+
    + + repo_get_merge_bases_many_dirty(the_repository,
    +   E, F, G);
    + 
     +@@
     +expression E;
     +expression F;
12:  044bf2917c ! 35:  c825f2703c commit: prepare get_commit_buffer to handle arbitrary repositories
    @@ -3,7 +3,6 @@
         commit: prepare get_commit_buffer to handle arbitrary repositories
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/commit.c b/commit.c
     --- a/commit.c
    @@ -50,9 +49,10 @@
     --- a/contrib/coccinelle/the_repository.cocci
     +++ b/contrib/coccinelle/the_repository.cocci
     @@
    - + repo_get_merge_bases_many_dirty(the_repository,
    + - in_merge_bases_many(
    + + repo_in_merge_bases_many(the_repository,
        E, F, G);
    - 
    ++
     +@@
     +expression E;
     +expression F;
14:  65771970b9 ! 36:  bea6ff2e6d commit: prepare repo_unuse_commit_buffer to handle arbitrary repositories
    @@ -3,7 +3,6 @@
         commit: prepare repo_unuse_commit_buffer to handle arbitrary repositories
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/commit.c b/commit.c
     --- a/commit.c
    @@ -46,9 +45,10 @@
     --- a/contrib/coccinelle/the_repository.cocci
     +++ b/contrib/coccinelle/the_repository.cocci
     @@
    + - get_commit_buffer(
      + repo_get_commit_buffer(the_repository,
        E, F);
    - 
    ++
     +@@
     +expression E;
     +expression F;
    @@ -56,7 +56,3 @@
     +- unuse_commit_buffer(
     ++ repo_unuse_commit_buffer(the_repository,
     +  E, F);
    -+
    - @@
    - expression E;
    - expression F;
15:  ddef24685d ! 37:  11ea084cd2 commit: prepare logmsg_reencode to handle arbitrary repositories
    @@ -28,9 +28,9 @@
     --- a/contrib/coccinelle/the_repository.cocci
     +++ b/contrib/coccinelle/the_repository.cocci
     @@
    - - in_merge_bases_many(
    - + repo_in_merge_bases_many(the_repository,
    -   E, F, G);
    + - unuse_commit_buffer(
    + + repo_unuse_commit_buffer(the_repository,
    +   E, F);
     +
     +@@
     +expression E;
16:  8612153e23 = 38:  9f50edd155 pretty: prepare format_commit_message to handle arbitrary repositories
17:  eba296fc18 ! 39:  1a4a392233 submodule: use submodule repos for object lookup
    @@ -7,7 +7,6 @@
         are not added to the main object store.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/submodule.c b/submodule.c
     --- a/submodule.c
    @@ -35,12 +34,15 @@
      			 DEFAULT_GIT_DIR_ENVIRONMENT);
      }
      
    +-/* Helper function to display the submodule header line prior to the full
    +- * summary output. If it can locate the submodule objects directory it will
    +- * attempt to lookup both the left and right commits and put them into the
    +- * left and right pointers.
     +/*
     + * Initialize 'out' based on the provided submodule path.
     + *
     + * Unlike repo_submodule_init, this tolerates submodules not present
    -+ * in .gitmodules. NEEDSWORK: The repo_submodule_init behavior is
    -+ * preferrable. This function exists only to preserve historical behavior.
    ++ * in .gitmodules. This function exists only to preserve historical behavior,
     + *
     + * Returns 0 on success, -1 when the submodule is not present.
     + */
    @@ -54,28 +56,38 @@
     +	}
     +
     +	out->submodule_prefix = xstrdup(path);
    ++	out->submodule_prefix = xstrfmt("%s%s/",
    ++					the_repository->submodule_prefix ?
    ++					the_repository->submodule_prefix :
    ++					"", path);
     +
     +	strbuf_release(&sb);
     +	return 0;
     +}
     +
    - /* Helper function to display the submodule header line prior to the full
    -  * summary output. If it can locate the submodule objects directory it will
    -  * attempt to lookup both the left and right commits and put them into the
    -  * left and right pointers.
    ++/*
    ++ * Helper function to display the submodule header line prior to the full
    ++ * summary output.
    ++ *
    ++ * If it can locate the submodule git directory it will create a repository
    ++ * handle for the submodule and lookup both the left and right commits and
    ++ * put them into the left and right pointers.
       */
     -static void show_submodule_header(struct diff_options *o, const char *path,
    -+static void show_submodule_header(struct diff_options *o, struct repository *sub,
    ++static void show_submodule_header(struct diff_options *o,
     +		const char *path,
      		struct object_id *one, struct object_id *two,
      		unsigned dirty_submodule,
    ++		struct repository *sub,
      		struct commit **left, struct commit **right,
    + 		struct commit_list **merge_bases)
    + {
     @@
      	else if (is_null_oid(two))
      		message = "(submodule deleted)";
      
     -	if (add_submodule_odb(path)) {
    -+	if (open_submodule(sub, path) < 0) {
    ++	if (!sub) {
      		if (!message)
      			message = "(commits not present)";
      		goto output_header;
    @@ -103,30 +115,63 @@
      	struct rev_info rev;
      	struct commit *left = NULL, *right = NULL;
      	struct commit_list *merge_bases = NULL;
    -+	struct repository sub;
    ++	struct repository subrepo, *sub = &subrepo;
    ++
    ++	if (open_submodule(&subrepo, path) < 0)
    ++		sub = NULL;
      
    --	show_submodule_header(o, path, one, two, dirty_submodule,
    -+	show_submodule_header(o, &sub, path, one, two, dirty_submodule,
    - 			      &left, &right, &merge_bases);
    + 	show_submodule_header(o, path, one, two, dirty_submodule,
    +-			      &left, &right, &merge_bases);
    ++			      sub, &left, &right, &merge_bases);
      
      	/*
    + 	 * If we don't have both a left and a right pointer, there is no
    + 	 * reason to try and display a summary. The header line should contain
    + 	 * all the information the user needs.
    + 	 */
    +-	if (!left || !right)
    ++	if (!left || !right || !sub)
    + 		goto out;
    + 
    + 	/* Treat revision walker failure the same as missing commits */
     @@
      		goto out;
      	}
      
     -	print_submodule_summary(&rev, o);
    -+	print_submodule_summary(&sub, &rev, o);
    ++	print_submodule_summary(sub, &rev, o);
      
      out:
      	if (merge_bases)
    + 		free_commit_list(merge_bases);
    + 	clear_commit_marks(left, ~0);
    + 	clear_commit_marks(right, ~0);
    ++	if (sub)
    ++		repo_clear(sub);
    + }
    + 
    + void show_submodule_inline_diff(struct diff_options *o, const char *path,
     @@
      	struct commit_list *merge_bases = NULL;
      	struct child_process cp = CHILD_PROCESS_INIT;
      	struct strbuf sb = STRBUF_INIT;
    -+	struct repository sub;
    ++	struct repository subrepo, *sub = &subrepo;
    ++
    ++	if (open_submodule(&subrepo, path) < 0)
    ++		sub = NULL;
      
    --	show_submodule_header(o, path, one, two, dirty_submodule,
    -+	show_submodule_header(o, &sub, path, one, two, dirty_submodule,
    - 			      &left, &right, &merge_bases);
    + 	show_submodule_header(o, path, one, two, dirty_submodule,
    +-			      &left, &right, &merge_bases);
    ++			      sub, &left, &right, &merge_bases);
      
      	/* We need a valid left and right commit to display a difference */
    + 	if (!(left || is_null_oid(one)) ||
    +@@
    + 		clear_commit_marks(left, ~0);
    + 	if (right)
    + 		clear_commit_marks(right, ~0);
    ++	if (sub)
    ++		repo_clear(sub);
    + }
    + 
    + int should_update_submodules(void)
18:  4b2033017a ! 40:  1ce2d0a867 submodule: don't add submodule as odb for push
    @@ -2,20 +2,13 @@
     
         submodule: don't add submodule as odb for push
     
    -    The submodule was added as an alternative in eb21c732d6 (push: teach
    -    --recurse-submodules the on-demand option, 2012-03-29), but was
    -    not explained, why.
    -
    -    In similar code, submodule_has_commits, the submodule is added as an
    -    alternative to perform a quick check if we need to dive into the submodule.
    -
    -    However in push_submodule
    -    (a) for_each_remote_ref_submodule will also provide the quick check and
    -    (b) after that we don't need to have submodule objects around, as all
    -        further code is to spawn a separate process.
    +    In push_submodule(), because we do not actually need access to objects
    +    in the submodule, do not invoke add_submodule_odb().
    +    (for_each_remote_ref_submodule() does not require access to those
    +    objects, and the actual push is done by spawning another process,
    +    which handles object access by itself.)
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/submodule.c b/submodule.c
     --- a/submodule.c
19:  c74af1496b <  -:  ---------- Apply semantic patches from previous patches

^ permalink raw reply	[relevance 7%]

* [PATCH 18/19] submodule: use submodule repos for object lookup
  2018-10-16 23:35 [PATCH 00/19] Bring more repository handles into our code base Stefan Beller
@ 2018-10-16 23:35 ` Stefan Beller
  2018-10-25  9:14   ` SZEDER Gábor
  2018-10-31 13:38   ` Derrick Stolee
  2018-10-16 23:35 ` [PATCH 19/19] submodule: don't add submodule as odb for push Stefan Beller
                   ` (2 subsequent siblings)
  3 siblings, 2 replies; 200+ results
From: Stefan Beller @ 2018-10-16 23:35 UTC (permalink / raw)
  To: git; +Cc: jonathantanmy, Stefan Beller

This converts the 'show_submodule_header' function to use
the repository API properly, such that the submodule objects
are not added to the main object store.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 75 ++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 60 insertions(+), 15 deletions(-)

diff --git a/submodule.c b/submodule.c
index 4ee69cc014..7305ae2e10 100644
--- a/submodule.c
+++ b/submodule.c
@@ -444,7 +444,7 @@ static int prepare_submodule_summary(struct rev_info *rev, const char *path,
 	return prepare_revision_walk(rev);
 }
 
-static void print_submodule_summary(struct rev_info *rev, struct diff_options *o)
+static void print_submodule_summary(struct repository *r, struct rev_info *rev, struct diff_options *o)
 {
 	static const char format[] = "  %m %s";
 	struct strbuf sb = STRBUF_INIT;
@@ -455,7 +455,8 @@ static void print_submodule_summary(struct rev_info *rev, struct diff_options *o
 		ctx.date_mode = rev->date_mode;
 		ctx.output_encoding = get_log_output_encoding();
 		strbuf_setlen(&sb, 0);
-		format_commit_message(commit, format, &sb, &ctx);
+		repo_format_commit_message(r, commit, format, &sb,
+				      &ctx);
 		strbuf_addch(&sb, '\n');
 		if (commit->object.flags & SYMMETRIC_LEFT)
 			diff_emit_submodule_del(o, sb.buf);
@@ -482,14 +483,46 @@ void prepare_submodule_repo_env(struct argv_array *out)
 			 DEFAULT_GIT_DIR_ENVIRONMENT);
 }
 
-/* Helper function to display the submodule header line prior to the full
- * summary output. If it can locate the submodule objects directory it will
- * attempt to lookup both the left and right commits and put them into the
- * left and right pointers.
+/*
+ * Initialize 'out' based on the provided submodule path.
+ *
+ * Unlike repo_submodule_init, this tolerates submodules not present
+ * in .gitmodules. This function exists only to preserve historical behavior,
+ *
+ * Returns 0 on success, -1 when the submodule is not present.
+ */
+static int open_submodule(struct repository *out, const char *path)
+{
+	struct strbuf sb = STRBUF_INIT;
+
+	if (submodule_to_gitdir(&sb, path) || repo_init(out, sb.buf, NULL)) {
+		strbuf_release(&sb);
+		return -1;
+	}
+
+	out->submodule_prefix = xstrdup(path);
+	out->submodule_prefix = xstrfmt("%s%s/",
+					the_repository->submodule_prefix ?
+					the_repository->submodule_prefix :
+					"", path);
+
+	strbuf_release(&sb);
+	return 0;
+}
+
+/*
+ * Helper function to display the submodule header line prior to the full
+ * summary output.
+ *
+ * If it can locate the submodule git directory it will create a repository
+ * handle for the submodule and lookup both the left and right commits and
+ * put them into the left and right pointers.
  */
-static void show_submodule_header(struct diff_options *o, const char *path,
+static void show_submodule_header(struct diff_options *o,
+		const char *path,
 		struct object_id *one, struct object_id *two,
 		unsigned dirty_submodule,
+		struct repository *sub,
 		struct commit **left, struct commit **right,
 		struct commit_list **merge_bases)
 {
@@ -508,7 +541,7 @@ static void show_submodule_header(struct diff_options *o, const char *path,
 	else if (is_null_oid(two))
 		message = "(submodule deleted)";
 
-	if (add_submodule_odb(path)) {
+	if (!sub) {
 		if (!message)
 			message = "(commits not present)";
 		goto output_header;
@@ -518,8 +551,8 @@ static void show_submodule_header(struct diff_options *o, const char *path,
 	 * Attempt to lookup the commit references, and determine if this is
 	 * a fast forward or fast backwards update.
 	 */
-	*left = lookup_commit_reference(the_repository, one);
-	*right = lookup_commit_reference(the_repository, two);
+	*left = lookup_commit_reference(sub, one);
+	*right = lookup_commit_reference(sub, two);
 
 	/*
 	 * Warn about missing commits in the submodule project, but only if
@@ -529,7 +562,7 @@ static void show_submodule_header(struct diff_options *o, const char *path,
 	     (!is_null_oid(two) && !*right))
 		message = "(commits not present)";
 
-	*merge_bases = get_merge_bases(*left, *right);
+	*merge_bases = repo_get_merge_bases(sub, *left, *right);
 	if (*merge_bases) {
 		if ((*merge_bases)->item == *left)
 			fast_forward = 1;
@@ -563,16 +596,20 @@ void show_submodule_summary(struct diff_options *o, const char *path,
 	struct rev_info rev;
 	struct commit *left = NULL, *right = NULL;
 	struct commit_list *merge_bases = NULL;
+	struct repository subrepo, *sub = &subrepo;
+
+	if (open_submodule(&subrepo, path) < 0)
+		sub = NULL;
 
 	show_submodule_header(o, path, one, two, dirty_submodule,
-			      &left, &right, &merge_bases);
+			      sub, &left, &right, &merge_bases);
 
 	/*
 	 * If we don't have both a left and a right pointer, there is no
 	 * reason to try and display a summary. The header line should contain
 	 * all the information the user needs.
 	 */
-	if (!left || !right)
+	if (!left || !right || !sub)
 		goto out;
 
 	/* Treat revision walker failure the same as missing commits */
@@ -581,13 +618,15 @@ void show_submodule_summary(struct diff_options *o, const char *path,
 		goto out;
 	}
 
-	print_submodule_summary(&rev, o);
+	print_submodule_summary(sub, &rev, o);
 
 out:
 	if (merge_bases)
 		free_commit_list(merge_bases);
 	clear_commit_marks(left, ~0);
 	clear_commit_marks(right, ~0);
+	if (sub)
+		repo_clear(sub);
 }
 
 void show_submodule_inline_diff(struct diff_options *o, const char *path,
@@ -599,9 +638,13 @@ void show_submodule_inline_diff(struct diff_options *o, const char *path,
 	struct commit_list *merge_bases = NULL;
 	struct child_process cp = CHILD_PROCESS_INIT;
 	struct strbuf sb = STRBUF_INIT;
+	struct repository subrepo, *sub = &subrepo;
+
+	if (open_submodule(&subrepo, path) < 0)
+		sub = NULL;
 
 	show_submodule_header(o, path, one, two, dirty_submodule,
-			      &left, &right, &merge_bases);
+			      sub, &left, &right, &merge_bases);
 
 	/* We need a valid left and right commit to display a difference */
 	if (!(left || is_null_oid(one)) ||
@@ -662,6 +705,8 @@ void show_submodule_inline_diff(struct diff_options *o, const char *path,
 		clear_commit_marks(left, ~0);
 	if (right)
 		clear_commit_marks(right, ~0);
+	if (sub)
+		repo_clear(sub);
 }
 
 int should_update_submodules(void)
-- 
2.19.0


^ permalink raw reply	[relevance 20%]

* [PATCH 19/19] submodule: don't add submodule as odb for push
  2018-10-16 23:35 [PATCH 00/19] Bring more repository handles into our code base Stefan Beller
  2018-10-16 23:35 ` [PATCH 18/19] submodule: use submodule repos for object lookup Stefan Beller
@ 2018-10-16 23:35 ` Stefan Beller
  2018-10-19 20:39   ` Jonathan Tan
  2018-10-17 12:41 ` [PATCH 00/19] Bring more repository handles into our code base Derrick Stolee
      [irrelevant] ` <20181022173935.GG30222@szeder.dev>
  3 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-16 23:35 UTC (permalink / raw)
  To: git; +Cc: jonathantanmy, Stefan Beller

In push_submodule(), because we do not actually need access to objects
in the submodule, do not invoke add_submodule_odb().
(for_each_remote_ref_submodule() does not require access to those
objects, and the actual push is done by spawning another process,
which handles object access by itself.)

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/submodule.c b/submodule.c
index 7305ae2e10..e623e6bf7f 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1024,9 +1024,6 @@ static int push_submodule(const char *path,
 			  const struct string_list *push_options,
 			  int dry_run)
 {
-	if (add_submodule_odb(path))
-		return 1;
-
 	if (for_each_remote_ref_submodule(path, has_remote, NULL) > 0) {
 		struct child_process cp = CHILD_PROCESS_INIT;
 		argv_array_push(&cp.args, "push");
-- 
2.19.0


^ permalink raw reply	[relevance 14%]

* [PATCH] builtin/submodule--helper: remove debugging leftover tracing
@ 2018-10-16 23:45 Stefan Beller
  2018-10-17  2:52 ` Jonathan Nieder
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-16 23:45 UTC (permalink / raw)
  To: gitster; +Cc: git, Stefan Beller

I noticed 74d4731da1 (submodule--helper: replace connect-gitdir-workingtree
by ensure-core-worktree, 2018-08-13) had two leftover debugging statements
when reading The coverage report [1]. Remove them.

https://public-inbox.org/git/e30a9c05-87d8-1f2b-182c-6d6a5fefe43c@gmail.com/

Signed-off-by: Stefan Beller <sbeller@google.com>
---

To be applied on (or squashed into the tip of)
  sb/submodule-update-in-c

 builtin/submodule--helper.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 5c9d1fb496..c7d3841ffc 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -1459,7 +1459,6 @@ static void determine_submodule_update_strategy(struct repository *r,
 	key = xstrfmt("submodule.%s.update", sub->name);
 
 	if (update) {
-		trace_printf("parsing update");
 		if (parse_submodule_update_strategy(update, out) < 0)
 			die(_("Invalid update mode '%s' for submodule path '%s'"),
 				update, path);
@@ -1468,7 +1467,6 @@ static void determine_submodule_update_strategy(struct repository *r,
 			die(_("Invalid update mode '%s' configured for submodule path '%s'"),
 				val, path);
 	} else if (sub->update_strategy.type != SM_UPDATE_UNSPECIFIED) {
-		trace_printf("loaded thing");
 		out->type = sub->update_strategy.type;
 		out->command = sub->update_strategy.command;
 	} else
-- 
2.19.0


^ permalink raw reply	[relevance 18%]

* Re: [PATCH] builtin/submodule--helper: remove debugging leftover tracing
  2018-10-16 23:45 [PATCH] builtin/submodule--helper: remove debugging leftover tracing Stefan Beller
@ 2018-10-17  2:52 ` Jonathan Nieder
  0 siblings, 0 replies; 200+ results
From: Jonathan Nieder @ 2018-10-17  2:52 UTC (permalink / raw)
  To: Stefan Beller; +Cc: gitster, git

Stefan Beller wrote:

> I noticed 74d4731da1 (submodule--helper: replace connect-gitdir-workingtree
> by ensure-core-worktree, 2018-08-13) had two leftover debugging statements
> when reading The coverage report [1]. Remove them.
>
> https://public-inbox.org/git/e30a9c05-87d8-1f2b-182c-6d6a5fefe43c@gmail.com/
>
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
>  builtin/submodule--helper.c | 2 --
>  1 file changed, 2 deletions(-)

Doh.  Glad you caught it!

Is there some reference for The Coverage Report other than the mailing
list?  E.g. I suspect a reference to

	make coverage-test
	make coverage-report

would be useful to readers finding this commit later.

> To be applied on (or squashed into the tip of)
>   sb/submodule-update-in-c

Looks like that's already in "master", so not a candidate for
squashing.

Reviewed-by: Jonathan Nieder <jrnieder@gmail.com>

^ permalink raw reply	[relevance 5%]

* Re: [PATCH 00/19] Bring more repository handles into our code base
  2018-10-16 23:35 [PATCH 00/19] Bring more repository handles into our code base Stefan Beller
  2018-10-16 23:35 ` [PATCH 18/19] submodule: use submodule repos for object lookup Stefan Beller
  2018-10-16 23:35 ` [PATCH 19/19] submodule: don't add submodule as odb for push Stefan Beller
@ 2018-10-17 12:41 ` Derrick Stolee
  2018-10-17 17:53   ` Stefan Beller
      [irrelevant] ` <20181022173935.GG30222@szeder.dev>
  3 siblings, 1 reply; 200+ results
From: Derrick Stolee @ 2018-10-17 12:41 UTC (permalink / raw)
  To: Stefan Beller, git; +Cc: jonathantanmy

On 10/16/2018 7:35 PM, Stefan Beller wrote:
>              
>      This series takes another approach as it doesn't change the signature of
>      functions, but introduces new functions that can deal with arbitrary
>      repositories, keeping the old function signature around using a shallow wrapper.
I think this is a good direction, and the changes look good to me.

>      Additionally each patch adds a semantic patch, that would port from the old to
>      the new function. These semantic patches are all applied in the very last patch,
>      but we could omit applying the last patch if it causes too many merge conflicts
>      and trickl in the semantic patches over time when there are no merge conflicts.

The semantic patches are a good idea. At some point in the future, we 
can submit a series that applies the patches to any leftover calls and 
removes the old callers. We can hopefully rely on review (and the 
semantic patches warning that there is work to do) to prevent new 
callers from being introduced in future topics. That doesn't count 
topics that come around while this one isn't merged down.

I had one high-level question: How are we testing that these "arbitrary 
repository" changes are safe? I just remember the issue we had with the 
commit-graph code relying on arbitrary repositories, but then adding a 
reference to the replace objects broke the code (because replace-objects 
wasn't using arbitrary repos correctly, but the commit-graph was tested 
with arbitrary repositories using "test-tool repository"). It would be 
nice to introduce more method calls in t/helper/test-repository.c that 
help us know these are safe conversions. Otherwise, we are essentially 
waiting until we try to take submodule things in-process and find out 
what breaks. As we discovered with the refstore, we can't just ensure 
that all references to the_repository are removed.

Thanks,
-Stolee

^ permalink raw reply	[relevance 4%]

* Re: [PATCH 00/19] Bring more repository handles into our code base
  2018-10-17 12:41 ` [PATCH 00/19] Bring more repository handles into our code base Derrick Stolee
@ 2018-10-17 17:53   ` Stefan Beller
      [irrelevant]     ` <20181018183758.81186-1-sbeller@google.com>
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-17 17:53 UTC (permalink / raw)
  To: Derrick Stolee; +Cc: git, Jonathan Tan

On Wed, Oct 17, 2018 at 5:41 AM Derrick Stolee <stolee@gmail.com> wrote:
>
> On 10/16/2018 7:35 PM, Stefan Beller wrote:
> >
> >      This series takes another approach as it doesn't change the signature of
> >      functions, but introduces new functions that can deal with arbitrary
> >      repositories, keeping the old function signature around using a shallow wrapper.
> I think this is a good direction, and the changes look good to me.
>
> >      Additionally each patch adds a semantic patch, that would port from the old to
> >      the new function. These semantic patches are all applied in the very last patch,
> >      but we could omit applying the last patch if it causes too many merge conflicts
> >      and trickl in the semantic patches over time when there are no merge conflicts.
>
> The semantic patches are a good idea. At some point in the future, we
> can submit a series that applies the patches to any leftover calls and
> removes the old callers. We can hopefully rely on review (and the
> semantic patches warning that there is work to do) to prevent new
> callers from being introduced in future topics. That doesn't count
> topics that come around while this one isn't merged down.

For those topics still in flight, I added re-defines, e.g.

#ifndef NO_THE_REPOSITORY_COMPATIBILITY_MACROS
#define get_merge_bases(r1, r2)
repo_get_merge_bases(the_repository, r1, r2)
#endif

so the base function still keeps working, and we can cleanup
multiple times, until eventually, we can get rid of the base function.

> I had one high-level question: How are we testing that these "arbitrary
> repository" changes are safe?

I did the bare minimum in conversions in this series, such that the
submodule code tests successfully. So if we'd revert some parts,
the submodule tests would break.



> I just remember the issue we had with the
> commit-graph code relying on arbitrary repositories, but then adding a
> reference to the replace objects broke the code (because replace-objects
> wasn't using arbitrary repos correctly, but the commit-graph was tested
> with arbitrary repositories using "test-tool repository"). It would be
> nice to introduce more method calls in t/helper/test-repository.c that
> help us know these are safe conversions.

Or instead we could accelerate the long term plan of removing a
hard coded the_repository and have each cmd builtin take an additional
repository pointer from the init code, such that we'd bring all of Git to
work on arbitrary repositories. Then the standard test suite should be
okay, as there is no special case for the_repository any more.

> Otherwise, we are essentially
> waiting until we try to take submodule things in-process and find out
> what breaks. As we discovered with the refstore, we can't just ensure
> that all references to the_repository are removed.

Yes, that is correct. We had a similar case with partial clone,
as laid out in the cover letter for the RFC.

I'll explore both the test tool approach as well as
repository-fication of the code base.

Thanks,
Stefan

^ permalink raw reply	[relevance 5%]

* Re: [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
                   ` (7 preceding siblings ...)
  2018-10-16 18:13 ` [PATCH 9/9] builtin/fetch: check for submodule updates for non branch fetches Stefan Beller
@ 2018-10-18  2:30 ` Junio C Hamano
  2018-10-18  7:30 ` Junio C Hamano
  9 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-10-18  2:30 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

Stefan Beller <sbeller@google.com> writes:

> This is based on ao/submodule-wo-gitmodules-checked-out.

Thanks.  I had an impression that we were not entirely happy with
the topic and are expecting a reroll, but let's hope that the part
we expect to be updated won't have much overlaps.

> A range-diff below shows how picking a different base changed the patches,
> apart from that no further adjustments have been made.

Thanks.  Let's see how well this plays together with other topics in 'pu'.q

^ permalink raw reply	[relevance 5%]

* Re: [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip
  2018-10-16 18:13 [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
                   ` (8 preceding siblings ...)
  2018-10-18  2:30 ` [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip Junio C Hamano
@ 2018-10-18  7:30 ` Junio C Hamano
  2018-10-18 18:00   ` Stefan Beller
  9 siblings, 1 reply; 200+ results
From: Junio C Hamano @ 2018-10-18  7:30 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git

Stefan Beller <sbeller@google.com> writes:

> This is based on ao/submodule-wo-gitmodules-checked-out.
>
> This resends origin/sb/submodule-recursive-fetch-gets-the-tip, resolving
> the issues pointed out via origin/xxx/sb-submodule-recursive-fetch-gets-the-tip-in-pu
> by basing this series on origin/ao/submodule-wo-gitmodules-checked-out

Applying this round to the result of merging ao/submodule-* to
'master' requires this to work, it seems, as you've introduced a
call to repo-init thing in the meantime with another topic.

Subject: [PATCH] fixup! repository: repo_submodule_init to take a submodule struct

---
 builtin/submodule--helper.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 5f8a804a6e..015aa1471f 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -2037,7 +2037,7 @@ static int ensure_core_worktree(int argc, const char **argv, const char *prefix)
 	if (!sub)
 		BUG("We could get the submodule handle before?");
 
-	if (repo_submodule_init(&subrepo, the_repository, path))
+	if (repo_submodule_init(&subrepo, the_repository, sub))
 		die(_("could not get a repository handle for submodule '%s'"), path);
 
 	if (!repo_config_get_string(&subrepo, "core.worktree", &cw)) {
-- 
2.19.1-450-ga4b8ab5363


^ permalink raw reply	[relevance 15%]

* Re: [PATCH 0/9] Resending sb/submodule-recursive-fetch-gets-the-tip
  2018-10-18  7:30 ` Junio C Hamano
@ 2018-10-18 18:00   ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-18 18:00 UTC (permalink / raw)
  To: Junio C Hamano; +Cc: git

On Thu, Oct 18, 2018 at 12:30 AM Junio C Hamano <gitster@pobox.com> wrote:
>
> Stefan Beller <sbeller@google.com> writes:
>
> > This is based on ao/submodule-wo-gitmodules-checked-out.
> >
> > This resends origin/sb/submodule-recursive-fetch-gets-the-tip, resolving
> > the issues pointed out via origin/xxx/sb-submodule-recursive-fetch-gets-the-tip-in-pu
> > by basing this series on origin/ao/submodule-wo-gitmodules-checked-out
>
> Applying this round to the result of merging ao/submodule-* to
> 'master' requires this to work, it seems, as you've introduced a
> call to repo-init thing in the meantime with another topic.

Unfortunately, yes, sb/submodule-update-in-c had one such call.

> > This is based on ao/submodule-wo-gitmodules-checked-out.
> Thanks.  I had an impression that we were not entirely happy with
> the topic and are expecting a reroll, but let's hope that the part
> we expect to be updated won't have much overlaps.

I slowly came to the realization that this topic might be ripped up
into 2 or more topics, as some of the cleanups seem to be orthogonal.

^ permalink raw reply	[relevance 7%]

* [RFC PATCH 1/2] repository: have get_the_repository() to remove the_repository dependency
      [irrelevant]     ` <20181018183758.81186-1-sbeller@google.com>
@ 2018-10-18 18:37       ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-18 18:37 UTC (permalink / raw)
  To: sbeller; +Cc: git, jonathantanmy, stolee

The struct 'the_repo' contains all data that of the main repository.
As we move more and more globals into this struct, the usual way of
accessing these is using 'the_repository', which can be used as a drop in
replacement for accessing the migrated globals.

However during the migration of globals into the repository object, it is
not clear if some code path rely on the_repository or can work on an
arbitrary repository (as we'd eventually want for submodules) due to the
excessive use of the_repository throughout the code base.

To address this, introduce a function 'get_the_repository(void)' which
will return the main repository and set the_repository to NULL when the
environment variable GIT_NO_THE_REPOSITORY is set.

This function is to be strictly used only at the beginning of builtin
command to assign it to a local repository pointer that we'll use to
pass through the code base.

By having the possibility to set the_repository to NULL, we'll get
a segfault when we try to access the_repository instead of using the
handle that we pass around.

This approach let's us have the_repository in the setup code, which
in its current form is not yet able to transition into a world where
the repository handle is passed around and only test the passing around
of the repository handle for later stage code.

Eventually in the future the setup code will produce the repository
handle and each 'cmd_foo(int argc, char **argv)' builtin would get the
repository via an additional parameter.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 repository.c | 10 ++++++++++
 repository.h | 13 ++++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/repository.c b/repository.c
index 5dd1486718..aaaababb88 100644
--- a/repository.c
+++ b/repository.c
@@ -20,6 +20,16 @@ void initialize_the_repository(void)
 	repo_set_hash_algo(&the_repo, GIT_HASH_SHA1);
 }
 
+struct repository *get_the_repository(void)
+{
+	struct repository *r = the_repository;
+
+	if (getenv("GIT_NO_THE_REPOSITORY"))
+		the_repository = NULL;
+
+	return r;
+}
+
 static void expand_base_dir(char **out, const char *in,
 			    const char *base_dir, const char *def_in)
 {
diff --git a/repository.h b/repository.h
index 9f16c42c1e..26f5d64f68 100644
--- a/repository.h
+++ b/repository.h
@@ -114,13 +114,24 @@ void repo_set_gitdir(struct repository *repo, const char *root,
 		     const struct set_gitdir_args *extra_args);
 void repo_set_worktree(struct repository *repo, const char *path);
 void repo_set_hash_algo(struct repository *repo, int algo);
-void initialize_the_repository(void);
 int repo_init(struct repository *r, const char *gitdir, const char *worktree);
 int repo_submodule_init(struct repository *submodule,
 			struct repository *superproject,
 			const char *path);
 void repo_clear(struct repository *repo);
 
+/*
+ * Initializes the repository 'the_repository', which is used in the transition
+ * phase of moving globals into the repository struct.
+ */
+void initialize_the_repository(void);
+
+/*
+ * To be called once; after the call use only returned repository, and do not
+ * use the_repository any more
+ */
+struct repository *get_the_repository(void);
+
 /*
  * Populates the repository's index from its index_file, an index struct will
  * be allocated if needed.
-- 
2.19.0


^ permalink raw reply	[relevance 8%]

* Re: [PATCH 4/9] submodule.c: move global changed_submodule_names into fetch submodule struct
      [irrelevant]   ` <20181017212624.196598-1-jonathantanmy@google.com>
@ 2018-10-18 19:09     ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-18 19:09 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: Junio C Hamano, git

On Wed, Oct 17, 2018 at 2:26 PM Jonathan Tan <jonathantanmy@google.com> wrote:
>
> > The `changed_submodule_names` are only used for fetching, so let's make it
> > part of the struct that is passed around for fetching submodules.
>
> Keep the titles of commit messages to 50 characters or under.

renamed

>
> > +static void calculate_changed_submodule_paths(
> > +     struct submodule_parallel_fetch *spf)
>
> Instead of taking the entire struct, could this just take the list of
> changed submodule names instead?

I think so, done.

^ permalink raw reply	[relevance 5%]

* Re: [PATCH v2 19/24] submodule.c: remove implicit dependency on the_index
      [irrelevant]         ` <20181019164237.GA24740@sigill.intra.peff.net>
@ 2018-10-19 16:57           ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-19 16:57 UTC (permalink / raw)
  To: Jeff King; +Cc: Duy Nguyen, git

> >
> > Yeah. Killing the_index is just the first small step (didn't look that
> > small when I started). Now it's all about the_repository ;-) and we
> > have like 400 references of it just in library code.
>
> I suspect it is much worse than that, even. There are many spots that
> likely are relying on global data that _should_ be in the repository
> struct but aren't yet. I don't think there's even an easy way to count
> those. ;)

This is a very interesting part of the discussion,
please note the series "RFC Bring the_repository into cmd_foo"[1]
as that proposes one way how to deal with this, exposing the
repository/index changes into the existing test suite.

[1] https://public-inbox.org/git/20181018183758.81186-1-sbeller@google.com/

I'll read on.

^ permalink raw reply	[relevance 5%]

* Re: [PATCH/WIP 00/19] Kill the_index, final part
      [irrelevant] <20181019145237.16079-1-pclouds@gmail.com>
@ 2018-10-19 17:38 ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-19 17:38 UTC (permalink / raw)
  To: Duy Nguyen; +Cc: git

On Fri, Oct 19, 2018 at 7:52 AM Nguyễn Thái Ngọc Duy <pclouds@gmail.com> wrote:
>
> Stefan I see you start removing more references of the_repository in
> another series (haven't really looked at it yet) so this is just heads
> up so we could coordinate if needed. Still WIP, not really ready for
> review yet.

I'll take a brief look anyway, otherwise you wouldn't have put in on
a mailing list :-P

Yes, coordination would be good; stolee brought up a good point
regarding testing these changes. Killing of the index in itself can be
just tested by checking if behavior stays the same, but killing of
the_repository is a slightly different beast, as it relates to submodules,
which means any global state that is still around, need to go into
the repository struct (we might have missed some there, whereas
the index struct has been around for a long time).

> This series removes use of the_index outside builtin/ and t/helper/.
> The only the_index reference left is in repository.c. The macro
> NO_THE_REPOSITORY_COMPATIBILITY_MACROS is now flipped to
> USE_THE_INDEX_COMPATIBILITY_MACROS. "the_index" is forbidden by
> default.

Wow, that is really cool. Looking forward for the patches. :-)

> After this I think we can start pushing the_repository outside library
> code. Then perhaps clean them up in builtin code too and you can
> finally get rid of it. But I don't think that'll happen in a year's
> time.

That sounds realistic.

Stefan

^ permalink raw reply	[relevance 4%]

* Re: [PATCH] submodule.c: remove some of the_repository references
      [irrelevant] ` <20181019173443.25760-1-pclouds@gmail.com>
@ 2018-10-19 17:46   ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-19 17:46 UTC (permalink / raw)
  To: Duy Nguyen; +Cc: git, Jeff King, Junio C Hamano

On Fri, Oct 19, 2018 at 10:34 AM Nguyễn Thái Ngọc Duy <pclouds@gmail.com> wrote:
>
>
>  struct collect_changed_submodules_cb_data {
> +       struct repository *repo;

This slightly overlaps with sb/submodule-recursive-fetch-gets-the-tip,
but we can have this patch on its own as I have to rebuild that
series, will build on top of this.

The patch looks good,
Stefan

^ permalink raw reply	[relevance 7%]

* Re: [PATCH 19/19] submodule: don't add submodule as odb for push
  2018-10-16 23:35 ` [PATCH 19/19] submodule: don't add submodule as odb for push Stefan Beller
@ 2018-10-19 20:39   ` Jonathan Tan
  0 siblings, 0 replies; 200+ results
From: Jonathan Tan @ 2018-10-19 20:39 UTC (permalink / raw)
  To: sbeller; +Cc: git, jonathantanmy

> In push_submodule(), because we do not actually need access to objects
> in the submodule, do not invoke add_submodule_odb().
> (for_each_remote_ref_submodule() does not require access to those
> objects, and the actual push is done by spawning another process,
> which handles object access by itself.)

The code looks good - my analysis is the same as that in my review of
the previous version [1].

Can you mention, in the commit message, the tests that exercise the
functionality here (and say that they still pass)?

[1] https://public-inbox.org/git/20181011230028.200488-1-jonathantanmy@google.com/
> 
> Signed-off-by: Stefan Beller <sbeller@google.com>
> ---
>  submodule.c | 3 ---
>  1 file changed, 3 deletions(-)
> 
> diff --git a/submodule.c b/submodule.c
> index 7305ae2e10..e623e6bf7f 100644
> --- a/submodule.c
> +++ b/submodule.c
> @@ -1024,9 +1024,6 @@ static int push_submodule(const char *path,
>  			  const struct string_list *push_options,
>  			  int dry_run)
>  {
> -	if (add_submodule_odb(path))
> -		return 1;
> -
>  	if (for_each_remote_ref_submodule(path, has_remote, NULL) > 0) {
>  		struct child_process cp = CHILD_PROCESS_INIT;
>  		argv_array_push(&cp.args, "push");
> -- 
> 2.19.0
> 

^ permalink raw reply	[relevance 3%]

* Re: Bug with "git mv" and submodules, and with "git submodule add something --force"
      [irrelevant] <c4792c95-9a19-fd0e-9171-fded095b78a6@jku.at>
@ 2018-10-19 20:40 ` Stefan Beller
  2018-10-19 20:58   ` Jonathan Nieder
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-19 20:40 UTC (permalink / raw)
  To: juergen.vogl; +Cc: git

On Fri, Oct 19, 2018 at 5:43 AM Juergen Vogl <juergen.vogl@jku.at> wrote:
>
> Hi there,
>
> tested on both, git 2.18 and git 2.19.1:
>
> moving a file with `git mv` from a project to a submodule results in an
> **undefined state** of the local repository.

Luckily we do have a submodule in git.git itself, so we can
try it out here as well (I'll use a separate worktree to not
hose my main repo):

  git worktree add ../testgit && cd ../testgit
  git checkout v2.19.1 && make install
  git --version
  git version 2.19.1
  git submodule update --init
  Cloning into '/home/sbeller/testgit/sha1collisiondetection'...
  Submodule path 'sha1collisiondetection': checked out
'232357eb2ea0397388254a4b188333a227bf5b10'
  git mv cache.h sha1collisiondetection/
  git status
HEAD detached at v2.19.1
Changes to be committed:
  (use "git reset HEAD <file>..." to unstage)

deleted:    sha1collisiondetection
renamed:    cache.h -> sha1collisiondetection/cache.h

Untracked files:
  (use "git add <file>..." to include in what will be committed)

sha1collisiondetection/.gitignore
sha1collisiondetection/.travis.yml
sha1collisiondetection/LICENSE.txt
sha1collisiondetection/Makefile
sha1collisiondetection/README.md
sha1collisiondetection/lib/
sha1collisiondetection/src/
sha1collisiondetection/test/
sha1collisiondetection/vs2015/

> It breaks up the submodule (it's still in .gitmodules, but not
> accessable via `git submodule`), and is not reversible on local repository.

This seems like what I just did. However reversing can be done via:

  git checkout -f
  git status
HEAD detached at v2.19.1
nothing to commit, working tree clean
  git submodule status
232357eb2ea0397388254a4b188333a227bf5b10 sha1collisiondetection
(stable-v1.0.3-31-g232357e)

So I think it's just "git-mv" that doesn't respect submodule
boundaries, which we would want to address.

The man page of git mv
(https://git-scm.com/docs/git-mv)
actually has a short note about submodules, but that is about
moving *a* submodule not about moving things in and out.

Maybe for now we can do with just an update of the documentation/bugs
section and say we cannot move files in and out of submodules?



>
> Either `git mv submodule/file .`

which is just running the command in reverse, (i.e. swapping
destination and target),
  git mv cache.h sha1collisiondetection/
  git mv sha1collisiondetection/cache.h cache.h
  git status
HEAD detached at v2.19.1
Changes to be committed:
  (use "git reset HEAD <file>..." to unstage)

deleted:    sha1collisiondetection

Untracked files:
  (use "git add <file>..." to include in what will be committed)

sha1collisiondetection/

which seems like the submodule is gone,

> nor deleting the folder works. For the
> locale repo the submodule is gone.

Yup, that seems to be the case.

> But: trying to add it with `git
> submodule add` also do not work and results in an error message (with
> and without `--force` flag):

Would checkout out a state where the submodule still exists
(such as HEAD) and then running "git submodule update --init"
fix it instead?

>
> $ git submodule add git@github.com:-----------/wiki-public.git public
> --force
> A git directory for 'public' is found locally with remote(s):
>   origin        git@github.com:-----------/wiki-public.git
> If you want to reuse this local git directory instead of cloning again from
>   git@github.com:-----------/wiki-public.git
> use the '--force' option. If the local git directory is not the correct repo
> or you are unsure what this means choose another name with the '--name'
> option.
>
> Therefore, it's in a undefined, broken state.
>
>
> Another bug I've got by testing upper line:
> * --force will be used as folder name * when used in `git submodule add
> git@github.com:someone/some.git --force`:

Yes, the order of arguments is important, the options
(such as --force) goes before the URL and path.

> /usr/libexec/git-core/git-submodule: line 273: cd: --: invalid option
> cd: usage: cd [-L|-P] [dir]
> Unable to checkout submodule '--force'

Gah! We'd need to escape the path after the options,
i.e. cd -- --force
would cd into that directory, but I am unsure if that
is accepted by all shells.

Thanks,
Stefan

^ permalink raw reply	[relevance 8%]

* Re: Bug with "git mv" and submodules, and with "git submodule add something --force"
  2018-10-19 20:40 ` Bug with "git mv" and submodules, and with "git submodule add something --force" Stefan Beller
@ 2018-10-19 20:58   ` Jonathan Nieder
  2018-10-22  2:52     ` Junio C Hamano
  0 siblings, 1 reply; 200+ results
From: Jonathan Nieder @ 2018-10-19 20:58 UTC (permalink / raw)
  To: Stefan Beller; +Cc: juergen.vogl, git

Stefan Beller wrote:

> Maybe for now we can do with just an update of the documentation/bugs
> section and say we cannot move files in and out of submodules?

I think we have some existing logic to prevent "git add"-ing a file
within a submodule to the superproject, for example.

So "git mv" should learn the same trick.  And perhaps the trick needs
to be moved down a layer (e.g. into the index API).  Hints?

Thanks,
Jonathan

^ permalink raw reply	[relevance 8%]

* Re: Bug with "git mv" and submodules, and with "git submodule add something --force"
  2018-10-19 20:58   ` Jonathan Nieder
@ 2018-10-22  2:52     ` Junio C Hamano
  2018-10-22 21:52       ` Stefan Beller
  0 siblings, 1 reply; 200+ results
From: Junio C Hamano @ 2018-10-22  2:52 UTC (permalink / raw)
  To: Jonathan Nieder; +Cc: Stefan Beller, juergen.vogl, git

Jonathan Nieder <jrnieder@gmail.com> writes:

> Stefan Beller wrote:
>
>> Maybe for now we can do with just an update of the documentation/bugs
>> section and say we cannot move files in and out of submodules?
>
> I think we have some existing logic to prevent "git add"-ing a file
> within a submodule to the superproject, for example.

There is die_path_inside_submodule() that sanity-checks the pathspec
and rejects.  But I think that is done primarily to give an error
message and not strictly necesary for correctness.

The real safety of "git add" is its call to dir.c::fill_directory();
it collects untracked paths that match the pathspec so that they can
be added as new paths, but because it won't cross the module
boundary, you won't get such a path in the index to begin with.

> So "git mv" should learn the same trick.  And perhaps the trick needs
> to be moved down a layer (e.g. into the index API).  Hints?

You would want to be able to remove a submodule and replace it with
a directory, but you can probably do it in two steps, i.e.

	git reset --hard
	git rm --cached sha1collisiondetection
	echo Now a regular dir >sha1collisiondetection/READ.ME
	find sha1collisiondetection ! -type d -print0 | 
	git update-index --add --stdin -z

So from that point of view, forbidding (starting from the same state
of our project) this sequence:

	git reset --hard
	echo Now a regular dir >sha1collisiondetection/READ.ME
	find sha1collisiondetection ! -type d -print0 | 
	git update-index --add --remove --stdin -z

that would nuke the submodule and replace it with a directory within
which there are files would be OK.  Making the latter's default
rejection overridable with ADD_CACHE_OK_TO_REPLACE would also be
fine.


^ permalink raw reply	[relevance 5%]

* Bug with "git mv" and submodules, and with "git submodule add something --force"
  2018-10-22  2:52     ` Junio C Hamano
@ 2018-10-22 21:52       ` Stefan Beller
  2018-10-24  7:18         ` Junio C Hamano
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-22 21:52 UTC (permalink / raw)
  To: gitster; +Cc: git, jrnieder, juergen.vogl, sbeller

On Sun, Oct 21, 2018 at 7:52 PM Junio C Hamano <gitster@pobox.com> wrote:
>
> Jonathan Nieder <jrnieder@gmail.com> writes:
>
> > Stefan Beller wrote:
> >
> >> Maybe for now we can do with just an update of the documentation/bugs
> >> section and say we cannot move files in and out of submodules?
> >
> > I think we have some existing logic to prevent "git add"-ing a file
> > within a submodule to the superproject, for example.
>
> There is die_path_inside_submodule() that sanity-checks the pathspec
> and rejects.  But I think that is done primarily to give an error
> message and not strictly necesary for correctness.

c08397e3aa (pathspec: remove PATHSPEC_STRIP_SUBMODULE_SLASH_EXPENSIVE
flag, 2017-05-11) seems to be relevant here, as that factors out the
warning.

> The real safety of "git add" is its call to dir.c::fill_directory();
> it collects untracked paths that match the pathspec so that they can
> be added as new paths, but because it won't cross the module
> boundary, you won't get such a path in the index to begin with.

It would not cross the boundary and fail silently as it would
treat a path into the submodule as a no-op.

> > So "git mv" should learn the same trick.  And perhaps the trick needs
> > to be moved down a layer (e.g. into the index API).  Hints?

Yeah, I think we'd want to teach git-mv about that trick.

Unfortunately git-mv is one of the last remainders of not
properly using pathspecs, and die_path_inside_submodule
expects a pathspec. :-/

> You would want to be able to remove a submodule and replace it with
> a directory, but you can probably do it in two steps, i.e.
>
>         git reset --hard
>         git rm --cached sha1collisiondetection
>         echo Now a regular dir >sha1collisiondetection/READ.ME
>         find sha1collisiondetection ! -type d -print0 |
>         git update-index --add --stdin -z

    "Ignoring path sha1collisiondetection/.git"

Nice!

>
> So from that point of view, forbidding (starting from the same state
> of our project) this sequence:
>
>         git reset --hard
>         echo Now a regular dir >sha1collisiondetection/READ.ME
>         find sha1collisiondetection ! -type d -print0 |
>         git update-index --add --remove --stdin -z
>
> that would nuke the submodule and replace it with a directory within
> which there are files would be OK.  Making the latter's default
> rejection overridable with ADD_CACHE_OK_TO_REPLACE would also be
> fine.

I am having trouble of relating these commands to the original git-mv
across submodule boundaries.

Moving files from the submodule out to the superproject, seems
to fail properly, though having a less-than-optimal error message:

$ git mv sha1collisiondetection/LICENSE.txt .
fatal: not under version control, source=sha1collisiondetection/LICENSE.txt, destination=LICENSE.txt

and moving things inside was the original report, below is a proof of concept
that would yield

./git mv -v cache.h sha1collisiondetection/
fatal: moving across submodule boundaries not supported, source=cache.h, destination=sha1collisiondetection/cache.h

--8<--
Subject: [WIP/PATCH] builtin/mv.c: disallow moving across submodule boundaries

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 builtin/mv.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/builtin/mv.c b/builtin/mv.c
index 80bb967a63..9ec4b2f0a3 100644
--- a/builtin/mv.c
+++ b/builtin/mv.c
@@ -172,7 +172,7 @@ int cmd_mv(int argc, const char **argv, const char *prefix)
 	/* Checking */
 	for (i = 0; i < argc; i++) {
 		const char *src = source[i], *dst = destination[i];
-		int length, src_is_dir;
+		int length, src_is_dir, pos;
 		const char *bad = NULL;
 
 		if (show_only)
@@ -243,6 +243,13 @@ int cmd_mv(int argc, const char **argv, const char *prefix)
 		else
 			string_list_insert(&src_for_dst, dst);
 
+		pos = cache_name_pos(dst, strlen(dst));
+		if (pos < 0) {
+			pos = -(pos + 1);
+			if (!S_ISGITLINK(active_cache[pos]->ce_mode))
+				bad = _("moving across submodule boundaries not supported");
+		}
+
 		if (!bad)
 			continue;
 		if (!ignore_errors)
-- 
2.19.0


^ permalink raw reply	[relevance 13%]

* Re: [PATCH 7/9] submodule: fetch in submodules git directory instead of in worktree
      [irrelevant]   ` <20181017225811.66554-1-jonathantanmy@google.com>
@ 2018-10-23 18:26     ` Stefan Beller
      [irrelevant]       ` <20181023225536.61508-1-jonathantanmy@google.com>
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-23 18:26 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: Junio C Hamano, git

On Wed, Oct 17, 2018 at 3:58 PM Jonathan Tan <jonathantanmy@google.com> wrote:
>
> > This patch started as a refactoring to make 'get_next_submodule' more
> > readable, but upon doing so, I realized that "git fetch" of the submodule
> > actually doesn't need to be run in the submodules worktree. So let's run
> > it in its git dir instead.
>
> The commit message needs to be updated, I think - this patch does
> significantly more than fetching in the gitdir.

From my point of view, it is not significant, but refactoring.
I'll think how to write a better commit message.

> > This patch leaks the cp->dir in get_next_submodule, as any further
> > callback in run_processes_parallel doesn't have access to the child
> > process any more.
>
> The cp->dir is already leaked - probably better to write "cp->dir in
> get_next_submodule() is still leaked, but this will be fixed in a
> subsequent patch".

... which fails to mention the reason why (as it is hard to do given
the current design) but is more concise.

> > +static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out)
> > +{
> > +     prepare_submodule_repo_env_no_git_dir(out);
> > +     argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
>
> Why does GIT_DIR need to be set? Is it to avoid subcommands recursively
> checking the parent directories in case the CWD is a malformed Git
> repository? If yes, maybe it's worth adding a comment.

It is copying the structure from prepare_submodule_repo_env,
specifically 10f5c52656 (submodule: avoid auto-discovery in
prepare_submodule_repo_env(), 2016-09-01), which sounds
appealing (and brings real benefits for the working directory),
but I have not thought about this protection for the git dir.

Maybe another approach is to not set the cwd for the child process
and instead point GIT_DIR_ENVIRONMENT only to the right
directory.

Then the use of GIT_DIR_ENVIRONMENT is obvious and
is not just for protection of corner cases.

However I think this protection is really valuable for the
.git dir as well as the submodule may be broken and we do not
want to end up in an infinite loop (as the discovery would find
the superproject which then tries to recurse, again, into the
submodule with the broken git dir)

When adding the comment here, we'd also want to have
the comment in prepare_submodule_repo_env, which
could be its own preparation commit.

> > +static struct repository *get_submodule_repo_for(struct repository *r,
> > +                                              const struct submodule *sub)
> > +{
> > +     struct repository *ret = xmalloc(sizeof(*ret));
> > +
> > +     if (repo_submodule_init(ret, r, sub)) {
> > +             /*
> > +              * No entry in .gitmodules? Technically not a submodule,
> > +              * but historically we supported repositories that happen to be
> > +              * in-place where a gitlink is. Keep supporting them.
> > +              */
> > +             struct strbuf gitdir = STRBUF_INIT;
> > +             strbuf_repo_worktree_path(&gitdir, r, "%s/.git", sub->path);
> > +             if (repo_init(ret, gitdir.buf, NULL)) {
> > +                     strbuf_release(&gitdir);
> > +                     return NULL;
> > +             }
> > +             strbuf_release(&gitdir);
> > +     }
> > +
> > +     return ret;
> > +}
>
> This is the significant thing that this patch does more - an unskipped
> submodule is now something that either passes the checks in
> repo_submodule_init() or the checks in repo_init(), which seems to be
> stricter than the current check that ".git" points to a directory or is
> one. This means that we skip certain broken repositories, and this
> necessitates a change in the test.

I see. However there is no change in function, the check in repo_init
(or repo_submodule_init) is less strict than the check in the child process.
So if there are broken submodule repositories, the difference of this
patch is the layer at which it is caught, i.e. we would not spawn a child
that fails, but skip the submodule.

Thinking of that, maybe we need to announce that in get_next_submodule

>
> I think we should be more particular about what we're allowed to skip -
> in particular, maybe if we're planning to skip this submodule, its
> corresponding directory in the worktree (if one exists) needs to be
> empty.

If the working tree directory is empty for that submodule, it means
it is likely not initialized. But why would we use that as a signal to
skip the submodule?



> > -                     cp->dir = strbuf_detach(&submodule_path, NULL);
> > -                     prepare_submodule_repo_env(&cp->env_array);
> > +                     prepare_submodule_repo_env_in_gitdir(&cp->env_array);
> > +                     cp->dir = xstrdup(repo->gitdir);
>
> Here is where the functionality change (fetch in ".git") described in
> the commit message occurs.

True.

Thanks for the review, I'll try to split up this commit a bit more and
explain each part on its own.

^ permalink raw reply	[relevance 8%]

* Re: git pull defaults for recursesubmodules
      [irrelevant] <CAGshahkvn3fcyuqtD-WQE9tn+7rSad84+mtA_cfkz+t42xqPdw@mail.gmail.com>
@ 2018-10-23 21:57 ` Stefan Beller
  2018-10-24  5:32   ` Tommi Vainikainen
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-23 21:57 UTC (permalink / raw)
  To: tvainika; +Cc: git

On Tue, Oct 23, 2018 at 2:04 PM Tommi Vainikainen <tvainika@gmail.com> wrote:
>
> I configured my local git to fetch with recurseSubmodules = on-demand,
> which I found the most convenient setting. However then I noticed that
> I mostly use git pull actually to fetch from remotes, but git pull
> does not utilize any recurseSubmoddules setting now, or at least I
> could not find such.
>
> I would expect that if git-config has fetch.recurseSubmodules set,
> also git pull should use this setting, or at least similar option such
> as pull.recurseSubmodules should be available. I'd prefer sharing
> fetch.recurseSubmodules setting here.
>
> I've attached a minimal patch, which I believe implements this
> configuration usage, and a test case to show my expected behavior for
> git pull.

This makes sense to me and the patch looks good to me.
It is unclear to me if this is a regression or an oversight of
of a6d7eb2c7a (pull: optionally rebase submodules (remote
submodule changes only), 2017-06-23)

Thanks,
Stefan

^ permalink raw reply	[relevance 5%]

* Re: [PATCH 8/9] fetch: retry fetching submodules if needed objects were not fetched
      [irrelevant]   ` <20181018003954.139498-1-jonathantanmy@google.com>
@ 2018-10-23 22:37     ` Stefan Beller
      [irrelevant]       ` <20181023233756.190026-1-jonathantanmy@google.com>
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-23 22:37 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: Junio C Hamano, git

On Wed, Oct 17, 2018 at 5:40 PM Jonathan Tan <jonathantanmy@google.com> wrote:
>
> > Currently when git-fetch is asked to recurse into submodules, it dispatches
> > a plain "git-fetch -C <submodule-dir>" (with some submodule related options
> > such as prefix and recusing strategy, but) without any information of the
> > remote or the tip that should be fetched.
> >
> > This works surprisingly well in some workflows (such as using submodules
> > as a third party library), while not so well in other scenarios, such
> > as in a Gerrit topic-based workflow, that can tie together changes
> > (potentially across repositories) on the server side. One of the parts
> > of such a Gerrit workflow is to download a change when wanting to examine
> > it, and you'd want to have its submodule changes that are in the same
> > topic downloaded as well. However these submodule changes reside in their
> > own repository in their own ref (refs/changes/<int>).
>
> It seems that the pertinent situation is when, in the superproject, you
> fetch a commit (which may be the target of a ref, or an ancestor of the
> target of a ref) that points to a submodule commit that was not fetched
> by the default refspec-less fetch that you describe in the first
> paragraph. I would just describe it as follows:
>
>   But this default fetch is not sufficient, as a newly fetched commit in
>   the superproject could point to a commit in the submodule that is not
>   in the default refspec. This is common in workflows like Gerrit's.
>   When fetching a Gerrit change under review (from refs/changes/??), the
>   commits in that change likely point to submodule commits that have not
>   been merged to a branch yet.

Makes sense.

> Another thing you need to clarify is what happens if the fetch-by-commit
> fails. Right now, it seems that it will make the whole thing fail, which
> might be a surprising change in behavior.

But a positive surprise, I would assume?

> The test stores the result in a normal branch, not a remote tracking
> branch. Is storing in a normal branch required?

In the test we fetch from another repository, such that in the
repository-under-test this will show up in a remote tracking branch?

> Also, do you know why this is required? A naive reading of the patch
> leads me to believe that this should work even if merely fetching to
> FETCH_HEAD.

See the next patch, check_for_new_submodule_commits() is missing
for FETCH_HEAD.

>
> > +struct get_next_submodule_task {
> > +     struct repository *repo;
> > +     const struct submodule *sub;
> > +     unsigned free_sub : 1; /* Do we need to free the submodule? */
> > +     struct oid_array *commits;
> > +};
>
> Firstly, I don't think "commits" needs to be a pointer.
>
> Document at least "commits". As far as I can tell, if NULL (or empty if
> you take my suggestion), this means that this task is the first pass for
> this particular submodule and the fetch needs to be done using the
> default refspec. Otherwise, this task is the second pass for this
> particular submodule and the fetch needs to be done passing the
> contained OIDs.

Makes sense. I think I'll make it a non-pointer, but will introduce another
flag or counter for the phase.

>
> > +static const struct submodule *get_default_submodule(const char *path)
> > +{
> > +     struct submodule *ret = NULL;
> > +     const char *name = default_name_or_path(path);
> > +
> > +     if (!name)
> > +             return NULL;
> > +
> > +     ret = xmalloc(sizeof(*ret));
> > +     memset(ret, 0, sizeof(*ret));
> > +     ret->path = name;
> > +     ret->name = name;
> > +
> > +     return (const struct submodule *) ret;
> > +}
>
> What is a "default" submodule and why would you need one?

s/default/artificial/. Such a submodule is a submodule that has no
config in the .gitmodules file and its name == path.
We need to keep those around for historic reasons AFAICT, c.f.
c68f837576 (implement fetching of moved submodules, 2017-10-16)

> > +             task = get_next_submodule_task_create(spf->r, ce->name);
> > +
> > +             if (!task->sub) {
> > +                     free(task);
> > +                     continue;
> >               }
>
> Will task->sub ever be NULL?

Yes, if we fall back to these "default" submodule and just try if it
can be handled
as a submodule, but it cannot be handled as such,
get_next_submodule_task_create has

    task->sub = submodule_from_path(r, &null_oid, path);
    if (!task->sub) {
        task->sub = get_default_submodule(path);

and get_default_submodule can return NULL.


>
> > +     if (spf->retry_nr) {
> > +             struct get_next_submodule_task *task = spf->retry[spf->retry_nr - 1];
> > +             struct strbuf submodule_prefix = STRBUF_INIT;
> > +             spf->retry_nr--;
> > +
> > +             strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, task->sub->path);
> > +
> > +             child_process_init(cp);
> > +             prepare_submodule_repo_env_in_gitdir(&cp->env_array);
> > +             cp->git_cmd = 1;
> > +             cp->dir = task->repo->gitdir;
> > +
> > +             argv_array_init(&cp->args);
> > +             argv_array_pushv(&cp->args, spf->args.argv);
> > +             argv_array_push(&cp->args, "on-demand");
>
> This means that we need to trust that the last entry in spf->args can
> take an "on-demand" parameter. Could we supply that argument here
> explicitly instead?
>
> > +             argv_array_push(&cp->args, "--submodule-prefix");
> > +             argv_array_push(&cp->args, submodule_prefix.buf);
> > +
> > +             /* NEEDSWORK: have get_default_remote from s--h */
>
> What is s--h?

builtin/submodule--helper, will clarify

>
> > +static int commit_exists_in_sub(const struct object_id *oid, void *data)
> > +{
> > +     struct repository *subrepo = data;
> > +
> > +     enum object_type type = oid_object_info(subrepo, oid, NULL);
> > +
> > +     return type != OBJ_COMMIT;
> > +}
>
> Shouldn't the function name be commit_missing_in_sub?

yes.

>
> >  static int fetch_finish(int retvalue, struct strbuf *err,
> >                       void *cb, void *task_cb)
> >  {
> >       struct submodule_parallel_fetch *spf = cb;
> > +     struct get_next_submodule_task *task = task_cb;
> > +     const struct submodule *sub;
> > +
> > +     struct string_list_item *it;
> > +     struct oid_array *commits;
> >
> >       if (retvalue)
> >               spf->result = 1;
> >
> > +     if (!task)
> > +             return 0;
>
> When will task be NULL?
>
> > +
> > +     sub = task->sub;
> > +     if (!sub)
> > +             goto out;
>
> Same as above - when will sub be NULL?
>
> > +     it = string_list_lookup(&spf->changed_submodule_names, sub->name);
> > +     if (!it)
> > +             goto out;
>
> And "it" as well.

I'll add comments.

>
> > +     commits = it->util;
> > +     oid_array_filter(commits,
> > +                      commit_exists_in_sub,
> > +                      task->repo);
> > +
> > +     /* Are there commits that do not exist? */
> > +     if (commits->nr) {
> > +             /* We already tried fetching them, do not try again. */
> > +             if (task->commits)
> > +                     return 0;
>
> Clearer and more efficient if the check for task->commits was first
> before all the filtering.
>
> > +test_expect_success "fetch new commits on-demand when they are not reachable" '
>
> "not reachable" confused me - they are indeed reachable, just not from
> the default refspec.

Makes sense

>
> > +     git checkout --detach &&
> > +     C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
> > +     git -C submodule update-ref refs/changes/1 $C &&
> > +     git update-index --cacheinfo 160000 $C submodule &&
> > +     git commit -m "updated submodule outside of refs/heads" &&
> > +     D=$(git rev-parse HEAD) &&
> > +     git update-ref refs/changes/2 $D &&
> > +     (
> > +             cd downstream &&
> > +             git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
> > +             git -C submodule cat-file -t $C &&
> > +             git checkout --recurse-submodules FETCH_HEAD
> > +     )
> > +'
>
> For maximum test coverage, I think this test should involve 2
> submodules:
>
>  B   C  E   F
>   \ /    \ /
>    A      D
>
> and the upstream superproject having:
>
>   G -> H -> I
>
> The downstream superproject already has G, and is fetching I. In H, the
> submodule gitlinks point to B and E respectively, and in I, the
> submodule gitlinks point to C and F respectively. This ensures that both
> multiple submodules work, and that submodule commits are also fetched
> for interior superproject commits.

ok.

^ permalink raw reply	[relevance 8%]

* Re: [PATCH 7/9] submodule: fetch in submodules git directory instead of in worktree
      [irrelevant]       ` <20181023225536.61508-1-jonathantanmy@google.com>
@ 2018-10-23 23:01         ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-23 23:01 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: Junio C Hamano, git

On Tue, Oct 23, 2018 at 3:55 PM Jonathan Tan <jonathantanmy@google.com> wrote:
> > When adding the comment here, we'd also want to have
> > the comment in prepare_submodule_repo_env, which
> > could be its own preparation commit.
>
> I agree with the protection. As for the preparation commit, I don't
> think it's always the code author's responsibility to tidy up the
> surrounding code, but since you're adding an identical comment here,
> it's probably worth it to add the comment there too.

Am I the only one who dislikes inconsistent files? ;-)
(ie. clean in one place, not cleaned up in another)
I can see your point. Will add a comment

> > Thinking of that, maybe we need to announce that in get_next_submodule
>
> The consequence of getting caught changes, though. Currently,
> spf->result is set to 1 whenever a child process fails. But in this
> patch, some of these repositories would be entirely skipped, meaning
> that no child process is run, and spf->result is never modified.

Right.

> > If the working tree directory is empty for that submodule, it means
> > it is likely not initialized. But why would we use that as a signal to
> > skip the submodule?
>
> What I meant was: if empty, skip it completely. Otherwise, do the
> repo_submodule_init() and repo_init() thing, and if they both fail, set
> spf->result to 1, preserving existing behavior.

I did it the other way round:

If repo_[submodule_]init fails, see if we have a gitlink in tree and
an empty dir in the FS, to decide if we need to signal failure.

I can switch it around again, but it seemed easier to write as
that puts corner cases away into one else {} case.

^ permalink raw reply	[relevance 5%]

* Re: git pull defaults for recursesubmodules
  2018-10-23 21:57 ` git pull defaults for recursesubmodules Stefan Beller
@ 2018-10-24  5:32   ` Tommi Vainikainen
  0 siblings, 0 replies; 200+ results
From: Tommi Vainikainen @ 2018-10-24  5:32 UTC (permalink / raw)
  To: sbeller; +Cc: git

ke 24. lokak. 2018 klo 0.57 Stefan Beller (sbeller@google.com) kirjoitti:
> On Tue, Oct 23, 2018 at 2:04 PM Tommi Vainikainen <tvainika@gmail.com> wrote:
> > I would expect that if git-config has fetch.recurseSubmodules set,
> > also git pull should use this setting, or at least similar option such
>
> This makes sense to me and the patch looks good to me.
> It is unclear to me if this is a regression or an oversight of
> of a6d7eb2c7a (pull: optionally rebase submodules (remote
> submodule changes only), 2017-06-23)

With my testing, no it was not regression at least from that commit.
It did not work as I expected before that commit.

What was unclear to me is why this config needs to be read as
pull calls fetch, why fetch does not use this configuration as is?
If fetch.prune=1, should git pull command also prune or not during
fetch? There does not seem to be test case for that behavior.

-- 
Tommi Vainikainen

^ permalink raw reply	[relevance 2%]

* Re: Bug with "git mv" and submodules, and with "git submodule add something --force"
  2018-10-22 21:52       ` Stefan Beller
@ 2018-10-24  7:18         ` Junio C Hamano
  0 siblings, 0 replies; 200+ results
From: Junio C Hamano @ 2018-10-24  7:18 UTC (permalink / raw)
  To: Stefan Beller; +Cc: git, jrnieder, juergen.vogl

Stefan Beller <sbeller@google.com> writes:

>> You would want to be able to remove a submodule and replace it with
>> a directory, but you can probably do it in two steps, i.e.
>>
>>         git reset --hard
>>         git rm --cached sha1collisiondetection
>>         echo Now a regular dir >sha1collisiondetection/READ.ME
>>         find sha1collisiondetection ! -type d -print0 |
>>         git update-index --add --stdin -z
>
>     "Ignoring path sha1collisiondetection/.git"
>
> Nice!

There actually is another possible outcome that anybody following
along must be aware of and be careful about: not even .git directory
exist there, i.e. it is possible that the submodule has never been
init'ed.

So, it is not that nice X-<.

^ permalink raw reply	[relevance 8%]

* Re: [PATCH 18/19] submodule: use submodule repos for object lookup
  2018-10-16 23:35 ` [PATCH 18/19] submodule: use submodule repos for object lookup Stefan Beller
@ 2018-10-25  9:14   ` SZEDER Gábor
  2018-10-31 13:38   ` Derrick Stolee
  1 sibling, 0 replies; 200+ results
From: SZEDER Gábor @ 2018-10-25  9:14 UTC (permalink / raw)
  To: Stefan Beller; +Cc: Derrick Stolee, git, jonathantanmy

On Tue, Oct 16, 2018 at 04:35:49PM -0700, Stefan Beller wrote:
> This converts the 'show_submodule_header' function to use
> the repository API properly, such that the submodule objects
> are not added to the main object store.

This patch breaks the test suite with 'GIT_TEST_COMMIT_GRAPH=1', in
particular 't4041-diff-submodule-option.sh' fails with:

  expecting success:
          git diff-index -p --submodule=log HEAD >actual &&
          cat >expected <<-EOF &&
          Submodule sm1 $head2..$head3 (rewind):
            < Add foo3 ($added foo3)
            < Add foo2 ($added foo2)
          EOF
          test_cmp expected actual
  
  + git diff-index -p --submodule=log HEAD
  + cat
  + test_cmp expected actual
  + diff -u expected actual
  --- expected    2018-10-25 09:10:00.541610016 +0000
  +++ actual      2018-10-25 09:10:00.537609885 +0000
  @@ -1,3 +1,5 @@
  -Submodule sm1 30b9670..dafb207 (rewind):
  +Submodule sm1 30b9670...dafb207:
     < Add foo3 (hinzugefügt foo3)
     < Add foo2 (hinzugefügt foo2)
  +  > Add foo1 (hinzugefügt foo1)
  +  < Add foo1 (hinzugefügt foo1)
  error: last command exited with $?=1
  not ok 9 - modified submodule(backward)

and 't4060-diff-submodule-option-diff-format.sh' fails with:

  expecting success:
          git diff-index -p --submodule=diff HEAD >actual &&
          cat >expected <<-EOF &&
          Submodule sm1 $head2..$head3 (rewind):
          diff --git a/sm1/foo2 b/sm1/foo2
          deleted file mode 100644
          index 54b060e..0000000
          --- a/sm1/foo2
          +++ /dev/null
          @@ -1 +0,0 @@
          -foo2
          diff --git a/sm1/foo3 b/sm1/foo3
          deleted file mode 100644
          index c1ec6c6..0000000
          --- a/sm1/foo3
          +++ /dev/null
          @@ -1 +0,0 @@
          -foo3
          EOF
          test_cmp expected actual
  
  + git diff-index -p --submodule=diff HEAD
  + cat
  + test_cmp expected actual
  + diff -u expected actual
  --- expected    2018-10-25 09:10:38.854868800 +0000
  +++ actual      2018-10-25 09:10:38.854868800 +0000
  @@ -1,4 +1,4 @@
  -Submodule sm1 30b9670..dafb207 (rewind):
  +Submodule sm1 30b9670...dafb207:
   diff --git a/sm1/foo2 b/sm1/foo2
   deleted file mode 100644
   index 54b060e..0000000
  error: last command exited with $?=1
  not ok 10 - modified submodule(backward)



^ permalink raw reply	[relevance 8%]

* [PATCH v7 00/10] Make submodules work if .gitmodules is not checked out
@ 2018-10-25 16:18 Antonio Ospite
  2018-10-25 18:49 ` Stefan Beller
  0 siblings, 1 reply; 200+ results
From: Antonio Ospite @ 2018-10-25 16:18 UTC (permalink / raw)
  To: gitster
  Cc: git, Jonathan Nieder, Stefan Beller, Jeff King,
	SZEDER Gábor, Antonio Ospite

Hi,

this series teaches git to try and read the .gitmodules file from the
index (:.gitmodules) or from the current branch (HEAD:.gitmodules) when
the file is not readily available in the working tree.

This can be used, together with sparse checkouts, to enable submodule
usage with programs like vcsh[1] which manage multiple repositories with
their working trees sharing the same path.

[1] https://github.com/RichiH/vcsh

In the previous series there were some comments about not using the enum
in patch 8, but I decided to leave the code as it was as I still think
that it make sense to use an enum there, and have the default value
unnamed; during the discussion I pointed out that other code in git do
the same.

In this series I am addressing the comments by Stefan Beller about the
tests in patch 9.

If the new tests look OK, I'd say we try moving the series to "next" and
see what happens?

I am available to address any further concerns in follow-up patches.

v6 of the series is here:
https://public-inbox.org/git/20181005130601.15879-1-ao2@ao2.it/

v5 of the series is here:
https://public-inbox.org/git/20180917140940.3839-1-ao2@ao2.it/

v4 of the series is here:
https://public-inbox.org/git/20180824132951.8000-1-ao2@ao2.it/

v3 of the series is here:
https://public-inbox.org/git/20180814110525.17801-1-ao2@ao2.it/

v2 of the series is here:
https://public-inbox.org/git/20180802134634.10300-1-ao2@ao2.it/

v1 of the series, with some background info, is here:
https://public-inbox.org/git/20180514105823.8378-1-ao2@ao2.it/

Changes since v6:

  * Renamed t7416-submodule-sparse-gitmodules.sh to
    t7418-submodule-sparse-gitmodules.sh because t7416 was already
    taken.  This has been already taken care of by Junio in "pu".

  * Improved tests in t7418: now, instead of just testing the return
    value of "git submodule ..." commands when .gitmodules is not in the
    working tree, the actual use case is checked in each test, with pre-
    and post-conditions.

Thank you,
   Antonio

Antonio Ospite (10):
  submodule: add a print_config_from_gitmodules() helper
  submodule: factor out a config_set_in_gitmodules_file_gently function
  t7411: merge tests 5 and 6
  t7411: be nicer to future tests and really clean things up
  submodule--helper: add a new 'config' subcommand
  submodule: use the 'submodule--helper config' command
  t7506: clean up .gitmodules properly before setting up new scenario
  submodule: add a helper to check if it is safe to write to .gitmodules
  submodule: support reading .gitmodules when it's not in the working
    tree
  t/helper: add test-submodule-nested-repo-config

 Makefile                                     |   1 +
 builtin/grep.c                               |  17 ++-
 builtin/submodule--helper.c                  |  40 ++++++
 cache.h                                      |   2 +
 git-submodule.sh                             |  13 +-
 submodule-config.c                           |  68 ++++++++-
 submodule-config.h                           |   2 +
 submodule.c                                  |  28 +++-
 submodule.h                                  |   1 +
 t/helper/test-submodule-nested-repo-config.c |  30 ++++
 t/helper/test-tool.c                         |   1 +
 t/helper/test-tool.h                         |   1 +
 t/t7411-submodule-config.sh                  | 141 +++++++++++++++++--
 t/t7418-submodule-sparse-gitmodules.sh       | 122 ++++++++++++++++
 t/t7506-status-submodule.sh                  |   3 +-
 t/t7814-grep-recurse-submodules.sh           |  16 +++
 16 files changed, 454 insertions(+), 32 deletions(-)
 create mode 100644 t/helper/test-submodule-nested-repo-config.c
 create mode 100755 t/t7418-submodule-sparse-gitmodules.sh

-- 
Antonio Ospite
https://ao2.it
https://twitter.com/ao2it

A: Because it messes up the order in which people normally read text.
   See http://en.wikipedia.org/wiki/Posting_style
Q: Why is top-posting such a bad thing?

^ permalink raw reply	[relevance 7%]

* Re: Confusing behavior with ignored submodules and `git commit -a`
      [irrelevant] ` <CAGw6cBvaC+TEOM9Tjdbs5zkz2hzW4649=4rsAo58cNOVHOQS=Q@mail.gmail.com>
@ 2018-10-25 18:26   ` Stefan Beller
  2018-11-15  5:12     ` Michael Forney
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-25 18:26 UTC (permalink / raw)
  To: mforney; +Cc: git

On Thu, Oct 25, 2018 at 11:03 AM Michael Forney <mforney@mforney.org> wrote:
>
> On 2018-03-16, Michael Forney <mforney@mforney.org> wrote:
> > Hi,
> >
> > In the past few months have noticed some confusing behavior with
> > ignored submodules. I finally got around to bisecting this to commit
> > 5556808690ea245708fb80383be5c1afee2fb3eb (add, reset: ensure
> > submodules can be added or reset).

Uh. :(

See the discussion starting at
https://public-inbox.org/git/20170725213928.125998-4-bmwill@google.com/
specifically
https://public-inbox.org/git/xmqqinieq49v.fsf@gitster.mtv.corp.google.com/


> >
> > Here is a demonstration of the problem:
> >
[...]
> > Up to here is all expected.

Makes sense.

> > However, if I go to update `foo.txt` and
> > commit with `git commit -a`, changes to inner get recorded
> > unexpectedly. What's worse is the shortstat output of `git commit -a`,
> > and the diff output of `git show` give no indication that the
> > submodule was changed.

This is really bad. git-status and git-commit share some code,
and we'll populate the commit message with a status output.
So it seems reasonable to expect the status and the commit to match,
i.e. if status tells me there is no change, then commit should not record
the submodule update.

> > $ git commit -a -m 'update foo.txt'
> > [master 6ec564c] update foo.txt
> >  1 file changed, 1 insertion(+), 1 deletion(-)
> > $ git show
> > commit 6ec564c15ddae099c71f01750b4c434557525653 (HEAD -> master)
> > Author: Michael Forney <mforney@mforney.org>
> > Date:   Fri Mar 16 20:18:37 2018 -0700
> >
> >     update foo.txt
> >
> > diff --git a/foo.txt b/foo.txt
> > index d00491f..0cfbf08 100644
> > --- a/foo.txt
> > +++ b/foo.txt
> > @@ -1 +1 @@
> > -1
> > +2
> > $
> >
> > There have been a couple occasions where I accidentally pushed local
> > changes to ignored submodules because of this. Since they don't show
> > up in the log output, it is difficult to figure out what actually has
> > gone wrong.

How was it prevented before? Just by git commit -a not picking up the
submodule change?

> >
> > Anyway, since the bisected commit (555680869) only mentions add and
> > reset, I'm assuming that this is a regression and not a deliberate
> > behavior change. The documentation for submodule.<name>.ignore states
> > that the setting should only affect `git status` and the diff family.
> > In terms of my expectations, I would go further and say it should only
> > affect `git status` and diffs against the working tree.
> >
> > I took a brief look through the relevant sources, and it wasn't clear
> > to me how to fix this without accidentally changing the behavior of
> > other subcommands.
> >
> > Any help with this issue is appreciated!

I guess reverting that commit is not a good idea now, as
I would expect something to break.

Maybe looking through the series 614ea03a71
(Merge branch 'bw/submodule-config-cleanup', 2017-08-26)
to understand why it happened in the context would be a good start.

> I accidentally pushed local changes to ignored submodules again due to this.
>
> Can anyone confirm whether this is the intended behavior of ignore? If
> it is, then at least the documentation needs an update saying that
> `commit -a` will commit all submodule changes, even if they are
> ignored.

The docs say "(but it will nonetheless show up in the output of
status and commit when it has been staged)" as well, so that commit
sounds like a regression?

Stefan

^ permalink raw reply	[relevance 7%]

* Re: [PATCH v7 00/10] Make submodules work if .gitmodules is not checked out
  2018-10-25 16:18 [PATCH v7 00/10] Make submodules work if .gitmodules is not checked out Antonio Ospite
@ 2018-10-25 18:49 ` Stefan Beller
  2018-10-26  1:59   ` Junio C Hamano
  0 siblings, 1 reply; 200+ results
From: Stefan Beller @ 2018-10-25 18:49 UTC (permalink / raw)
  To: Antonio Ospite
  Cc: Junio C Hamano, git, Jonathan Nieder, Jeff King, SZEDER Gábor

> In this series I am addressing the comments by Stefan Beller about the
> tests in patch 9.
>
> If the new tests look OK, I'd say we try moving the series to "next" and
> see what happens?

Sounds good to me.

Thanks,
Stefan

^ permalink raw reply	[relevance 5%]

* Re: New semantic patches vs. in-flight topics [was: Re: [PATCH 00/19] Bring more repository handles into our code base]
      [irrelevant]     ` <20181025015910.GJ30222@szeder.dev>
@ 2018-10-25 19:25       ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-25 19:25 UTC (permalink / raw)
  To: SZEDER Gábor; +Cc: Jeff King, git, Jonathan Tan

On Wed, Oct 24, 2018 at 6:59 PM SZEDER Gábor <szeder.dev@gmail.com> wrote:
>
> On Mon, Oct 22, 2018 at 11:54:06AM -0700, Stefan Beller wrote:
>
> > For the sake of a good history, I would think running 'make coccicheck'
> > and applying the resulting patches would be best as part of the (dirty)
> > merge of any topic that proposes new semantic patches, but that would
> > add load to Junio as it would be an extra step during the merge.
> >
> > One could argue that the step of applying such transformations into
> > the dirty merge is cheaper than resolving merge conflicts that are
> > had when the topic includes the transformation.
>
> Please consider that merge commits' have uglier diffs than regular
> commits, and that merge commits cause additional complications when
> 'git bisect' points the finger at them, both of which are exacerbated
> by additional changes squeezed into evil merges.
>
> > >     Consequently, 'make coccicheck' won't run clean and the
> > >     static analysis build job will fail until all those topics reach
> > >     'master', and the remaining transformations are applied on top.
> > >
> > >     This was (and still is!) an issue with the hasheq()/oideq() series
> > >     as well: that series was added on 2018-08-28, and the static
> > >     analysis build job is red on 'pu' ever since.  See the follow-up
> > >     patch e43d2dcce1 (more oideq/hasheq conversions, 2018-10-02), and
> > >     one more follow-up will be necessary after the builtin stash topic
> > >     is merged to 'master'.
> >
> > In my understanding this follow up is a feature, as it helps to avoid
> > merge conflicts with other topics in flight.
>
> I don't see how such a follow up patch helps to avoid merge conflicts.

Well, you merge first (the new topic and the cocci patches), and then
do the transformation. But that is putting a lot more work on Junio
as the way to integrate is not just merge a new topic into the pile of
topics (whether it is pu/next/master), but to first perform a merge
of the topic and the coccinelle patches, apply the transformation
and then merge to the pile, assuming the pile is already transformed
(as it happened in "treewide: apply cocci patch" in next/pu).

> > as 'make coccicheck' is an integral part of your review?
>
> Erm, right, "review" was not the right word here.  Anyway, as it is,
> 'make coccicheck' is an integral part of our automated tests, not only
> on Travis CI but on the upcoming Azure thing as well.  I just try to
> pay attention to its results and the results of a bunch of my
> additional builds, and complain or even send a fix when something goes
> reproducibly wrong.  This has certainly became more cumbersome with
> the permanently failing static analysis build job in the last couple
> of weeks.

I seem to not pay as much attention as I should to these,
mostly because they are unreliable  on the aggregate level (a failure
there most likely means another topic than the one I am interested
broke; except in this case, where we discuss the fallout there via
this topic.)

> > I like the approach of having separate classes of semantic patches:
> > (a) the regular "we need to keep checking these" as they address
> >     undesirable code patterns, which is what we currently have,
> >     and what 'make coccicheck' would complain about.
> > (b) The pending patches as you propose. However I would
> >     argue that we'd not want to include the transformation into
> >     the same patch as then the patch will have merge conflicts.
>
> Since we have a lot of parallel running topics, merge conflicts are
> basically unavoidable anyway.  If the conflicts from the
> transformation are really that severe, then perhaps the whole series
> should be postponed to a calmer, more suitable time.

Merge conflicts of this kind could be avoided, by running the
transformation on both sides before merging (or not running them
on both sides, but only after merging).

So maybe for these larger 'the_repository.pending.cocci' patches
we could get them into the tree and wait until all (most) of the topics
are including that semantic patch in their base, such that application
of the patch is easy whether before or after writing a series
(as the semantic patch is in its base).

Another short term plan would be renaming the_repository.cocci
such that it would break the 'make coccicheck'

> In the case of 'the_repository.cocci', merging its transformations
> into 'pu' resulted in only four conflicts, and I found all four on the
> easy side to resolve.  I don't think it's worth waiting with the
> transformations in this particular case.

I am not worried about the current conflicts, but about those to come
in new series.

>
> >     Ideally we'd have an automated process/bot that would apply
> >     all pending semantic patches onto master and then checks for
> >     conflicts in HEAD..pu, and only sends off the non-conflicting
> >     diffs as a topic.
>
> New semantic patches didn't pop up all that frequently in the past, so
> I'm not sure it's worth investing in such an automation.  Of course
> they can become more frequent in the future, and in that case we might
> want to reconsider it.  Unfortunately, however, Coccinelle's results
> can't be completely trusted, either because our semantic patches or
> because Coccinelle itself are buggy...

For the first, we could just be become better at reviewing the Cocci
patch. ;-) (Well, they are harder to review than usual patches, so
this doesn't surprise me. Also not fully understanding the whole
tool may have impact on reviewability)

>
> >     Then after a couple integration cycles we'd have all pending
> >     changes in, with no conflicts on Junios side.
> >
> > So I think we should add a patch like you post, but we would
> > need to discuss the exact approach how to deal with pending
> > patches. Is it the original dev who should push forward on their
> > own pending patches, or does it become a pooled effort?
>
> Well, it makes sense to me that whoever proposes a change with an
> accompanying new semantic patch should also deal with the necessary
> followups.  However, it doesn't really matter who deals with them, as
> long as somebody deals with them.  I don't think it's much different
> from e.g. sending a followup bugfix to someone else's patch series.

Ok, I plan on resending these patches after I get
origin/sb/submodule-recursive-fetch-gets-the-tip going
resent.

Stefan

^ permalink raw reply	[relevance 4%]

* Re: [PATCH 8/9] fetch: retry fetching submodules if needed objects were not fetched
      [irrelevant]       ` <20181023233756.190026-1-jonathantanmy@google.com>
@ 2018-10-25 21:42         ` Stefan Beller
  0 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-25 21:42 UTC (permalink / raw)
  To: Jonathan Tan; +Cc: Junio C Hamano, git

On Tue, Oct 23, 2018 at 4:38 PM Jonathan Tan <jonathantanmy@google.com> wrote:
>
> > > Another thing you need to clarify is what happens if the fetch-by-commit
> > > fails. Right now, it seems that it will make the whole thing fail, which
> > > might be a surprising change in behavior.
> >
> > But a positive surprise, I would assume?
>
> Whether positive or negative, I think that this needs to be mentioned in
> the commit message.
>
> As for positive or negative, I tend to agree that it's positive - sure,
> some previously successful fetches would now fail, but the results of
> those fetches could not be recursively checked out anyway.
>
> > > The test stores the result in a normal branch, not a remote tracking
> > > branch. Is storing in a normal branch required?
> >
> > In the test we fetch from another repository, such that in the
> > repository-under-test this will show up in a remote tracking branch?

I messed up there. Yes, we need to fetch into a normal branch
such that the logic of check_for_new_submodule_commits triggers
no matter where it is on the remote.

Your experiment below shows that we cannot fetch into FETCH_HEAD:

> If that were true, I would expect that when this line:
>
> > git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
>
> is replaced by this line:
>
> > git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2 &&
>
> then things would still work. The tests pass with the first line (after
> I fixed a type mismatch) but not with the second. (Also I don't think a
> remote-tracking branch is generated here - the output printed doesn't
> indicate so, and refs/changes/2 is not a branch anyway.)

> > > Also, do you know why this is required? A naive reading of the patch
> > > leads me to believe that this should work even if merely fetching to
> > > FETCH_HEAD.
> >
> > See the next patch, check_for_new_submodule_commits() is missing
> > for FETCH_HEAD.
>
> I see in the next patch that there is an "if" branch in
> store_updated_refs() without update_local_ref() in which
> "check_for_new_submodule_commits(&rm->old_oid)" needs to be inserted. I
> think this is a symptom that maybe check_for_new_submodule_commits()
> needs to be extracted from update_local_ref() and put into
> store_updated_refs() instead? In update_local_ref(), it is called on
> ref->new_oid, which is actually the same as rm->old_oid anyway (there is
> an oidcpy earlier).

I'll look into that.

> > > What is a "default" submodule and why would you need one?
> >
> > s/default/artificial/. Such a submodule is a submodule that has no
> > config in the .gitmodules file and its name == path.
> > We need to keep those around for historic reasons AFAICT, c.f.
> > c68f837576 (implement fetching of moved submodules, 2017-10-16)
>
> Ah, OK. I would call it a fake submodule then, and copy over the "No
> entry in .gitmodules?" comment.

"fake submodule" sounds like
http://debuggable.com/posts/git-fake-submodules:4b563ee4-f3cc-4061-967e-0e48cbdd56cb
which is what I think of when hearing fake submodules.

^ permalink raw reply	[relevance 8%]

* [PATCH 00/10] Resending sb/submodule-recursive-fetch-gets-the-tip
@ 2018-10-25 23:32 Stefan Beller
  2018-10-25 23:32 ` [PATCH 02/10] submodule.c: fix indentation Stefan Beller
                   ` (9 more replies)
  0 siblings, 10 replies; 200+ results
From: Stefan Beller @ 2018-10-25 23:32 UTC (permalink / raw)
  To: jonathantanmy; +Cc: git, Stefan Beller

Thanks Jonathan for your thoughtful comments,
here is the product of the discussion:
* I split up the patch to fetch in the worktree to be 2 patches,
  each giving motivation on its own.
* the last patch is vastly simplified in code, but takes an extra test
* in [1], you remark "commits" ought not to be a pointer, but I decided against
  that, as we keep the pointed-at value around for the same time span (until
  we're done with that submodule) and we don't need to copy over the pointed-at
  value into the new struct.

[1] https://public-inbox.org/git/20181018003954.139498-1-jonathantanmy@google.com/

This is still based on ao/submodule-wo-gitmodules-checked-out.
previous version
https://public-inbox.org/git/20181016181327.107186-1-sbeller@google.com/

Stefan Beller (10):
  sha1-array: provide oid_array_filter
  submodule.c: fix indentation
  submodule.c: sort changed_submodule_names before searching it
  submodule.c: tighten scope of changed_submodule_names struct
  submodule: store OIDs in changed_submodule_names
  repository: repo_submodule_init to take a submodule struct
  submodule: migrate get_next_submodule to use repository structs
  submodule.c: fetch in submodules git directory instead of in worktree
  fetch: try fetching submodules if needed objects were not fetched
  builtin/fetch: check for submodule updates in any ref update

 Documentation/technical/api-oid-array.txt    |   5 +
 builtin/fetch.c                              |  11 +-
 builtin/grep.c                               |  17 +-
 builtin/ls-files.c                           |  12 +-
 builtin/submodule--helper.c                  |   2 +-
 repository.c                                 |  27 +-
 repository.h                                 |  12 +-
 sha1-array.c                                 |  17 ++
 sha1-array.h                                 |   3 +
 submodule.c                                  | 265 ++++++++++++++++---
 t/helper/test-submodule-nested-repo-config.c |   8 +-
 t/t5526-fetch-submodules.sh                  |  55 ++++
 12 files changed, 346 insertions(+), 88 deletions(-)

  git range-diff origin/sb/submodule-recursive-fetch-gets-the-tip... 

1:  3fbb06aedd ! 1:  0fdb0e2ad9 submodule.c: move global changed_submodule_names into fetch submodule struct
    @@ -1,12 +1,11 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    submodule.c: move global changed_submodule_names into fetch submodule struct
    +    submodule.c: tighten scope of changed_submodule_names struct
     
         The `changed_submodule_names` are only used for fetching, so let's make it
         part of the struct that is passed around for fetching submodules.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/submodule.c b/submodule.c
     --- a/submodule.c
    @@ -16,7 +15,6 @@
      
      static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
     -static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
    -+
      static int initialized_fetch_ref_tips;
      static struct oid_array ref_tips_before_fetch;
      static struct oid_array ref_tips_after_fetch;
    @@ -25,22 +23,8 @@
      }
      
     -static void calculate_changed_submodule_paths(void)
    -+struct submodule_parallel_fetch {
    -+	int count;
    -+	struct argv_array args;
    -+	struct repository *r;
    -+	const char *prefix;
    -+	int command_line_option;
    -+	int default_option;
    -+	int quiet;
    -+	int result;
    -+
    -+	struct string_list changed_submodule_names;
    -+};
    -+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
    -+
     +static void calculate_changed_submodule_paths(
    -+	struct submodule_parallel_fetch *spf)
    ++	struct string_list *changed_submodule_names)
      {
      	struct argv_array argv = ARGV_ARRAY_INIT;
      	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
    @@ -49,30 +33,23 @@
      
      		if (!submodule_has_commits(path, commits))
     -			string_list_append(&changed_submodule_names, name->string);
    -+			string_list_append(&spf->changed_submodule_names,
    ++			string_list_append(changed_submodule_names,
     +					   name->string);
      	}
      
      	free_submodules_oids(&changed_submodules);
     @@
    - 	return ret;
    - }
    - 
    --struct submodule_parallel_fetch {
    --	int count;
    --	struct argv_array args;
    --	struct repository *r;
    --	const char *prefix;
    --	int command_line_option;
    --	int default_option;
    --	int quiet;
    --	int result;
    --};
    + 	int default_option;
    + 	int quiet;
    + 	int result;
    ++
    ++	struct string_list changed_submodule_names;
    + };
     -#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
    --
    ++#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
    + 
      static int get_fetch_recurse_config(const struct submodule *submodule,
      				    struct submodule_parallel_fetch *spf)
    - {
     @@
      		case RECURSE_SUBMODULES_ON_DEMAND:
      			if (!submodule ||
    @@ -88,7 +65,7 @@
      
     -	calculate_changed_submodule_paths();
     -	string_list_sort(&changed_submodule_names);
    -+	calculate_changed_submodule_paths(&spf);
    ++	calculate_changed_submodule_paths(&spf.changed_submodule_names);
     +	string_list_sort(&spf.changed_submodule_names);
      	run_processes_parallel(max_parallel_jobs,
      			       get_next_submodule,
2:  a64a8322a1 ! 2:  a11e6e39a2 submodule.c: do not copy around submodule list
    @@ -1,6 +1,6 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    submodule.c: do not copy around submodule list
    +    submodule: store OIDs in changed_submodule_names
     
         'calculate_changed_submodule_paths' uses a local list to compute the
         changed submodules, and then produces the result by copying appropriate
    @@ -14,13 +14,13 @@
         useful in a later patch.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
    +    Reviewed-by: Jonathan Tan <jonathantanmy@google.com>
     
     diff --git a/submodule.c b/submodule.c
     --- a/submodule.c
     +++ b/submodule.c
     @@
    - 	struct submodule_parallel_fetch *spf)
    + 	struct string_list *changed_submodule_names)
      {
      	struct argv_array argv = ARGV_ARRAY_INIT;
     -	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
    @@ -34,10 +34,10 @@
      	 * commits have been recorded upstream in "changed_submodule_names".
      	 */
     -	collect_changed_submodules(&changed_submodules, &argv);
    -+	collect_changed_submodules(&spf->changed_submodule_names, &argv);
    ++	collect_changed_submodules(changed_submodule_names, &argv);
      
     -	for_each_string_list_item(name, &changed_submodules) {
    -+	for_each_string_list_item(name, &spf->changed_submodule_names) {
    ++	for_each_string_list_item(name, changed_submodule_names) {
      		struct oid_array *commits = name->util;
      		const struct submodule *submodule;
      		const char *path = NULL;
    @@ -46,7 +46,7 @@
      			continue;
      
     -		if (!submodule_has_commits(path, commits))
    --			string_list_append(&spf->changed_submodule_names,
    +-			string_list_append(changed_submodule_names,
     -					   name->string);
     +		if (submodule_has_commits(path, commits)) {
     +			oid_array_clear(commits);
    @@ -55,7 +55,7 @@
      	}
      
     -	free_submodules_oids(&changed_submodules);
    -+	string_list_remove_empty_items(&spf->changed_submodule_names, 1);
    ++	string_list_remove_empty_items(changed_submodule_names, 1);
     +
      	argv_array_clear(&argv);
      	oid_array_clear(&ref_tips_before_fetch);
3:  9341b92c83 ! 3:  3f4e0d4b8d repository: repo_submodule_init to take a submodule struct
    @@ -5,17 +5,19 @@
         When constructing a struct repository for a submodule for some revision
         of the superproject where the submodule is not contained in the index,
         it may not be present in the working tree currently either. In that
    -    siutation giving a 'path' argument is not useful. Upgrade the
    +    situation giving a 'path' argument is not useful. Upgrade the
         repo_submodule_init function to take a struct submodule instead.
    +    The submodule struct can be obtained via submodule_from_{path, name} or
    +    an artificial submodule struct can be passed in.
     
    -    While we are at it, overhaul the repo_submodule_init function by renaming
    -    the submodule repository struct, which is to be initialized, to a name
    -    that is not confused with the struct submodule as easily.
    +    While we are at it, rename the repository struct in the repo_submodule_init
    +    function, which is to be initialized, to a name that is not confused with
    +    the struct submodule as easily. Perform such renames in similar functions
    +    as well.
     
         Also move its documentation into the header file.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/builtin/grep.c b/builtin/grep.c
     --- a/builtin/grep.c
    @@ -104,6 +106,19 @@
      
      static void show_ce(struct repository *repo, struct dir_struct *dir,
     
    +diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
    +--- a/builtin/submodule--helper.c
    ++++ b/builtin/submodule--helper.c
    +@@
    + 	if (!sub)
    + 		BUG("We could get the submodule handle before?");
    + 
    +-	if (repo_submodule_init(&subrepo, the_repository, path))
    ++	if (repo_submodule_init(&subrepo, the_repository, sub))
    + 		die(_("could not get a repository handle for submodule '%s'"), path);
    + 
    + 	if (!repo_config_get_string(&subrepo, "core.worktree", &cw)) {
    +
     diff --git a/repository.c b/repository.c
     --- a/repository.c
     +++ b/repository.c
    @@ -178,7 +193,8 @@
     +/*
     + * Initialize the repository 'subrepo' as the submodule given by the
     + * struct submodule 'sub' in parent repository 'superproject'.
    -+ * Return 0 upon success and a non-zero value upon failure.
    ++ * Return 0 upon success and a non-zero value upon failure, which may happen
    ++ * if the submodule is not found, or 'sub' is NULL.
     + */
     +struct submodule;
     +int repo_submodule_init(struct repository *subrepo,
4:  cea909cbd4 ! 4:  b1566069e7 submodule: fetch in submodules git directory instead of in worktree
    @@ -1,44 +1,19 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    submodule: fetch in submodules git directory instead of in worktree
    +    submodule: migrate get_next_submodule to use repository structs
     
    -    This patch started as a refactoring to make 'get_next_submodule' more
    -    readable, but upon doing so, I realized that "git fetch" of the submodule
    -    actually doesn't need to be run in the submodules worktree. So let's run
    -    it in its git dir instead.
    +    We used to recurse into submodules, even if they were broken having
    +    only an objects directory. The child process executed in the submodule
    +    would fail though if the submodule was broken.
     
    -    That should pave the way towards fetching submodules that are currently
    -    not checked out.
    -
    -    This patch leaks the cp->dir in get_next_submodule, as any further
    -    callback in run_processes_parallel doesn't have access to the child
    -    process any more. In an early iteration of this patch, the function
    -    get_submodule_repo_for directly returned the string containing the
    -    git directory, which would be a better design choice for this patch.
    -
    -    However the next patch both fixes the memory leak of cp->dir and also has
    -    a use case for using the full repository handle of the submodule, so
    -    it makes sense to introduce the get_submodule_repo_for here already.
    +    This patch tightens the check upfront, such that we do not need
    +    to spawn a child process to find out if the submodule is broken.
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/submodule.c b/submodule.c
     --- a/submodule.c
     +++ b/submodule.c
    -@@
    - 			 DEFAULT_GIT_DIR_ENVIRONMENT);
    - }
    - 
    -+static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out)
    -+{
    -+	prepare_submodule_repo_env_no_git_dir(out);
    -+	argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
    -+}
    -+
    - /* Helper function to display the submodule header line prior to the full
    -  * summary output. If it can locate the submodule objects directory it will
    -  * attempt to lookup both the left and right commits and put them into the
     @@
      	return spf->default_option;
      }
    @@ -99,9 +74,8 @@
     +		if (repo) {
      			child_process_init(cp);
     -			cp->dir = strbuf_detach(&submodule_path, NULL);
    --			prepare_submodule_repo_env(&cp->env_array);
    -+			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
    -+			cp->dir = xstrdup(repo->gitdir);
    + 			prepare_submodule_repo_env(&cp->env_array);
    ++			cp->dir = xstrdup(repo->worktree);
      			cp->git_cmd = 1;
      			if (!spf->quiet)
      				strbuf_addf(err, "Fetching submodule %s%s\n",
    @@ -113,27 +87,17 @@
     +			repo_clear(repo);
     +			free(repo);
      			ret = 1;
    ++		} else {
    ++			/*
    ++			 * An empty directory is normal,
    ++			 * the submodule is not initialized
    ++			 */
    ++			if (S_ISGITLINK(ce->ce_mode) &&
    ++			    !is_empty_dir(ce->name))
    ++				die(_("Could not access submodule '%s'"), ce->name);
      		}
     -		strbuf_release(&submodule_path);
     -		strbuf_release(&submodule_git_dir);
      		strbuf_release(&submodule_prefix);
      		if (ret) {
      			spf->count++;
    -
    -diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
    ---- a/t/t5526-fetch-submodules.sh
    -+++ b/t/t5526-fetch-submodules.sh
    -@@
    - 
    - 	test_must_fail git -C dst status &&
    - 	test_must_fail git -C dst diff &&
    --	test_must_fail git -C dst fetch --recurse-submodules
    -+
    -+	# git-fetch cannot find the git directory of the submodule,
    -+	# so it will do nothing, successfully, as it cannot distinguish between
    -+	# this broken submodule and a submodule that was just set active but
    -+	# not cloned yet
    -+	git -C dst fetch --recurse-submodules
    - '
    - 
    - test_expect_success "fetch new commits when submodule got renamed" '
-:  ---------- > 5:  2d98ff1201 submodule.c: fetch in submodules git directory instead of in worktree
5:  9ad0125310 ! 6:  092b9cbcba fetch: retry fetching submodules if needed objects were not fetched
    @@ -1,25 +1,23 @@
     Author: Stefan Beller <sbeller@google.com>
     
    -    fetch: retry fetching submodules if needed objects were not fetched
    +    fetch: try fetching submodules if needed objects were not fetched
     
         Currently when git-fetch is asked to recurse into submodules, it dispatches
         a plain "git-fetch -C <submodule-dir>" (with some submodule related options
         such as prefix and recusing strategy, but) without any information of the
         remote or the tip that should be fetched.
     
    -    This works surprisingly well in some workflows (such as using submodules
    -    as a third party library), while not so well in other scenarios, such
    -    as in a Gerrit topic-based workflow, that can tie together changes
    -    (potentially across repositories) on the server side. One of the parts
    -    of such a Gerrit workflow is to download a change when wanting to examine
    -    it, and you'd want to have its submodule changes that are in the same
    -    topic downloaded as well. However these submodule changes reside in their
    -    own repository in their own ref (refs/changes/<int>).
    +    But this default fetch is not sufficient, as a newly fetched commit in
    +    the superproject could point to a commit in the submodule that is not
    +    in the default refspec. This is common in workflows like Gerrit's.
    +    When fetching a Gerrit change under review (from refs/changes/??), the
    +    commits in that change likely point to submodule commits that have not
    +    been merged to a branch yet.
     
    -    Retry fetching a submodule by object name if the object id that the
    +    Try fetching a submodule by object id if the object id that the
         superproject points to, cannot be found.
     
    -    This retrying does not happen when the "git fetch" done at the
    +    The try does not happen when the "git fetch" done at the
         superproject is not storing the fetched results in remote
         tracking branches (i.e. instead just recording them to
         FETCH_HEAD) in this step. A later patch will fix this.
    @@ -32,7 +30,6 @@
         in case the submodule recursion is set to "on".
     
         Signed-off-by: Stefan Beller <sbeller@google.com>
    -    Signed-off-by: Junio C Hamano <gitster@pobox.com>
     
     diff --git a/builtin/fetch.c b/builtin/fetch.c
     --- a/builtin/fetch.c
    @@ -75,16 +72,16 @@
      	int result;
      
      	struct string_list changed_submodule_names;
    -+	struct get_next_submodule_task **retry;
    -+	int retry_nr, retry_alloc;
    ++	struct get_next_submodule_task **fetch_specific_oids;
    ++	int fetch_specific_oids_nr, fetch_specific_oids_alloc;
      };
     -#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
     +#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \
     +		  STRING_LIST_INIT_DUP, \
     +		  NULL, 0, 0}
      
    - static void calculate_changed_submodule_paths(
    - 	struct submodule_parallel_fetch *spf)
    + static int get_fetch_recurse_config(const struct submodule *submodule,
    + 				    struct submodule_parallel_fetch *spf)
     @@
      	return spf->default_option;
      }
    @@ -93,6 +90,8 @@
     +	struct repository *repo;
     +	const struct submodule *sub;
     +	unsigned free_sub : 1; /* Do we need to free the submodule? */
    ++
    ++	/* fetch specific oids if set, otherwise fetch default refspec */
     +	struct oid_array *commits;
     +};
     +
    @@ -224,24 +223,29 @@
     -			repo_clear(repo);
     -			free(repo);
     -			ret = 1;
    --		}
    --		strbuf_release(&submodule_prefix);
    --		if (ret) {
    - 			spf->count++;
    ++			spf->count++;
     +			*task_cb = task;
     +
     +			strbuf_release(&submodule_prefix);
    - 			return 1;
    -+		} else {
    ++			return 1;
    + 		} else {
    ++
     +			get_next_submodule_task_release(task);
     +			free(task);
    ++
    + 			/*
    + 			 * An empty directory is normal,
    + 			 * the submodule is not initialized
    +@@
    + 			    !is_empty_dir(ce->name))
    + 				die(_("Could not access submodule '%s'"), ce->name);
      		}
    - 	}
    ++	}
     +
    -+	if (spf->retry_nr) {
    -+		struct get_next_submodule_task *task = spf->retry[spf->retry_nr - 1];
    ++	if (spf->fetch_specific_oids_nr) {
    ++		struct get_next_submodule_task *task = spf->fetch_specific_oids[spf->fetch_specific_oids_nr - 1];
     +		struct strbuf submodule_prefix = STRBUF_INIT;
    -+		spf->retry_nr--;
    ++		spf->fetch_specific_oids_nr--;
     +
     +		strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, task->sub->path);
     +
    @@ -262,9 +266,13 @@
     +					  append_oid_to_argv, &cp->args);
     +
     +		*task_cb = task;
    -+		strbuf_release(&submodule_prefix);
    + 		strbuf_release(&submodule_prefix);
    +-		if (ret) {
    +-			spf->count++;
    +-			return 1;
    +-		}
     +		return 1;
    -+	}
    + 	}
     +
      	return 0;
      }
    @@ -326,9 +334,11 @@
     +			return 0;
     +
     +		task->commits = commits;
    -+		ALLOC_GROW(spf->retry, spf->retry_nr + 1, spf->retry_alloc);
    -+		spf->retry[spf->retry_nr] = task;
    -+		spf->retry_nr++;
    ++		ALLOC_GROW(spf->fetch_specific_oids,
    ++			   spf->fetch_specific_oids_nr + 1,
    ++			   spf->fetch_specific_oids_alloc);
    ++		spf->fetch_specific_oids[spf->fetch_specific_oids_nr] = task;
    ++		spf->fetch_specific_oids_nr++;
     +		return 0;
     +	}
     +
    @@ -346,18 +356,33 @@
      	test_cmp expect actual
      '
      
    -+test_expect_success "fetch new commits on-demand when they are not reachable" '
    ++test_expect_success "fetch new submodule commits on-demand outside standard refspec" '
    ++	# add a second submodule and ensure it is around in downstream first
    ++	git clone submodule sub1 &&
    ++	git submodule add ./sub1 &&
    ++	git commit -m "adding a second submodule" &&
    ++	git -C downstream pull &&
    ++	git -C downstream submodule update --init --recursive &&
    ++
     +	git checkout --detach &&
    ++
     +	C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
     +	git -C submodule update-ref refs/changes/1 $C &&
     +	git update-index --cacheinfo 160000 $C submodule &&
    -+	git commit -m "updated submodule outside of refs/heads" &&
    -+	D=$(git rev-parse HEAD) &&
    -+	git update-ref refs/changes/2 $D &&
    ++	test_tick &&
    ++
    ++	D=$(git -C sub1 commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
    ++	git -C sub1 update-ref refs/changes/2 $D &&
    ++	git update-index --cacheinfo 160000 $D sub1 &&
    ++
    ++	git commit -m "updated submodules outside of refs/heads" &&
    ++	E=$(git rev-parse HEAD) &&
    ++	git update-ref refs/changes/2 $E &&
     +	(
     +		cd downstream &&
    -+		git fetch --recurse-submodules --recurse-submodules-default on-demand origin refs/changes/2:refs/heads/my_branch &&
    ++		git fetch --recurse-submodules origin refs/changes/2:refs/heads/my_branch &&
     +		git -C submodule cat-file -t $C &&
    ++		git -C sub1 cat-file -t $D &&
     +		git checkout --recurse-submodules FETCH_HEAD
     +	)
     +'
6:  b8db3b45bf < -:  ---------- builtin/fetch: check for submodule updates for non branch fetches
7:  905a4f0d4f < -:  ---------- fixup! repository: repo_submodule_init to take a submodule struct
-:  ---------- > 7:  11bf819782 builtin/fetch: check for submodule updates in any ref update

^ permalink raw reply	[relevance 8%]

* [PATCH 03/10] submodule.c: sort changed_submodule_names before searching it
  2018-10-25 23:32 [PATCH 00/10] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
  2018-10-25 23:32 ` [PATCH 02/10] submodule.c: fix indentation Stefan Beller
@ 2018-10-25 23:32 ` Stefan Beller
  2018-10-25 23:32 ` [PATCH 04/10] submodule.c: tighten scope of changed_submodule_names struct Stefan Beller
                   ` (7 subsequent siblings)
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-25 23:32 UTC (permalink / raw)
  To: jonathantanmy; +Cc: git, Stefan Beller, Junio C Hamano

We can string_list_insert() to maintain sorted-ness of the
list as we find new items, or we can string_list_append() to
build an unsorted list and sort it at the end just once.

As we do not rely on the sortedness while building the
list, we pick the "append and sort at the end" as it
has better worst case execution times.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 submodule.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/submodule.c b/submodule.c
index e145ebbb16..9fbfcfcfe1 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1270,7 +1270,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
-			    !unsorted_string_list_lookup(
+			    !string_list_lookup(
 					&changed_submodule_names,
 					submodule->name))
 				continue;
@@ -1364,6 +1364,7 @@ int fetch_populated_submodules(struct repository *r,
 	/* default value, "--submodule-prefix" and its value are added later */
 
 	calculate_changed_submodule_paths();
+	string_list_sort(&changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* [PATCH 02/10] submodule.c: fix indentation
  2018-10-25 23:32 [PATCH 00/10] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
@ 2018-10-25 23:32 ` Stefan Beller
  2018-10-25 23:32 ` [PATCH 03/10] submodule.c: sort changed_submodule_names before searching it Stefan Beller
                   ` (8 subsequent siblings)
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-25 23:32 UTC (permalink / raw)
  To: jonathantanmy; +Cc: git, Stefan Beller, Junio C Hamano

The submodule subsystem is really bad at staying within 80 characters.
Fix it while we are here.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 submodule.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/submodule.c b/submodule.c
index 2b7082b2db..e145ebbb16 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1258,7 +1258,8 @@ static int get_next_submodule(struct child_process *cp,
 		if (!submodule) {
 			const char *name = default_name_or_path(ce->name);
 			if (name) {
-				default_submodule.path = default_submodule.name = name;
+				default_submodule.path = name;
+				default_submodule.name = name;
 				submodule = &default_submodule;
 			}
 		}
@@ -1268,8 +1269,10 @@ static int get_next_submodule(struct child_process *cp,
 		default:
 		case RECURSE_SUBMODULES_DEFAULT:
 		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names,
-							 submodule->name))
+			if (!submodule ||
+			    !unsorted_string_list_lookup(
+					&changed_submodule_names,
+					submodule->name))
 				continue;
 			default_argv = "on-demand";
 			break;
-- 
2.19.0


^ permalink raw reply	[relevance 26%]

* [PATCH 04/10] submodule.c: tighten scope of changed_submodule_names struct
  2018-10-25 23:32 [PATCH 00/10] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
  2018-10-25 23:32 ` [PATCH 02/10] submodule.c: fix indentation Stefan Beller
  2018-10-25 23:32 ` [PATCH 03/10] submodule.c: sort changed_submodule_names before searching it Stefan Beller
@ 2018-10-25 23:32 ` Stefan Beller
  2018-10-25 23:32 ` [PATCH 05/10] submodule: store OIDs in changed_submodule_names Stefan Beller
                   ` (6 subsequent siblings)
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-25 23:32 UTC (permalink / raw)
  To: jonathantanmy; +Cc: git, Stefan Beller

The `changed_submodule_names` are only used for fetching, so let's make it
part of the struct that is passed around for fetching submodules.

Signed-off-by: Stefan Beller <sbeller@google.com>
---
 submodule.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/submodule.c b/submodule.c
index 9fbfcfcfe1..6fb0b9d783 100644
--- a/submodule.c
+++ b/submodule.c
@@ -24,7 +24,6 @@
 #include "object-store.h"
 
 static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
-static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
 static int initialized_fetch_ref_tips;
 static struct oid_array ref_tips_before_fetch;
 static struct oid_array ref_tips_after_fetch;
@@ -1124,7 +1123,8 @@ void check_for_new_submodule_commits(struct object_id *oid)
 	oid_array_append(&ref_tips_after_fetch, oid);
 }
 
-static void calculate_changed_submodule_paths(void)
+static void calculate_changed_submodule_paths(
+	struct string_list *changed_submodule_names)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
 	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
@@ -1162,7 +1162,8 @@ static void calculate_changed_submodule_paths(void)
 			continue;
 
 		if (!submodule_has_commits(path, commits))
-			string_list_append(&changed_submodule_names, name->string);
+			string_list_append(changed_submodule_names,
+					   name->string);
 	}
 
 	free_submodules_oids(&changed_submodules);
@@ -1208,8 +1209,10 @@ struct submodule_parallel_fetch {
 	int default_option;
 	int quiet;
 	int result;
+
+	struct string_list changed_submodule_names;
 };
-#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, STRING_LIST_INIT_DUP }
 
 static int get_fetch_recurse_config(const struct submodule *submodule,
 				    struct submodule_parallel_fetch *spf)
@@ -1271,7 +1274,7 @@ static int get_next_submodule(struct child_process *cp,
 		case RECURSE_SUBMODULES_ON_DEMAND:
 			if (!submodule ||
 			    !string_list_lookup(
-					&changed_submodule_names,
+					&spf->changed_submodule_names,
 					submodule->name))
 				continue;
 			default_argv = "on-demand";
@@ -1363,8 +1366,8 @@ int fetch_populated_submodules(struct repository *r,
 	argv_array_push(&spf.args, "--recurse-submodules-default");
 	/* default value, "--submodule-prefix" and its value are added later */
 
-	calculate_changed_submodule_paths();
-	string_list_sort(&changed_submodule_names);
+	calculate_changed_submodule_paths(&spf.changed_submodule_names);
+	string_list_sort(&spf.changed_submodule_names);
 	run_processes_parallel(max_parallel_jobs,
 			       get_next_submodule,
 			       fetch_start_failure,
@@ -1373,7 +1376,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	argv_array_clear(&spf.args);
 out:
-	string_list_clear(&changed_submodule_names, 1);
+	string_list_clear(&spf.changed_submodule_names, 1);
 	return spf.result;
 }
 
-- 
2.19.0


^ permalink raw reply	[relevance 17%]

* [PATCH 05/10] submodule: store OIDs in changed_submodule_names
  2018-10-25 23:32 [PATCH 00/10] Resending sb/submodule-recursive-fetch-gets-the-tip Stefan Beller
                   ` (2 preceding siblings ...)
  2018-10-25 23:32 ` [PATCH 04/10] submodule.c: tighten scope of changed_submodule_names struct Stefan Beller
@ 2018-10-25 23:32 ` Stefan Beller
  2018-10-25 23:32 ` [PATCH 06/10] repository: repo_submodule_init to take a submodule struct Stefan Beller
                   ` (5 subsequent siblings)
  9 siblings, 0 replies; 200+ results
From: Stefan Beller @ 2018-10-25 23:32 UTC (permalink / raw)
  To: jonathantanmy; +Cc: git, Stefan Beller

'calculate_changed_submodule_paths' uses a local list to compute the
changed submodules, and then produces the result by copying appropriate
items into the result list.

Instead use the result list directly and prune items afterwards
using string_list_remove_empty_items.

By doing so we'll have access to the util pointer for longer that
contains the commits that we need to fetch, which will be
useful in a later patch.

Signed-off-by: Stefan Beller <sbeller@google.com>
Reviewed-by: Jonathan Tan <jonathantanmy@google.com>
---
 submodule.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/submodule.c b/submodule.c
index 6fb0b9d783..e4b494af7b 100644
--- a/submodule.c
+++ b/submodule.c
@@ -1127,8 +1127,7 @@ static void calculate_changed_submodule_paths(
 	struct string_list *changed_submodule_names)
 {
 	struct argv_array argv = ARGV_ARRAY_INIT;
-	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
-	const struct string_list_item *name;
+	struct string_list_item *name;
 
 	/* No need to check