* [PATCH v1 1/8] fetch-object: make functions return an error code
2018-12-11 5:27 [PATCH v1 0/8] Many promisor remotes Christian Couder
@ 2018-12-11 5:27 ` Christian Couder
2018-12-11 5:27 ` [PATCH v1 2/8] Add initial support for many promisor remotes Christian Couder
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Couder @ 2018-12-11 5:27 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Ben Peart, Jonathan Tan,
Jonathan Nieder, Stefan Beller, Nguyen Thai Ngoc Duy,
Mike Hommey, Lars Schneider, Eric Wong, Christian Couder,
Jeff Hostetler, Eric Sunshine, Beat Bolli, Christian Couder
From: Christian Couder <christian.couder@gmail.com>
The callers of the fetch_object() and fetch_objects() might
be interested in knowing if these functions succeeded or not.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
fetch-object.c | 13 ++++++++-----
fetch-object.h | 4 ++--
sha1-file.c | 4 ++--
3 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/fetch-object.c b/fetch-object.c
index 4266548800..eac4d448ef 100644
--- a/fetch-object.c
+++ b/fetch-object.c
@@ -5,11 +5,12 @@
#include "transport.h"
#include "fetch-object.h"
-static void fetch_refs(const char *remote_name, struct ref *ref)
+static int fetch_refs(const char *remote_name, struct ref *ref)
{
struct remote *remote;
struct transport *transport;
int original_fetch_if_missing = fetch_if_missing;
+ int res;
fetch_if_missing = 0;
remote = remote_get(remote_name);
@@ -19,12 +20,14 @@ static void fetch_refs(const char *remote_name, struct ref *ref)
transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1");
- transport_fetch_refs(transport, ref);
+ res = transport_fetch_refs(transport, ref);
fetch_if_missing = original_fetch_if_missing;
+
+ return res;
}
-void fetch_objects(const char *remote_name, const struct object_id *oids,
- int oid_nr)
+int fetch_objects(const char *remote_name, const struct object_id *oids,
+ int oid_nr)
{
struct ref *ref = NULL;
int i;
@@ -36,5 +39,5 @@ void fetch_objects(const char *remote_name, const struct object_id *oids,
new_ref->next = ref;
ref = new_ref;
}
- fetch_refs(remote_name, ref);
+ return fetch_refs(remote_name, ref);
}
diff --git a/fetch-object.h b/fetch-object.h
index d6444caa5a..7bcc7cadb0 100644
--- a/fetch-object.h
+++ b/fetch-object.h
@@ -3,7 +3,7 @@
struct object_id;
-void fetch_objects(const char *remote_name, const struct object_id *oids,
- int oid_nr);
+int fetch_objects(const char *remote_name, const struct object_id *oids,
+ int oid_nr);
#endif
diff --git a/sha1-file.c b/sha1-file.c
index 5bd11c85bc..cf88eeb147 100644
--- a/sha1-file.c
+++ b/sha1-file.c
@@ -1312,8 +1312,8 @@ int oid_object_info_extended(struct repository *r, const struct object_id *oid,
if (fetch_if_missing && repository_format_partial_clone &&
!already_retried && r == the_repository) {
/*
- * TODO Investigate having fetch_object() return
- * TODO error/success and stopping the music here.
+ * TODO Investigate checking fetch_object() return
+ * TODO value and stopping on error here.
* TODO Pass a repository struct through fetch_object,
* such that arbitrary repositories work.
*/
--
2.20.0.rc2.14.g1379de12fa.dirty
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v1 2/8] Add initial support for many promisor remotes
2018-12-11 5:27 [PATCH v1 0/8] Many promisor remotes Christian Couder
2018-12-11 5:27 ` [PATCH v1 1/8] fetch-object: make functions return an error code Christian Couder
@ 2018-12-11 5:27 ` Christian Couder
2018-12-11 5:27 ` [PATCH v1 3/8] promisor-remote: implement promisors_get_direct() Christian Couder
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Couder @ 2018-12-11 5:27 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Ben Peart, Jonathan Tan,
Jonathan Nieder, Stefan Beller, Nguyen Thai Ngoc Duy,
Mike Hommey, Lars Schneider, Eric Wong, Christian Couder,
Jeff Hostetler, Eric Sunshine, Beat Bolli, Christian Couder
From: Christian Couder <christian.couder@gmail.com>
The promisor-remote.{c,h} files will contain functions to
manage many promisor remotes.
We expect that there will not be a lot of promisor remotes,
so it is ok to use a simple linked list to manage them.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
Makefile | 1 +
promisor-remote.c | 90 +++++++++++++++++++++++++++++++++++++++++++++++
promisor-remote.h | 17 +++++++++
3 files changed, 108 insertions(+)
create mode 100644 promisor-remote.c
create mode 100644 promisor-remote.h
diff --git a/Makefile b/Makefile
index 1a44c811aa..565a5fbb4e 100644
--- a/Makefile
+++ b/Makefile
@@ -949,6 +949,7 @@ LIB_OBJS += preload-index.o
LIB_OBJS += pretty.o
LIB_OBJS += prio-queue.o
LIB_OBJS += progress.o
+LIB_OBJS += promisor-remote.o
LIB_OBJS += prompt.o
LIB_OBJS += protocol.o
LIB_OBJS += quote.o
diff --git a/promisor-remote.c b/promisor-remote.c
new file mode 100644
index 0000000000..701f5a351b
--- /dev/null
+++ b/promisor-remote.c
@@ -0,0 +1,90 @@
+#include "cache.h"
+#include "promisor-remote.h"
+#include "config.h"
+
+static struct promisor_remote *promisors;
+static struct promisor_remote **promisors_tail = &promisors;
+
+struct promisor_remote *promisor_remote_new(const char *remote_name)
+{
+ struct promisor_remote *o;
+
+ o = xcalloc(1, sizeof(*o));
+ o->remote_name = xstrdup(remote_name);
+
+ *promisors_tail = o;
+ promisors_tail = &o->next;
+
+ return o;
+}
+
+static struct promisor_remote *do_find_promisor_remote(const char *remote_name)
+{
+ struct promisor_remote *o;
+
+ for (o = promisors; o; o = o->next)
+ if (o->remote_name && !strcmp(o->remote_name, remote_name))
+ return o;
+
+ return NULL;
+}
+
+static int promisor_remote_config(const char *var, const char *value, void *data)
+{
+ struct promisor_remote *o;
+ const char *name;
+ int namelen;
+ const char *subkey;
+
+ if (parse_config_key(var, "remote", &name, &namelen, &subkey) < 0)
+ return 0;
+
+ if (!strcmp(subkey, "promisor")) {
+ char *remote_name;
+
+ if (!git_config_bool(var, value))
+ return 0;
+
+ remote_name = xmemdupz(name, namelen);
+
+ if (do_find_promisor_remote(remote_name)) {
+ free(remote_name);
+ return error(_("when parsing config key '%s' "
+ "promisor remote '%s' already exists"),
+ var, remote_name);
+ }
+
+ promisor_remote_new(remote_name);
+
+ free(remote_name);
+ return 0;
+ }
+
+ return 0;
+}
+
+static void promisor_remote_init(void)
+{
+ static int initialized;
+
+ if (initialized)
+ return;
+ initialized = 1;
+
+ git_config(promisor_remote_config, NULL);
+}
+
+struct promisor_remote *find_promisor_remote(const char *remote_name)
+{
+ promisor_remote_init();
+
+ if (!remote_name)
+ return promisors;
+
+ return do_find_promisor_remote(remote_name);
+}
+
+int has_promisor_remote(void)
+{
+ return !!find_promisor_remote(NULL);
+}
diff --git a/promisor-remote.h b/promisor-remote.h
new file mode 100644
index 0000000000..d07ac07a43
--- /dev/null
+++ b/promisor-remote.h
@@ -0,0 +1,17 @@
+#ifndef PROMISOR_REMOTE_H
+#define PROMISOR_REMOTE_H
+
+/*
+ * Promisor remote linked list
+ * Its information come from remote.XXX config entries.
+ */
+struct promisor_remote {
+ const char *remote_name;
+ struct promisor_remote *next;
+};
+
+extern struct promisor_remote *promisor_remote_new(const char *remote_name);
+extern struct promisor_remote *find_promisor_remote(const char *remote_name);
+extern int has_promisor_remote(void);
+
+#endif /* PROMISOR_REMOTE_H */
--
2.20.0.rc2.14.g1379de12fa.dirty
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v1 3/8] promisor-remote: implement promisors_get_direct()
2018-12-11 5:27 [PATCH v1 0/8] Many promisor remotes Christian Couder
2018-12-11 5:27 ` [PATCH v1 1/8] fetch-object: make functions return an error code Christian Couder
2018-12-11 5:27 ` [PATCH v1 2/8] Add initial support for many promisor remotes Christian Couder
@ 2018-12-11 5:27 ` Christian Couder
2018-12-11 5:27 ` [PATCH v1 4/8] promisor-remote: add promisor_remote_reinit() Christian Couder
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Couder @ 2018-12-11 5:27 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Ben Peart, Jonathan Tan,
Jonathan Nieder, Stefan Beller, Nguyen Thai Ngoc Duy,
Mike Hommey, Lars Schneider, Eric Wong, Christian Couder,
Jeff Hostetler, Eric Sunshine, Beat Bolli, Christian Couder
From: Christian Couder <christian.couder@gmail.com>
This is implemented for now by calling fetch_objects(). It fetches
from all the promisor remotes.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
promisor-remote.c | 33 +++++++++++++++++++++++++++++++++
promisor-remote.h | 1 +
2 files changed, 34 insertions(+)
diff --git a/promisor-remote.c b/promisor-remote.c
index 701f5a351b..e0724bdc20 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -1,6 +1,7 @@
#include "cache.h"
#include "promisor-remote.h"
#include "config.h"
+#include "fetch-object.h"
static struct promisor_remote *promisors;
static struct promisor_remote **promisors_tail = &promisors;
@@ -88,3 +89,35 @@ int has_promisor_remote(void)
{
return !!find_promisor_remote(NULL);
}
+
+static int promisor_remote_get_direct(struct promisor_remote *o,
+ const struct object_id *oids,
+ int oid_nr)
+{
+ int res;
+ uint64_t start = getnanotime();
+
+ res = fetch_objects(o->remote_name, oids, oid_nr);
+
+ trace_performance_since(start, "promisor_remote_get_direct");
+
+ return res;
+}
+
+int promisors_get_direct(const struct object_id *oids, int oid_nr)
+{
+ struct promisor_remote *o;
+
+ trace_printf("trace: promisor_remote_get_direct: nr: %d", oid_nr);
+
+ promisor_remote_init();
+
+ for (o = promisors; o; o = o->next) {
+ if (promisor_remote_get_direct(o, oids, oid_nr) < 0)
+ continue;
+ return 0;
+ }
+
+ return -1;
+}
+
diff --git a/promisor-remote.h b/promisor-remote.h
index d07ac07a43..8b89221b33 100644
--- a/promisor-remote.h
+++ b/promisor-remote.h
@@ -13,5 +13,6 @@ struct promisor_remote {
extern struct promisor_remote *promisor_remote_new(const char *remote_name);
extern struct promisor_remote *find_promisor_remote(const char *remote_name);
extern int has_promisor_remote(void);
+extern int promisors_get_direct(const struct object_id *oids, int oid_nr);
#endif /* PROMISOR_REMOTE_H */
--
2.20.0.rc2.14.g1379de12fa.dirty
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v1 4/8] promisor-remote: add promisor_remote_reinit()
2018-12-11 5:27 [PATCH v1 0/8] Many promisor remotes Christian Couder
` (2 preceding siblings ...)
2018-12-11 5:27 ` [PATCH v1 3/8] promisor-remote: implement promisors_get_direct() Christian Couder
@ 2018-12-11 5:27 ` Christian Couder
2018-12-11 5:27 ` [PATCH v1 5/8] promisor-remote: use repository_format_partial_clone Christian Couder
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Couder @ 2018-12-11 5:27 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Ben Peart, Jonathan Tan,
Jonathan Nieder, Stefan Beller, Nguyen Thai Ngoc Duy,
Mike Hommey, Lars Schneider, Eric Wong, Christian Couder,
Jeff Hostetler, Eric Sunshine, Beat Bolli, Christian Couder
From: Christian Couder <christian.couder@gmail.com>
We will need to reinitialize the promisor remote configuration
as we will make some changes to it in a later commit.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
promisor-remote.c | 14 ++++++++++++--
promisor-remote.h | 1 +
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/promisor-remote.c b/promisor-remote.c
index e0724bdc20..ca2b8bf6bb 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -64,17 +64,27 @@ static int promisor_remote_config(const char *var, const char *value, void *data
return 0;
}
-static void promisor_remote_init(void)
+static void promisor_remote_do_init(int force)
{
static int initialized;
- if (initialized)
+ if (!force && initialized)
return;
initialized = 1;
git_config(promisor_remote_config, NULL);
}
+static inline void promisor_remote_init(void)
+{
+ promisor_remote_do_init(0);
+}
+
+void promisor_remote_reinit(void)
+{
+ promisor_remote_do_init(1);
+}
+
struct promisor_remote *find_promisor_remote(const char *remote_name)
{
promisor_remote_init();
diff --git a/promisor-remote.h b/promisor-remote.h
index 8b89221b33..873ddfb5ed 100644
--- a/promisor-remote.h
+++ b/promisor-remote.h
@@ -10,6 +10,7 @@ struct promisor_remote {
struct promisor_remote *next;
};
+extern void promisor_remote_reinit(void);
extern struct promisor_remote *promisor_remote_new(const char *remote_name);
extern struct promisor_remote *find_promisor_remote(const char *remote_name);
extern int has_promisor_remote(void);
--
2.20.0.rc2.14.g1379de12fa.dirty
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v1 5/8] promisor-remote: use repository_format_partial_clone
2018-12-11 5:27 [PATCH v1 0/8] Many promisor remotes Christian Couder
` (3 preceding siblings ...)
2018-12-11 5:27 ` [PATCH v1 4/8] promisor-remote: add promisor_remote_reinit() Christian Couder
@ 2018-12-11 5:27 ` Christian Couder
2018-12-11 5:27 ` [PATCH v1 6/8] Use promisors_get_direct() and has_promisor_remote() Christian Couder
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Couder @ 2018-12-11 5:27 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Ben Peart, Jonathan Tan,
Jonathan Nieder, Stefan Beller, Nguyen Thai Ngoc Duy,
Mike Hommey, Lars Schneider, Eric Wong, Christian Couder,
Jeff Hostetler, Eric Sunshine, Beat Bolli
A remote specified using the extensions.partialClone config
option should be considered a promisor remote too.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
promisor-remote.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/promisor-remote.c b/promisor-remote.c
index ca2b8bf6bb..e4a0625426 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -73,6 +73,11 @@ static void promisor_remote_do_init(int force)
initialized = 1;
git_config(promisor_remote_config, NULL);
+
+ if (repository_format_partial_clone &&
+ !do_find_promisor_remote(repository_format_partial_clone))
+ promisor_remote_new(repository_format_partial_clone,
+ strlen(repository_format_partial_clone));
}
static inline void promisor_remote_init(void)
--
2.20.0.rc2.14.g1379de12fa.dirty
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v1 6/8] Use promisors_get_direct() and has_promisor_remote()
2018-12-11 5:27 [PATCH v1 0/8] Many promisor remotes Christian Couder
` (4 preceding siblings ...)
2018-12-11 5:27 ` [PATCH v1 5/8] promisor-remote: use repository_format_partial_clone Christian Couder
@ 2018-12-11 5:27 ` Christian Couder
2018-12-11 5:27 ` [PATCH v1 7/8] promisor-remote: parse remote.*.partialclonefilter Christian Couder
2018-12-11 5:27 ` [PATCH v1 8/8] t0410: test fetching from many promisor remotes Christian Couder
7 siblings, 0 replies; 9+ messages in thread
From: Christian Couder @ 2018-12-11 5:27 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Ben Peart, Jonathan Tan,
Jonathan Nieder, Stefan Beller, Nguyen Thai Ngoc Duy,
Mike Hommey, Lars Schneider, Eric Wong, Christian Couder,
Jeff Hostetler, Eric Sunshine, Beat Bolli
Instead of using the repository_format_partial_clone global
and fetch_objects() directly, let's use has_promisor_remote()
and promisors_get_direct().
This way all the configured promisor remotes will be taken
into account, not only the one specified by
extensions.partialClone.
Also when cloning or fetching using a partial clone filter,
remote.origin.promisor will be set to "true" instead of
setting extensions.partialClone to "origin". This makes it
possible to use many promisor remote just by fetching from
them.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
builtin/cat-file.c | 5 +++--
builtin/fetch.c | 11 ++++++-----
builtin/gc.c | 3 ++-
builtin/repack.c | 3 ++-
cache-tree.c | 3 ++-
connected.c | 3 ++-
list-objects-filter-options.c | 28 +++++++++++++++-------------
packfile.c | 3 ++-
sha1-file.c | 14 ++++++++------
t/t5601-clone.sh | 2 +-
t/t5616-partial-clone.sh | 2 +-
unpack-trees.c | 6 +++---
12 files changed, 47 insertions(+), 36 deletions(-)
diff --git a/builtin/cat-file.c b/builtin/cat-file.c
index 2ca56fd086..bdb0331ba4 100644
--- a/builtin/cat-file.c
+++ b/builtin/cat-file.c
@@ -14,6 +14,7 @@
#include "sha1-array.h"
#include "packfile.h"
#include "object-store.h"
+#include "promisor-remote.h"
struct batch_options {
int enabled;
@@ -517,8 +518,8 @@ static int batch_objects(struct batch_options *opt)
if (opt->all_objects) {
struct object_cb_data cb;
- if (repository_format_partial_clone)
- warning("This repository has extensions.partialClone set. Some objects may not be loaded.");
+ if (has_promisor_remote())
+ warning("This repository uses promisor remotes. Some objects may not be loaded.");
cb.opt = opt;
cb.expand = &data;
diff --git a/builtin/fetch.c b/builtin/fetch.c
index e0140327aa..647a60b26c 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -23,6 +23,7 @@
#include "packfile.h"
#include "list-objects-filter-options.h"
#include "commit-reach.h"
+#include "promisor-remote.h"
static const char * const builtin_fetch_usage[] = {
N_("git fetch [<options>] [<repository> [<refspec>...]]"),
@@ -1459,7 +1460,7 @@ static inline void fetch_one_setup_partial(struct remote *remote)
* If no prior partial clone/fetch and the current fetch DID NOT
* request a partial-fetch, do a normal fetch.
*/
- if (!repository_format_partial_clone && !filter_options.choice)
+ if (!has_promisor_remote() && !filter_options.choice)
return;
/*
@@ -1467,7 +1468,7 @@ static inline void fetch_one_setup_partial(struct remote *remote)
* on this repo and remember the given filter-spec as the default
* for subsequent fetches to this remote.
*/
- if (!repository_format_partial_clone && filter_options.choice) {
+ if (!has_promisor_remote() && filter_options.choice) {
partial_clone_register(remote->name, &filter_options);
return;
}
@@ -1476,7 +1477,7 @@ static inline void fetch_one_setup_partial(struct remote *remote)
* We are currently limited to only ONE promisor remote and only
* allow partial-fetches from the promisor remote.
*/
- if (strcmp(remote->name, repository_format_partial_clone)) {
+ if (!find_promisor_remote(remote->name)) {
if (filter_options.choice)
die(_("--filter can only be used with the remote configured in core.partialClone"));
return;
@@ -1607,7 +1608,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
if (depth || deepen_since || deepen_not.nr)
deepen = 1;
- if (filter_options.choice && !repository_format_partial_clone)
+ if (filter_options.choice && !has_promisor_remote())
die("--filter can only be used when extensions.partialClone is set");
if (all) {
@@ -1641,7 +1642,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
}
if (remote) {
- if (filter_options.choice || repository_format_partial_clone)
+ if (filter_options.choice || has_promisor_remote())
fetch_one_setup_partial(remote);
result = fetch_one(remote, argc, argv, prune_tags_ok);
} else {
diff --git a/builtin/gc.c b/builtin/gc.c
index 871a56f1c5..42dfa3a23c 100644
--- a/builtin/gc.c
+++ b/builtin/gc.c
@@ -27,6 +27,7 @@
#include "pack-objects.h"
#include "blob.h"
#include "tree.h"
+#include "promisor-remote.h"
#define FAILED_RUN "failed to run %s"
@@ -640,7 +641,7 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
argv_array_push(&prune, prune_expire);
if (quiet)
argv_array_push(&prune, "--no-progress");
- if (repository_format_partial_clone)
+ if (has_promisor_remote())
argv_array_push(&prune,
"--exclude-promisor-objects");
if (run_command_v_opt(prune.argv, RUN_GIT_CMD))
diff --git a/builtin/repack.c b/builtin/repack.c
index 45583683ee..26e02e3135 100644
--- a/builtin/repack.c
+++ b/builtin/repack.c
@@ -11,6 +11,7 @@
#include "midx.h"
#include "packfile.h"
#include "object-store.h"
+#include "promisor-remote.h"
static int delta_base_offset = 1;
static int pack_kept_objects = -1;
@@ -366,7 +367,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
argv_array_push(&cmd.args, "--all");
argv_array_push(&cmd.args, "--reflog");
argv_array_push(&cmd.args, "--indexed-objects");
- if (repository_format_partial_clone)
+ if (has_promisor_remote())
argv_array_push(&cmd.args, "--exclude-promisor-objects");
if (write_bitmaps)
argv_array_push(&cmd.args, "--write-bitmap-index");
diff --git a/cache-tree.c b/cache-tree.c
index 9d454d24bc..a1ecb2a1fa 100644
--- a/cache-tree.c
+++ b/cache-tree.c
@@ -5,6 +5,7 @@
#include "cache-tree.h"
#include "object-store.h"
#include "replace-object.h"
+#include "promisor-remote.h"
#ifndef DEBUG
#define DEBUG 0
@@ -357,7 +358,7 @@ static int update_one(struct cache_tree *it,
}
ce_missing_ok = mode == S_IFGITLINK || missing_ok ||
- (repository_format_partial_clone &&
+ (has_promisor_remote() &&
ce_skip_worktree(ce));
if (is_null_oid(oid) ||
(!ce_missing_ok && !has_object_file(oid))) {
diff --git a/connected.c b/connected.c
index 1bba888eff..0eaaedee6a 100644
--- a/connected.c
+++ b/connected.c
@@ -4,6 +4,7 @@
#include "connected.h"
#include "transport.h"
#include "packfile.h"
+#include "promisor-remote.h"
/*
* If we feed all the commits we want to verify to this command
@@ -56,7 +57,7 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
argv_array_push(&rev_list.args,"rev-list");
argv_array_push(&rev_list.args, "--objects");
argv_array_push(&rev_list.args, "--stdin");
- if (repository_format_partial_clone)
+ if (has_promisor_remote())
argv_array_push(&rev_list.args, "--exclude-promisor-objects");
if (!opt->is_deepening_fetch) {
argv_array_push(&rev_list.args, "--not");
diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c
index e8da2e8581..3b5ff55480 100644
--- a/list-objects-filter-options.c
+++ b/list-objects-filter-options.c
@@ -6,6 +6,7 @@
#include "list-objects.h"
#include "list-objects-filter.h"
#include "list-objects-filter-options.h"
+#include "promisor-remote.h"
/*
* Parse value of the argument to the "filter" keyword.
@@ -125,30 +126,31 @@ void partial_clone_register(
const char *remote,
const struct list_objects_filter_options *filter_options)
{
- /*
- * Record the name of the partial clone remote in the
- * config and in the global variable -- the latter is
- * used throughout to indicate that partial clone is
- * enabled and to expect missing objects.
- */
- if (repository_format_partial_clone &&
- *repository_format_partial_clone &&
- strcmp(remote, repository_format_partial_clone))
- die(_("cannot change partial clone promisor remote"));
+ char *cfg_name;
- git_config_set("core.repositoryformatversion", "1");
- git_config_set("extensions.partialclone", remote);
+ /* Check if it is already registered */
+ if (!find_promisor_remote(remote)) {
+ git_config_set("core.repositoryformatversion", "1");
- repository_format_partial_clone = xstrdup(remote);
+ /* Add promisor config for the remote */
+ cfg_name = xstrfmt("remote.%s.promisor", remote);
+ git_config_set(cfg_name, "true");
+ free(cfg_name);
+ }
/*
* Record the initial filter-spec in the config as
* the default for subsequent fetches from this remote.
+ *
+ * TODO: record it into remote.<name>.partialclonefilter
*/
core_partial_clone_filter_default =
xstrdup(filter_options->filter_spec);
git_config_set("core.partialclonefilter",
core_partial_clone_filter_default);
+
+ /* Make sure the config info are reset */
+ promisor_remote_reinit();
}
void partial_clone_get_default_filter_spec(
diff --git a/packfile.c b/packfile.c
index d1e6683ffe..7aaeea7693 100644
--- a/packfile.c
+++ b/packfile.c
@@ -16,6 +16,7 @@
#include "tree.h"
#include "object-store.h"
#include "midx.h"
+#include "promisor-remote.h"
char *odb_pack_name(struct strbuf *buf,
const unsigned char *sha1,
@@ -2113,7 +2114,7 @@ int is_promisor_object(const struct object_id *oid)
static int promisor_objects_prepared;
if (!promisor_objects_prepared) {
- if (repository_format_partial_clone) {
+ if (has_promisor_remote()) {
for_each_packed_object(add_promisor_object,
&promisor_objects,
FOR_EACH_OBJECT_PROMISOR_ONLY);
diff --git a/sha1-file.c b/sha1-file.c
index cf88eeb147..f1acaa3254 100644
--- a/sha1-file.c
+++ b/sha1-file.c
@@ -32,6 +32,7 @@
#include "packfile.h"
#include "fetch-object.h"
#include "object-store.h"
+#include "promisor-remote.h"
/* The maximum size for an object header. */
#define MAX_HEADER_LEN 32
@@ -1309,15 +1310,16 @@ int oid_object_info_extended(struct repository *r, const struct object_id *oid,
}
/* Check if it is a missing object */
- if (fetch_if_missing && repository_format_partial_clone &&
+ if (fetch_if_missing && has_promisor_remote() &&
!already_retried && r == the_repository) {
/*
- * TODO Investigate checking fetch_object() return
- * TODO value and stopping on error here.
- * TODO Pass a repository struct through fetch_object,
- * such that arbitrary repositories work.
+ * TODO Investigate checking promisors_get_direct()
+ * TODO return value and stopping on error here.
+ * TODO Pass a repository struct through
+ * promisors_get_direct(), such that arbitrary
+ * repositories work.
*/
- fetch_objects(repository_format_partial_clone, real, 1);
+ promisors_get_direct(real, 1);
already_retried = 1;
continue;
}
diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh
index 8bbc7068ac..aafe55b8b7 100755
--- a/t/t5601-clone.sh
+++ b/t/t5601-clone.sh
@@ -653,7 +653,7 @@ partial_clone () {
git -C client fsck &&
# Ensure that unneeded blobs are not inadvertently fetched.
- test_config -C client extensions.partialclone "not a remote" &&
+ test_config -C client remote.origin.promisor "false" &&
test_must_fail git -C client cat-file -e "$HASH1" &&
# But this blob was fetched, because clone performs an initial checkout
diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh
index 336f02a41a..d09dee1f14 100755
--- a/t/t5616-partial-clone.sh
+++ b/t/t5616-partial-clone.sh
@@ -42,7 +42,7 @@ test_expect_success 'do partial clone 1' '
test_cmp expect_1.oids observed.oids &&
test "$(git -C pc1 config --local core.repositoryformatversion)" = "1" &&
- test "$(git -C pc1 config --local extensions.partialclone)" = "origin" &&
+ test "$(git -C pc1 config --local remote.origin.promisor)" = "true" &&
test "$(git -C pc1 config --local core.partialclonefilter)" = "blob:none"
'
diff --git a/unpack-trees.c b/unpack-trees.c
index 7570df481b..9ad157eeab 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -18,6 +18,7 @@
#include "fsmonitor.h"
#include "object-store.h"
#include "fetch-object.h"
+#include "promisor-remote.h"
/*
* Error messages expected by scripts out of plumbing commands such as
@@ -418,7 +419,7 @@ static int check_updates(struct unpack_trees_options *o)
load_gitmodules_file(index, &state);
enable_delayed_checkout(&state);
- if (repository_format_partial_clone && o->update && !o->dry_run) {
+ if (has_promisor_remote() && o->update && !o->dry_run) {
/*
* Prefetch the objects that are to be checked out in the loop
* below.
@@ -435,8 +436,7 @@ static int check_updates(struct unpack_trees_options *o)
}
}
if (to_fetch.nr)
- fetch_objects(repository_format_partial_clone,
- to_fetch.oid, to_fetch.nr);
+ promisors_get_direct(to_fetch.oid, to_fetch.nr);
fetch_if_missing = fetch_if_missing_store;
oid_array_clear(&to_fetch);
}
--
2.20.0.rc2.14.g1379de12fa.dirty
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v1 7/8] promisor-remote: parse remote.*.partialclonefilter
2018-12-11 5:27 [PATCH v1 0/8] Many promisor remotes Christian Couder
` (5 preceding siblings ...)
2018-12-11 5:27 ` [PATCH v1 6/8] Use promisors_get_direct() and has_promisor_remote() Christian Couder
@ 2018-12-11 5:27 ` Christian Couder
2018-12-11 5:27 ` [PATCH v1 8/8] t0410: test fetching from many promisor remotes Christian Couder
7 siblings, 0 replies; 9+ messages in thread
From: Christian Couder @ 2018-12-11 5:27 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Ben Peart, Jonathan Tan,
Jonathan Nieder, Stefan Beller, Nguyen Thai Ngoc Duy,
Mike Hommey, Lars Schneider, Eric Wong, Christian Couder,
Jeff Hostetler, Eric Sunshine, Beat Bolli
This makes it possible to specify a different partial clone
filter for each promisor remote.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
builtin/fetch.c | 2 +-
list-objects-filter-options.c | 27 +++++++++++++++------------
list-objects-filter-options.h | 3 ++-
promisor-remote.c | 13 +++++++++++--
promisor-remote.h | 5 ++++-
t/t0410-partial-clone.sh | 2 +-
t/t5601-clone.sh | 1 +
t/t5616-partial-clone.sh | 2 +-
8 files changed, 36 insertions(+), 19 deletions(-)
diff --git a/builtin/fetch.c b/builtin/fetch.c
index 647a60b26c..53494c5864 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -1489,7 +1489,7 @@ static inline void fetch_one_setup_partial(struct remote *remote)
* the config.
*/
if (!filter_options.choice)
- partial_clone_get_default_filter_spec(&filter_options);
+ partial_clone_get_default_filter_spec(&filter_options, remote->name);
return;
}
diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c
index 3b5ff55480..ea0148f709 100644
--- a/list-objects-filter-options.c
+++ b/list-objects-filter-options.c
@@ -29,6 +29,9 @@ static int gently_parse_list_objects_filter(
{
const char *v0;
+ if (!arg)
+ return 0;
+
if (filter_options->choice) {
if (errbuf) {
strbuf_addstr(
@@ -127,6 +130,7 @@ void partial_clone_register(
const struct list_objects_filter_options *filter_options)
{
char *cfg_name;
+ char *filter_name;
/* Check if it is already registered */
if (!find_promisor_remote(remote)) {
@@ -141,27 +145,26 @@ void partial_clone_register(
/*
* Record the initial filter-spec in the config as
* the default for subsequent fetches from this remote.
- *
- * TODO: record it into remote.<name>.partialclonefilter
*/
- core_partial_clone_filter_default =
- xstrdup(filter_options->filter_spec);
- git_config_set("core.partialclonefilter",
- core_partial_clone_filter_default);
+ filter_name = xstrfmt("remote.%s.partialclonefilter", remote);
+ git_config_set(filter_name, filter_options->filter_spec);
+ free(filter_name);
/* Make sure the config info are reset */
promisor_remote_reinit();
}
void partial_clone_get_default_filter_spec(
- struct list_objects_filter_options *filter_options)
+ struct list_objects_filter_options *filter_options,
+ const char *remote)
{
+ struct promisor_remote *promisor = find_promisor_remote(remote);
+
/*
* Parse default value, but silently ignore it if it is invalid.
*/
- if (!core_partial_clone_filter_default)
- return;
- gently_parse_list_objects_filter(filter_options,
- core_partial_clone_filter_default,
- NULL);
+ if (promisor)
+ gently_parse_list_objects_filter(filter_options,
+ promisor->partial_clone_filter,
+ NULL);
}
diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h
index af64e5c66f..8c10ec2d3a 100644
--- a/list-objects-filter-options.h
+++ b/list-objects-filter-options.h
@@ -75,6 +75,7 @@ void partial_clone_register(
const char *remote,
const struct list_objects_filter_options *filter_options);
void partial_clone_get_default_filter_spec(
- struct list_objects_filter_options *filter_options);
+ struct list_objects_filter_options *filter_options,
+ const char *remote);
#endif /* LIST_OBJECTS_FILTER_OPTIONS_H */
diff --git a/promisor-remote.c b/promisor-remote.c
index e4a0625426..d1bd9fbf49 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -60,6 +60,16 @@ static int promisor_remote_config(const char *var, const char *value, void *data
free(remote_name);
return 0;
}
+ if (!strcmp(subkey, "partialclonefilter")) {
+ char *remote_name = xmemdupz(name, namelen);
+
+ o = do_find_promisor_remote(remote_name);
+ if (!o)
+ o = promisor_remote_new(remote_name);
+
+ free(remote_name);
+ return git_config_string(&o->partial_clone_filter, var, value);
+ }
return 0;
}
@@ -76,8 +86,7 @@ static void promisor_remote_do_init(int force)
if (repository_format_partial_clone &&
!do_find_promisor_remote(repository_format_partial_clone))
- promisor_remote_new(repository_format_partial_clone,
- strlen(repository_format_partial_clone));
+ promisor_remote_new(repository_format_partial_clone);
}
static inline void promisor_remote_init(void)
diff --git a/promisor-remote.h b/promisor-remote.h
index 873ddfb5ed..431edafcdd 100644
--- a/promisor-remote.h
+++ b/promisor-remote.h
@@ -3,10 +3,13 @@
/*
* Promisor remote linked list
- * Its information come from remote.XXX config entries.
+ *
+ * Information in its fields come from remote.XXX config entries or
+ * from extensions.partialclone or core.partialclonefilter.
*/
struct promisor_remote {
const char *remote_name;
+ const char *partial_clone_filter;
struct promisor_remote *next;
};
diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh
index ba3887f178..462229e445 100755
--- a/t/t0410-partial-clone.sh
+++ b/t/t0410-partial-clone.sh
@@ -26,7 +26,7 @@ promise_and_delete () {
test_expect_success 'extensions.partialclone without filter' '
test_create_repo server &&
git clone --filter="blob:none" "file://$(pwd)/server" client &&
- git -C client config --unset core.partialclonefilter &&
+ git -C client config --unset remote.origin.partialclonefilter &&
git -C client fetch origin
'
diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh
index aafe55b8b7..72a04669e4 100755
--- a/t/t5601-clone.sh
+++ b/t/t5601-clone.sh
@@ -654,6 +654,7 @@ partial_clone () {
# Ensure that unneeded blobs are not inadvertently fetched.
test_config -C client remote.origin.promisor "false" &&
+ git -C client config --unset remote.origin.partialclonefilter &&
test_must_fail git -C client cat-file -e "$HASH1" &&
# But this blob was fetched, because clone performs an initial checkout
diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh
index d09dee1f14..9057ad25d3 100755
--- a/t/t5616-partial-clone.sh
+++ b/t/t5616-partial-clone.sh
@@ -43,7 +43,7 @@ test_expect_success 'do partial clone 1' '
test_cmp expect_1.oids observed.oids &&
test "$(git -C pc1 config --local core.repositoryformatversion)" = "1" &&
test "$(git -C pc1 config --local remote.origin.promisor)" = "true" &&
- test "$(git -C pc1 config --local core.partialclonefilter)" = "blob:none"
+ test "$(git -C pc1 config --local remote.origin.partialclonefilter)" = "blob:none"
'
# checkout master to force dynamic object fetch of blobs at HEAD.
--
2.20.0.rc2.14.g1379de12fa.dirty
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v1 8/8] t0410: test fetching from many promisor remotes
2018-12-11 5:27 [PATCH v1 0/8] Many promisor remotes Christian Couder
` (6 preceding siblings ...)
2018-12-11 5:27 ` [PATCH v1 7/8] promisor-remote: parse remote.*.partialclonefilter Christian Couder
@ 2018-12-11 5:27 ` Christian Couder
7 siblings, 0 replies; 9+ messages in thread
From: Christian Couder @ 2018-12-11 5:27 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Ben Peart, Jonathan Tan,
Jonathan Nieder, Stefan Beller, Nguyen Thai Ngoc Duy,
Mike Hommey, Lars Schneider, Eric Wong, Christian Couder,
Jeff Hostetler, Eric Sunshine, Beat Bolli, Christian Couder
From: Christian Couder <christian.couder@gmail.com>
This shows that it is now possible to fetch objects from more
than one promisor remote.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0410-partial-clone.sh | 24 +++++++++++++++++++++++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh
index 462229e445..a09a53ab0d 100755
--- a/t/t0410-partial-clone.sh
+++ b/t/t0410-partial-clone.sh
@@ -182,8 +182,30 @@ test_expect_success 'fetching of missing objects works with ref-in-want enabled'
grep "git< fetch=.*ref-in-want" trace
'
+test_expect_success 'fetching of missing objects from another promisor remote' '
+ git clone "file://$(pwd)/server" server2 &&
+ test_commit -C server2 bar &&
+ git -C server2 repack -a -d --write-bitmap-index &&
+ HASH2=$(git -C server2 rev-parse bar) &&
+
+ git -C repo remote add server2 "file://$(pwd)/server2" &&
+ git -C repo config remote.server2.promisor true &&
+ git -C repo cat-file -p "$HASH2" &&
+
+ git -C repo fetch server2 &&
+ rm -rf repo/.git/objects/* &&
+ git -C repo cat-file -p "$HASH2" &&
+
+ # Ensure that the .promisor file is written, and check that its
+ # associated packfile contains the object
+ ls repo/.git/objects/pack/pack-*.promisor >promisorlist &&
+ test_line_count = 1 promisorlist &&
+ IDX=$(cat promisorlist | sed "s/promisor$/idx/") &&
+ git verify-pack --verbose "$IDX" | grep "$HASH2"
+'
+
test_expect_success 'fetching of missing blobs works' '
- rm -rf server repo &&
+ rm -rf server server2 repo &&
test_create_repo server &&
test_commit -C server foo &&
git -C server repack -a -d --write-bitmap-index &&
--
2.20.0.rc2.14.g1379de12fa.dirty
^ permalink raw reply related [flat|nested] 9+ messages in thread