* [RFC/PATCH v2 01/10] Add initial external odb support
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 02/10] external odb foreach Christian Couder
` (9 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
From: Jeff King <peff@peff.net>
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
Makefile | 2 +
cache.h | 9 ++
external-odb.c | 115 +++++++++++++++++++++++
external-odb.h | 8 ++
odb-helper.c | 239 ++++++++++++++++++++++++++++++++++++++++++++++++
odb-helper.h | 25 +++++
sha1_file.c | 64 ++++++++++---
t/t0400-external-odb.sh | 48 ++++++++++
8 files changed, 495 insertions(+), 15 deletions(-)
create mode 100644 external-odb.c
create mode 100644 external-odb.h
create mode 100644 odb-helper.c
create mode 100644 odb-helper.h
create mode 100755 t/t0400-external-odb.sh
diff --git a/Makefile b/Makefile
index de5a030..8641318 100644
--- a/Makefile
+++ b/Makefile
@@ -726,6 +726,7 @@ LIB_OBJS += ewah/ewah_bitmap.o
LIB_OBJS += ewah/ewah_io.o
LIB_OBJS += ewah/ewah_rlw.o
LIB_OBJS += exec_cmd.o
+LIB_OBJS += external-odb.o
LIB_OBJS += fetch-pack.o
LIB_OBJS += fsck.o
LIB_OBJS += gettext.o
@@ -757,6 +758,7 @@ LIB_OBJS += notes-cache.o
LIB_OBJS += notes-merge.o
LIB_OBJS += notes-utils.o
LIB_OBJS += object.o
+LIB_OBJS += odb-helper.o
LIB_OBJS += pack-bitmap.o
LIB_OBJS += pack-bitmap-write.o
LIB_OBJS += pack-check.o
diff --git a/cache.h b/cache.h
index 6049f86..cc0a934 100644
--- a/cache.h
+++ b/cache.h
@@ -864,6 +864,12 @@ const char *git_path_shallow(void);
*/
extern const char *sha1_file_name(const unsigned char *sha1);
+/*
+ * Like sha1_file_name, but return the filename within a specific alternate
+ * object directory. Shares the same static buffer with sha1_file_name.
+ */
+extern const char *sha1_file_name_alt(const char *objdir, const unsigned char *sha1);
+
/*
* Return the name of the (local) packfile with the specified sha1 in
* its name. The return value is a pointer to memory that is
@@ -1093,6 +1099,8 @@ extern int do_check_packed_object_crc;
extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type);
+extern int create_object_tmpfile(struct strbuf *tmp, const char *filename);
+extern void close_sha1_file(int fd);
extern int finalize_object_file(const char *tmpfile, const char *filename);
extern int has_sha1_pack(const unsigned char *sha1);
@@ -1338,6 +1346,7 @@ extern void read_info_alternates(const char * relative_base, int depth);
extern void add_to_alternates_file(const char *reference);
typedef int alt_odb_fn(struct alternate_object_database *, void *);
extern int foreach_alt_odb(alt_odb_fn, void*);
+extern void prepare_external_alt_odb(void);
struct pack_window {
struct pack_window *next;
diff --git a/external-odb.c b/external-odb.c
new file mode 100644
index 0000000..1ccfa99
--- /dev/null
+++ b/external-odb.c
@@ -0,0 +1,115 @@
+#include "cache.h"
+#include "external-odb.h"
+#include "odb-helper.h"
+
+static struct odb_helper *helpers;
+static struct odb_helper **helpers_tail = &helpers;
+
+static struct odb_helper *find_or_create_helper(const char *name, int len)
+{
+ struct odb_helper *o;
+
+ for (o = helpers; o; o = o->next)
+ if (!strncmp(o->name, name, len) && !o->name[len])
+ return o;
+
+ o = odb_helper_new(name, len);
+ *helpers_tail = o;
+ helpers_tail = &o->next;
+
+ return o;
+}
+
+static int external_odb_config(const char *var, const char *value, void *data)
+{
+ struct odb_helper *o;
+ const char *key, *dot;
+
+ if (!skip_prefix(var, "odb.", &key))
+ return 0;
+ dot = strrchr(key, '.');
+ if (!dot)
+ return 0;
+
+ o = find_or_create_helper(key, dot - key);
+ key = dot + 1;
+
+ if (!strcmp(key, "command"))
+ return git_config_string(&o->cmd, var, value);
+
+ return 0;
+}
+
+static void external_odb_init(void)
+{
+ static int initialized;
+
+ if (initialized)
+ return;
+ initialized = 1;
+
+ git_config(external_odb_config, NULL);
+}
+
+const char *external_odb_root(void)
+{
+ static const char *root;
+ if (!root)
+ root = git_pathdup("objects/external");
+ return root;
+}
+
+int external_odb_has_object(const unsigned char *sha1)
+{
+ struct odb_helper *o;
+
+ external_odb_init();
+
+ for (o = helpers; o; o = o->next)
+ if (odb_helper_has_object(o, sha1))
+ return 1;
+ return 0;
+}
+
+int external_odb_fetch_object(const unsigned char *sha1)
+{
+ struct odb_helper *o;
+ const char *path;
+
+ if (!external_odb_has_object(sha1))
+ return -1;
+
+ path = sha1_file_name_alt(external_odb_root(), sha1);
+ safe_create_leading_directories_const(path);
+ prepare_external_alt_odb();
+
+ for (o = helpers; o; o = o->next) {
+ struct strbuf tmpfile = STRBUF_INIT;
+ int ret;
+ int fd;
+
+ if (!odb_helper_has_object(o, sha1))
+ continue;
+
+ fd = create_object_tmpfile(&tmpfile, path);
+ if (fd < 0) {
+ strbuf_release(&tmpfile);
+ return -1;
+ }
+
+ if (odb_helper_fetch_object(o, sha1, fd) < 0) {
+ close(fd);
+ unlink(tmpfile.buf);
+ strbuf_release(&tmpfile);
+ continue;
+ }
+
+ close_sha1_file(fd);
+ ret = finalize_object_file(tmpfile.buf, path);
+ strbuf_release(&tmpfile);
+ if (!ret)
+ return 0;
+ }
+
+ return -1;
+}
diff --git a/external-odb.h b/external-odb.h
new file mode 100644
index 0000000..2397477
--- /dev/null
+++ b/external-odb.h
@@ -0,0 +1,8 @@
+#ifndef EXTERNAL_ODB_H
+#define EXTERNAL_ODB_H
+
+const char *external_odb_root(void);
+int external_odb_has_object(const unsigned char *sha1);
+int external_odb_fetch_object(const unsigned char *sha1);
+
+#endif /* EXTERNAL_ODB_H */
diff --git a/odb-helper.c b/odb-helper.c
new file mode 100644
index 0000000..7029a59
--- /dev/null
+++ b/odb-helper.c
@@ -0,0 +1,239 @@
+#include "cache.h"
+#include "object.h"
+#include "argv-array.h"
+#include "odb-helper.h"
+#include "run-command.h"
+#include "sha1-lookup.h"
+
+struct odb_helper *odb_helper_new(const char *name, int namelen)
+{
+ struct odb_helper *o;
+
+ o = xcalloc(1, sizeof(*o));
+ o->name = xmemdupz(name, namelen);
+
+ return o;
+}
+
+struct odb_helper_cmd {
+ struct argv_array argv;
+ struct child_process child;
+};
+
+static void prepare_helper_command(struct argv_array *argv, const char *cmd,
+ const char *fmt, va_list ap)
+{
+ struct strbuf buf = STRBUF_INIT;
+
+ strbuf_addstr(&buf, cmd);
+ strbuf_addch(&buf, ' ');
+ strbuf_vaddf(&buf, fmt, ap);
+
+ argv_array_push(argv, buf.buf);
+ strbuf_release(&buf);
+}
+
+__attribute__((format (printf,3,4)))
+static int odb_helper_start(struct odb_helper *o,
+ struct odb_helper_cmd *cmd,
+ const char *fmt, ...)
+{
+ va_list ap;
+
+ memset(cmd, 0, sizeof(*cmd));
+ argv_array_init(&cmd->argv);
+
+ if (!o->cmd)
+ return -1;
+
+ va_start(ap, fmt);
+ prepare_helper_command(&cmd->argv, o->cmd, fmt, ap);
+ va_end(ap);
+
+ cmd->child.argv = cmd->argv.argv;
+ cmd->child.use_shell = 1;
+ cmd->child.no_stdin = 1;
+ cmd->child.out = -1;
+
+ if (start_command(&cmd->child) < 0) {
+ argv_array_clear(&cmd->argv);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int odb_helper_finish(struct odb_helper *o,
+ struct odb_helper_cmd *cmd)
+{
+ int ret = finish_command(&cmd->child);
+ argv_array_clear(&cmd->argv);
+ if (ret) {
+ warning("odb helper '%s' reported failure", o->name);
+ return -1;
+ }
+ return 0;
+}
+
+static int parse_object_line(struct odb_helper_object *o, const char *line)
+{
+ char *end;
+ if (get_sha1_hex(line, o->sha1) < 0)
+ return -1;
+
+ line += 40;
+ if (*line++ != ' ')
+ return -1;
+
+ o->size = strtoul(line, &end, 10);
+ if (line == end || *end++ != ' ')
+ return -1;
+
+ o->type = type_from_string(end);
+ return 0;
+}
+
+static int odb_helper_object_cmp(const void *va, const void *vb)
+{
+ const struct odb_helper_object *a = va, *b = vb;
+ return hashcmp(a->sha1, b->sha1);
+}
+
+static void odb_helper_load_have(struct odb_helper *o)
+{
+ struct odb_helper_cmd cmd;
+ FILE *fh;
+ struct strbuf line = STRBUF_INIT;
+
+ if (o->have_valid)
+ return;
+ o->have_valid = 1;
+
+ if (odb_helper_start(o, &cmd, "have") < 0)
+ return;
+
+ fh = xfdopen(cmd.child.out, "r");
+ while (strbuf_getline(&line, fh) != EOF) {
+ ALLOC_GROW(o->have, o->have_nr+1, o->have_alloc);
+ if (parse_object_line(&o->have[o->have_nr], line.buf) < 0) {
+ warning("bad 'have' input from odb helper '%s': %s",
+ o->name, line.buf);
+ break;
+ }
+ o->have_nr++;
+ }
+
+ strbuf_release(&line);
+ fclose(fh);
+ odb_helper_finish(o, &cmd);
+
+ qsort(o->have, o->have_nr, sizeof(*o->have), odb_helper_object_cmp);
+}
+
+static struct odb_helper_object *odb_helper_lookup(struct odb_helper *o,
+ const unsigned char *sha1)
+{
+ int idx;
+
+ odb_helper_load_have(o);
+ idx = sha1_entry_pos(o->have, sizeof(*o->have), 0,
+ 0, o->have_nr, o->have_nr,
+ sha1);
+ if (idx < 0)
+ return NULL;
+ return &o->have[idx];
+}
+
+int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1)
+{
+ return !!odb_helper_lookup(o, sha1);
+}
+
+int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
+ int fd)
+{
+ struct odb_helper_object *obj;
+ struct odb_helper_cmd cmd;
+ unsigned long total_got;
+ git_zstream stream;
+ int zret;
+ git_SHA_CTX hash;
+ unsigned char real_sha1[20];
+
+ obj = odb_helper_lookup(o, sha1);
+ if (!obj)
+ return -1;
+
+ if (odb_helper_start(o, &cmd, "get %s", sha1_to_hex(sha1)) < 0)
+ return -1;
+
+ memset(&stream, 0, sizeof(stream));
+ git_inflate_init(&stream);
+ git_SHA1_Init(&hash);
+ total_got = 0;
+
+ for (;;) {
+ unsigned char buf[4096];
+ int r;
+
+ r = xread(cmd.child.out, buf, sizeof(buf));
+ if (r < 0) {
+ error("unable to read from odb helper '%s': %s",
+ o->name, strerror(errno));
+ close(cmd.child.out);
+ odb_helper_finish(o, &cmd);
+ git_inflate_end(&stream);
+ return -1;
+ }
+ if (r == 0)
+ break;
+
+ write_or_die(fd, buf, r);
+
+ stream.next_in = buf;
+ stream.avail_in = r;
+ do {
+ unsigned char inflated[4096];
+ unsigned long got;
+
+ stream.next_out = inflated;
+ stream.avail_out = sizeof(inflated);
+ zret = git_inflate(&stream, Z_SYNC_FLUSH);
+ got = sizeof(inflated) - stream.avail_out;
+
+ git_SHA1_Update(&hash, inflated, got);
+ /* skip header when counting size */
+ if (!total_got) {
+ const unsigned char *p = memchr(inflated, '\0', got);
+ if (p)
+ got -= p - inflated + 1;
+ else
+ got = 0;
+ }
+ total_got += got;
+ } while (stream.avail_in && zret == Z_OK);
+ }
+
+ close(cmd.child.out);
+ git_inflate_end(&stream);
+ git_SHA1_Final(real_sha1, &hash);
+ if (odb_helper_finish(o, &cmd))
+ return -1;
+ if (zret != Z_STREAM_END) {
+ warning("bad zlib data from odb helper '%s' for %s",
+ o->name, sha1_to_hex(sha1));
+ return -1;
+ }
+ if (total_got != obj->size) {
+ warning("size mismatch from odb helper '%s' for %s (%lu != %lu)",
+ o->name, sha1_to_hex(sha1), total_got, obj->size);
+ return -1;
+ }
+ if (hashcmp(real_sha1, sha1)) {
+ warning("sha1 mismatch from odb helper '%s' for %s (got %s)",
+ o->name, sha1_to_hex(sha1), sha1_to_hex(real_sha1));
+ return -1;
+ }
+
+ return 0;
+}
diff --git a/odb-helper.h b/odb-helper.h
new file mode 100644
index 0000000..0f704f9
--- /dev/null
+++ b/odb-helper.h
@@ -0,0 +1,25 @@
+#ifndef ODB_HELPER_H
+#define ODB_HELPER_H
+
+struct odb_helper {
+ const char *name;
+ const char *cmd;
+
+ struct odb_helper_object {
+ unsigned char sha1[20];
+ unsigned long size;
+ enum object_type type;
+ } *have;
+ int have_nr;
+ int have_alloc;
+ int have_valid;
+
+ struct odb_helper *next;
+};
+
+struct odb_helper *odb_helper_new(const char *name, int namelen);
+int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1);
+int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
+ int fd);
+
+#endif /* ODB_HELPER_H */
diff --git a/sha1_file.c b/sha1_file.c
index d5e1121..a707bc1 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -23,6 +23,7 @@
#include "bulk-checkin.h"
#include "streaming.h"
#include "dir.h"
+#include "external-odb.h"
#ifndef O_NOATIME
#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
@@ -183,13 +184,11 @@ static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
}
}
-const char *sha1_file_name(const unsigned char *sha1)
+const char *sha1_file_name_alt(const char *objdir, const unsigned char *sha1)
{
static char buf[PATH_MAX];
- const char *objdir;
int len;
- objdir = get_object_directory();
len = strlen(objdir);
/* '/' + sha1(2) + '/' + sha1(38) + '\0' */
@@ -203,6 +202,11 @@ const char *sha1_file_name(const unsigned char *sha1)
return buf;
}
+const char *sha1_file_name(const unsigned char *sha1)
+{
+ return sha1_file_name_alt(get_object_directory(), sha1);
+}
+
/*
* Return the name of the pack or index file with the specified sha1
* in its filename. *base and *name are scratch space that must be
@@ -439,6 +443,21 @@ int foreach_alt_odb(alt_odb_fn fn, void *cb)
return r;
}
+void prepare_external_alt_odb(void)
+{
+ static int linked_external;
+ const char *path;
+
+ if (linked_external)
+ return;
+
+ path = external_odb_root();
+ if (!access(path, F_OK)) {
+ link_alt_odb_entry(path, NULL, 0, "");
+ linked_external = 1;
+ }
+}
+
void prepare_alt_odb(void)
{
const char *alt;
@@ -453,6 +472,7 @@ void prepare_alt_odb(void)
link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0);
read_info_alternates(get_object_directory(), 0);
+ prepare_external_alt_odb();
}
/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
@@ -493,7 +513,7 @@ static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
if (check_and_freshen_file(alt->base, freshen))
return 1;
}
- return 0;
+ return external_odb_has_object(sha1);
}
static int check_and_freshen(const unsigned char *sha1, int freshen)
@@ -1523,19 +1543,15 @@ static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
return -1;
}
-static int open_sha1_file(const unsigned char *sha1)
+static int open_sha1_file_alt(const unsigned char *sha1)
{
- int fd;
struct alternate_object_database *alt;
- int most_interesting_errno;
-
- fd = git_open_noatime(sha1_file_name(sha1));
- if (fd >= 0)
- return fd;
- most_interesting_errno = errno;
+ int most_interesting_errno = errno;
prepare_alt_odb();
for (alt = alt_odb_list; alt; alt = alt->next) {
+ int fd;
+
fill_sha1_path(alt->name, sha1);
fd = git_open_noatime(alt->base);
if (fd >= 0)
@@ -1547,6 +1563,24 @@ static int open_sha1_file(const unsigned char *sha1)
return -1;
}
+static int open_sha1_file(const unsigned char *sha1)
+{
+ int fd;
+
+ fd = git_open_noatime(sha1_file_name(sha1));
+ if (fd >= 0)
+ return fd;
+
+ fd = open_sha1_file_alt(sha1);
+ if (fd >= 0)
+ return fd;
+
+ if (!external_odb_fetch_object(sha1))
+ fd = open_sha1_file_alt(sha1);
+
+ return fd;
+}
+
void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
{
void *map;
@@ -3011,7 +3045,7 @@ int hash_sha1_file(const void *buf, unsigned long len, const char *type,
}
/* Finalize a file on disk, and close it. */
-static void close_sha1_file(int fd)
+void close_sha1_file(int fd)
{
if (fsync_object_files)
fsync_or_die(fd, "sha1 file");
@@ -3035,7 +3069,7 @@ static inline int directory_size(const char *filename)
* We want to avoid cross-directory filename renames, because those
* can have problems on various filesystems (FAT, NFS, Coda).
*/
-static int create_tmpfile(struct strbuf *tmp, const char *filename)
+int create_object_tmpfile(struct strbuf *tmp, const char *filename)
{
int fd, dirlen = directory_size(filename);
@@ -3075,7 +3109,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
static struct strbuf tmp_file = STRBUF_INIT;
const char *filename = sha1_file_name(sha1);
- fd = create_tmpfile(&tmp_file, filename);
+ fd = create_object_tmpfile(&tmp_file, filename);
if (fd < 0) {
if (errno == EACCES)
return error("insufficient permission for adding an object to repository database %s", get_object_directory());
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
new file mode 100755
index 0000000..2b01617
--- /dev/null
+++ b/t/t0400-external-odb.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+test_description='basic tests for external object databases'
+
+. ./test-lib.sh
+
+ALT_SOURCE="$PWD/alt-repo/.git"
+export ALT_SOURCE
+write_script odb-helper <<\EOF
+GIT_DIR=$ALT_SOURCE; export GIT_DIR
+case "$1" in
+have)
+ git rev-list --all --objects |
+ cut -d' ' -f1 |
+ git cat-file --batch-check |
+ awk '{print $1 " " $3 " " $2}'
+ ;;
+get)
+ cat "$GIT_DIR"/objects/$(echo $2 | sed 's#..#&/#')
+ ;;
+esac
+EOF
+HELPER="\"$PWD\"/odb-helper"
+
+test_expect_success 'setup alternate repo' '
+ git init alt-repo &&
+ (cd alt-repo &&
+ test_commit one &&
+ test_commit two
+ ) &&
+ alt_head=`cd alt-repo && git rev-parse HEAD`
+'
+
+test_expect_success 'alt objects are missing' '
+ test_must_fail git log --format=%s $alt_head
+'
+
+test_expect_success 'helper can retrieve alt objects' '
+ test_config odb.magic.command "$HELPER" &&
+ cat >expect <<-\EOF &&
+ two
+ one
+ EOF
+ git log --format=%s $alt_head >actual &&
+ test_cmp expect actual
+'
+
+test_done
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 02/10] external odb foreach
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 01/10] Add initial external odb support Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 03/10] t0400: use --batch-all-objects to get all objects Christian Couder
` (8 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider
From: Jeff King <peff@peff.net>
---
external-odb.c | 14 ++++++++++++++
external-odb.h | 6 ++++++
odb-helper.c | 15 +++++++++++++++
odb-helper.h | 4 ++++
4 files changed, 39 insertions(+)
diff --git a/external-odb.c b/external-odb.c
index 1ccfa99..42978a3 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -113,3 +113,17 @@ int external_odb_fetch_object(const unsigned char *sha1)
return -1;
}
+
+int external_odb_for_each_object(each_external_object_fn fn, void *data)
+{
+ struct odb_helper *o;
+
+ external_odb_init();
+
+ for (o = helpers; o; o = o->next) {
+ int r = odb_helper_for_each_object(o, fn, data);
+ if (r)
+ return r;
+ }
+ return 0;
+}
diff --git a/external-odb.h b/external-odb.h
index 2397477..cea8570 100644
--- a/external-odb.h
+++ b/external-odb.h
@@ -5,4 +5,10 @@ const char *external_odb_root(void);
int external_odb_has_object(const unsigned char *sha1);
int external_odb_fetch_object(const unsigned char *sha1);
+typedef int (*each_external_object_fn)(const unsigned char *sha1,
+ enum object_type type,
+ unsigned long size,
+ void *data);
+int external_odb_for_each_object(each_external_object_fn, void *);
+
#endif /* EXTERNAL_ODB_H */
diff --git a/odb-helper.c b/odb-helper.c
index 7029a59..045cf6f 100644
--- a/odb-helper.c
+++ b/odb-helper.c
@@ -235,5 +235,20 @@ int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
return -1;
}
+ return 0;
+}
+
+int odb_helper_for_each_object(struct odb_helper *o,
+ each_external_object_fn fn,
+ void *data)
+{
+ int i;
+ for (i = 0; i < o->have_nr; i++) {
+ struct odb_helper_object *obj = &o->have[i];
+ int r = fn(obj->sha1, obj->type, obj->size, data);
+ if (r)
+ return r;
+ }
+
return 0;
}
diff --git a/odb-helper.h b/odb-helper.h
index 0f704f9..8c3916d 100644
--- a/odb-helper.h
+++ b/odb-helper.h
@@ -1,6 +1,8 @@
#ifndef ODB_HELPER_H
#define ODB_HELPER_H
+#include "external-odb.h"
+
struct odb_helper {
const char *name;
const char *cmd;
@@ -21,5 +23,7 @@ struct odb_helper *odb_helper_new(const char *name, int namelen);
int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1);
int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
int fd);
+int odb_helper_for_each_object(struct odb_helper *o,
+ each_external_object_fn, void *);
#endif /* ODB_HELPER_H */
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 03/10] t0400: use --batch-all-objects to get all objects
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 01/10] Add initial external odb support Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 02/10] external odb foreach Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 04/10] t0400: add 'put' command to odb-helper script Christian Couder
` (7 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0400-external-odb.sh | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index 2b01617..fe85413 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -10,9 +10,7 @@ write_script odb-helper <<\EOF
GIT_DIR=$ALT_SOURCE; export GIT_DIR
case "$1" in
have)
- git rev-list --all --objects |
- cut -d' ' -f1 |
- git cat-file --batch-check |
+ git cat-file --batch-check --batch-all-objects |
awk '{print $1 " " $3 " " $2}'
;;
get)
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 04/10] t0400: add 'put' command to odb-helper script
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
` (2 preceding siblings ...)
2016-06-28 18:19 ` [RFC/PATCH v2 03/10] t0400: use --batch-all-objects to get all objects Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 05/10] t0400: add test for 'put' command Christian Couder
` (6 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0400-external-odb.sh | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index fe85413..0f1bb97 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -7,6 +7,10 @@ test_description='basic tests for external object databases'
ALT_SOURCE="$PWD/alt-repo/.git"
export ALT_SOURCE
write_script odb-helper <<\EOF
+die() {
+ printf >&2 "%s\n" "$@"
+ exit 1
+}
GIT_DIR=$ALT_SOURCE; export GIT_DIR
case "$1" in
have)
@@ -16,6 +20,16 @@ have)
get)
cat "$GIT_DIR"/objects/$(echo $2 | sed 's#..#&/#')
;;
+put)
+ sha1="$2"
+ size="$3"
+ kind="$4"
+ writen=$(git hash-object -w -t "$kind" --stdin)
+ test "$writen" = "$sha1" || die "bad sha1 passed '$sha1' vs writen '$writen'"
+ ;;
+*)
+ die "unknown command '$1'"
+ ;;
esac
EOF
HELPER="\"$PWD\"/odb-helper"
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 05/10] t0400: add test for 'put' command
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
` (3 preceding siblings ...)
2016-06-28 18:19 ` [RFC/PATCH v2 04/10] t0400: add 'put' command to odb-helper script Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 06/10] external odb: add write support Christian Couder
` (5 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0400-external-odb.sh | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index 0f1bb97..6c6da5c 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -57,4 +57,13 @@ test_expect_success 'helper can retrieve alt objects' '
test_cmp expect actual
'
+test_expect_success 'helper can add objects to alt repo' '
+ hash=$(echo "Hello odb!" | git hash-object -w -t blob --stdin) &&
+ test -f .git/objects/$(echo $hash | sed "s#..#&/#") &&
+ size=$(git cat-file -s "$hash") &&
+ git cat-file blob "$hash" | ./odb-helper put "$hash" "$size" blob &&
+ alt_size=$(cd alt-repo && git cat-file -s "$hash") &&
+ test "$size" -eq "$alt_size"
+'
+
test_done
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 06/10] external odb: add write support
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
` (4 preceding siblings ...)
2016-06-28 18:19 ` [RFC/PATCH v2 05/10] t0400: add test for 'put' command Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 07/10] external-odb: accept only blobs for now Christian Couder
` (4 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
external-odb.c | 15 +++++++++++++++
external-odb.h | 2 ++
odb-helper.c | 41 +++++++++++++++++++++++++++++++++++++----
odb-helper.h | 3 +++
sha1_file.c | 2 ++
5 files changed, 59 insertions(+), 4 deletions(-)
diff --git a/external-odb.c b/external-odb.c
index 42978a3..bb70fe3 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -127,3 +127,18 @@ int external_odb_for_each_object(each_external_object_fn fn, void *data)
}
return 0;
}
+
+int external_odb_write_object(const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1)
+{
+ struct odb_helper *o;
+
+ external_odb_init();
+
+ for (o = helpers; o; o = o->next) {
+ int r = odb_helper_write_object(o, buf, len, type, sha1);
+ if (r <= 0)
+ return r;
+ }
+ return 1;
+}
diff --git a/external-odb.h b/external-odb.h
index cea8570..55d291d 100644
--- a/external-odb.h
+++ b/external-odb.h
@@ -10,5 +10,7 @@ typedef int (*each_external_object_fn)(const unsigned char *sha1,
unsigned long size,
void *data);
int external_odb_for_each_object(each_external_object_fn, void *);
+int external_odb_write_object(const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1);
#endif /* EXTERNAL_ODB_H */
diff --git a/odb-helper.c b/odb-helper.c
index 045cf6f..677a5e7 100644
--- a/odb-helper.c
+++ b/odb-helper.c
@@ -33,9 +33,10 @@ static void prepare_helper_command(struct argv_array *argv, const char *cmd,
strbuf_release(&buf);
}
-__attribute__((format (printf,3,4)))
+__attribute__((format (printf,4,5)))
static int odb_helper_start(struct odb_helper *o,
struct odb_helper_cmd *cmd,
+ int use_stdin,
const char *fmt, ...)
{
va_list ap;
@@ -52,7 +53,10 @@ static int odb_helper_start(struct odb_helper *o,
cmd->child.argv = cmd->argv.argv;
cmd->child.use_shell = 1;
- cmd->child.no_stdin = 1;
+ if (use_stdin)
+ cmd->child.in = -1;
+ else
+ cmd->child.no_stdin = 1;
cmd->child.out = -1;
if (start_command(&cmd->child) < 0) {
@@ -109,7 +113,7 @@ static void odb_helper_load_have(struct odb_helper *o)
return;
o->have_valid = 1;
- if (odb_helper_start(o, &cmd, "have") < 0)
+ if (odb_helper_start(o, &cmd, 0, "have") < 0)
return;
fh = xfdopen(cmd.child.out, "r");
@@ -164,7 +168,7 @@ int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
if (!obj)
return -1;
- if (odb_helper_start(o, &cmd, "get %s", sha1_to_hex(sha1)) < 0)
+ if (odb_helper_start(o, &cmd, 0, "get %s", sha1_to_hex(sha1)) < 0)
return -1;
memset(&stream, 0, sizeof(stream));
@@ -252,3 +256,32 @@ int odb_helper_for_each_object(struct odb_helper *o,
return 0;
}
+
+int odb_helper_write_object(struct odb_helper *o,
+ const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1)
+{
+ struct odb_helper_cmd cmd;
+
+ if (odb_helper_start(o, &cmd, 1, "put %s %lu %s",
+ sha1_to_hex(sha1), len, type) < 0)
+ return -1;
+
+ do {
+ int w = xwrite(cmd.child.in, buf, len);
+ if (w < 0) {
+ error("unable to write to odb helper '%s': %s",
+ o->name, strerror(errno));
+ close(cmd.child.in);
+ close(cmd.child.out);
+ odb_helper_finish(o, &cmd);
+ return -1;
+ }
+ len -= w;
+ } while (len > 0);
+
+ close(cmd.child.in);
+ close(cmd.child.out);
+ odb_helper_finish(o, &cmd);
+ return 0;
+}
diff --git a/odb-helper.h b/odb-helper.h
index 8c3916d..af31cc2 100644
--- a/odb-helper.h
+++ b/odb-helper.h
@@ -25,5 +25,8 @@ int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
int fd);
int odb_helper_for_each_object(struct odb_helper *o,
each_external_object_fn, void *);
+int odb_helper_write_object(struct odb_helper *o,
+ const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1);
#endif /* ODB_HELPER_H */
diff --git a/sha1_file.c b/sha1_file.c
index a707bc1..90f19de 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -3192,6 +3192,8 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign
* it out into .git/objects/??/?{38} file.
*/
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
+ if (!external_odb_write_object(buf, len, type, sha1))
+ return 0;
if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
return 0;
return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 07/10] external-odb: accept only blobs for now
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
` (5 preceding siblings ...)
2016-06-28 18:19 ` [RFC/PATCH v2 06/10] external odb: add write support Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 08/10] t0400: add test for external odb write support Christian Couder
` (3 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
external-odb.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/external-odb.c b/external-odb.c
index bb70fe3..6dd7b25 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -133,6 +133,10 @@ int external_odb_write_object(const void *buf, unsigned long len,
{
struct odb_helper *o;
+ /* For now accept only blobs */
+ if (strcmp(type, "blob"))
+ return 1;
+
external_odb_init();
for (o = helpers; o; o = o->next) {
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 08/10] t0400: add test for external odb write support
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
` (6 preceding siblings ...)
2016-06-28 18:19 ` [RFC/PATCH v2 07/10] external-odb: accept only blobs for now Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 09/10] Add GIT_NO_EXTERNAL_ODB env variable Christian Couder
` (2 subsequent siblings)
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0400-external-odb.sh | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index 6c6da5c..3c868ca 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -66,4 +66,12 @@ test_expect_success 'helper can add objects to alt repo' '
test "$size" -eq "$alt_size"
'
+test_expect_success 'commit adds objects to alt repo' '
+ test_config odb.magic.command "$HELPER" &&
+ test_commit three &&
+ hash3=$(git ls-tree HEAD | grep three.t | cut -f1 | cut -d\ -f3) &&
+ content=$(cd alt-repo && git show "$hash3") &&
+ test "$content" = "three"
+'
+
test_done
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 09/10] Add GIT_NO_EXTERNAL_ODB env variable
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
` (7 preceding siblings ...)
2016-06-28 18:19 ` [RFC/PATCH v2 08/10] t0400: add test for external odb write support Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-28 18:19 ` [RFC/PATCH v2 10/10] Add t0410 to test external ODB transfer Christian Couder
2016-06-29 20:01 ` [RFC/PATCH v2 00/10] Add initial experimental external ODB support Eric Wong
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
cache.h | 9 +++++++++
environment.c | 4 ++++
external-odb.c | 6 ++++++
sha1_file.c | 3 +++
4 files changed, 22 insertions(+)
diff --git a/cache.h b/cache.h
index cc0a934..b0fe2bc 100644
--- a/cache.h
+++ b/cache.h
@@ -420,6 +420,7 @@ static inline enum object_type object_type(unsigned int mode)
#define CEILING_DIRECTORIES_ENVIRONMENT "GIT_CEILING_DIRECTORIES"
#define NO_REPLACE_OBJECTS_ENVIRONMENT "GIT_NO_REPLACE_OBJECTS"
#define GIT_REPLACE_REF_BASE_ENVIRONMENT "GIT_REPLACE_REF_BASE"
+#define NO_EXTERNAL_ODB_ENVIRONMENT "GIT_NO_EXTERNAL_ODB"
#define GITATTRIBUTES_FILE ".gitattributes"
#define INFOATTRIBUTES_FILE "info/attributes"
#define ATTRIBUTE_MACRO_PREFIX "[attr]"
@@ -678,6 +679,14 @@ int get_shared_repository(void);
extern int check_replace_refs;
extern char *git_replace_ref_base;
+/*
+ * Do external odbs need to be used this run? This variable is
+ * initialized to true unless $GIT_NO_EXTERNAL_ODB is set, but it
+ * maybe set to false by some commands that do not want external
+ * odbs to be active.
+ */
+extern int use_external_odb;
+
extern int fsync_object_files;
extern int core_preload_index;
extern int core_apply_sparse_checkout;
diff --git a/environment.c b/environment.c
index ca72464..1799339 100644
--- a/environment.c
+++ b/environment.c
@@ -48,6 +48,7 @@ const char *excludes_file;
enum auto_crlf auto_crlf = AUTO_CRLF_FALSE;
int check_replace_refs = 1;
char *git_replace_ref_base;
+int use_external_odb = 1;
enum eol core_eol = EOL_UNSET;
enum safe_crlf safe_crlf = SAFE_CRLF_WARN;
unsigned whitespace_rule_cfg = WS_DEFAULT_RULE;
@@ -119,6 +120,7 @@ const char * const local_repo_env[] = {
INDEX_ENVIRONMENT,
NO_REPLACE_OBJECTS_ENVIRONMENT,
GIT_REPLACE_REF_BASE_ENVIRONMENT,
+ NO_EXTERNAL_ODB_ENVIRONMENT,
GIT_PREFIX_ENVIRONMENT,
GIT_SHALLOW_FILE_ENVIRONMENT,
GIT_COMMON_DIR_ENVIRONMENT,
@@ -183,6 +185,8 @@ static void setup_git_env(void)
replace_ref_base = getenv(GIT_REPLACE_REF_BASE_ENVIRONMENT);
git_replace_ref_base = xstrdup(replace_ref_base ? replace_ref_base
: "refs/replace/");
+ if (getenv(NO_EXTERNAL_ODB_ENVIRONMENT))
+ use_external_odb = 0;
namespace = expand_namespace(getenv(GIT_NAMESPACE_ENVIRONMENT));
namespace_len = strlen(namespace);
shallow_file = getenv(GIT_SHALLOW_FILE_ENVIRONMENT);
diff --git a/external-odb.c b/external-odb.c
index 6dd7b25..a980fbf 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -63,6 +63,9 @@ int external_odb_has_object(const unsigned char *sha1)
{
struct odb_helper *o;
+ if (!use_external_odb)
+ return 0;
+
external_odb_init();
for (o = helpers; o; o = o->next)
@@ -133,6 +136,9 @@ int external_odb_write_object(const void *buf, unsigned long len,
{
struct odb_helper *o;
+ if (!use_external_odb)
+ return 1;
+
/* For now accept only blobs */
if (strcmp(type, "blob"))
return 1;
diff --git a/sha1_file.c b/sha1_file.c
index 90f19de..13d4d75 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -448,6 +448,9 @@ void prepare_external_alt_odb(void)
static int linked_external;
const char *path;
+ if (!use_external_odb)
+ return;
+
if (linked_external)
return;
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* [RFC/PATCH v2 10/10] Add t0410 to test external ODB transfer
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
` (8 preceding siblings ...)
2016-06-28 18:19 ` [RFC/PATCH v2 09/10] Add GIT_NO_EXTERNAL_ODB env variable Christian Couder
@ 2016-06-28 18:19 ` Christian Couder
2016-06-29 20:01 ` [RFC/PATCH v2 00/10] Add initial experimental external ODB support Eric Wong
10 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-28 18:19 UTC (permalink / raw)
To: git
Cc: Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
t/t0410-transfer-e-odb.sh | 136 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 136 insertions(+)
create mode 100755 t/t0410-transfer-e-odb.sh
diff --git a/t/t0410-transfer-e-odb.sh b/t/t0410-transfer-e-odb.sh
new file mode 100755
index 0000000..868b55d
--- /dev/null
+++ b/t/t0410-transfer-e-odb.sh
@@ -0,0 +1,136 @@
+#!/bin/sh
+
+test_description='basic tests for transfering external ODBs'
+
+. ./test-lib.sh
+
+ORIG_SOURCE="$PWD/.git"
+export ORIG_SOURCE
+
+ALT_SOURCE1="$PWD/alt-repo1/.git"
+export ALT_SOURCE1
+write_script odb-helper1 <<\EOF
+die() {
+ printf >&2 "%s\n" "$@"
+ exit 1
+}
+GIT_DIR=$ALT_SOURCE1; export GIT_DIR
+case "$1" in
+have)
+ git cat-file --batch-check --batch-all-objects |
+ awk '{print $1 " " $3 " " $2}'
+ ;;
+get)
+ cat "$GIT_DIR"/objects/$(echo $2 | sed 's#..#&/#')
+ ;;
+put)
+ sha1="$2"
+ size="$3"
+ kind="$4"
+ writen=$(git hash-object -w -t "$kind" --stdin)
+ test "$writen" = "$sha1" || die "bad sha1 passed '$sha1' vs writen '$writen'"
+ ref_hash=$(echo "$sha1 $size $kind" | GIT_DIR=$ORIG_SOURCE GIT_NO_EXTERNAL_ODB=1 git hash-object -w -t blob --stdin) || exit
+ GIT_DIR=$ORIG_SOURCE git update-ref refs/odbs/magic/"$sha1" "$ref_hash"
+ ;;
+*)
+ die "unknown command '$1'"
+ ;;
+esac
+EOF
+HELPER1="\"$PWD\"/odb-helper1"
+
+OTHER_SOURCE="$PWD/.git"
+export OTHER_SOURCE
+
+ALT_SOURCE2="$PWD/alt-repo2/.git"
+export ALT_SOURCE2
+write_script odb-helper2 <<\EOF
+die() {
+ printf >&2 "%s\n" "$@"
+ exit 1
+}
+GIT_DIR=$ALT_SOURCE2; export GIT_DIR
+case "$1" in
+have)
+ GIT_DIR=$OTHER_SOURCE git for-each-ref --format='%(objectname)' refs/odbs/magic/ | GIT_DIR=$OTHER_SOURCE xargs git show
+ ;;
+get)
+ OBJ_FILE="$GIT_DIR"/objects/$(echo $2 | sed 's#..#&/#')
+ if ! test -f "$OBJ_FILE"
+ then
+ # "Download" the missing object by copying it from alt-repo1
+ OBJ_DIR=$(echo $2 | sed 's/\(..\).*/\1/')
+ OBJ_BASE=$(basename "$OBJ_FILE")
+ ALT_OBJ_DIR1="$ALT_SOURCE1/objects/$OBJ_DIR"
+ ALT_OBJ_DIR2="$ALT_SOURCE2/objects/$OBJ_DIR"
+ mkdir -p "$ALT_OBJ_DIR2" || die "Could not mkdir '$ALT_OBJ_DIR2'"
+ OBJ_SRC="$ALT_OBJ_DIR1/$OBJ_BASE"
+ cp "$OBJ_SRC" "$ALT_OBJ_DIR2" ||
+ die "Could not cp '$OBJ_SRC' into '$ALT_OBJ_DIR2'"
+ fi
+ cat "$OBJ_FILE" || die "Could not cat '$OBJ_FILE'"
+ ;;
+put)
+ sha1="$2"
+ size="$3"
+ kind="$4"
+ writen=$(git hash-object -w -t "$kind" --stdin)
+ test "$writen" = "$sha1" || die "bad sha1 passed '$sha1' vs writen '$writen'"
+ ref_hash=$(echo "$sha1 $size $kind" | GIT_DIR=$OTHER_SOURCE GIT_NO_EXTERNAL_ODB=1 git hash-object -w -t blob --stdin) || exit
+ GIT_DIR=$OTHER_SOURCE git update-ref refs/odbs/magic/"$sha1" "$ref_hash"
+ ;;
+*)
+ die "unknown command '$1'"
+ ;;
+esac
+EOF
+HELPER2="\"$PWD\"/odb-helper2"
+
+test_expect_success 'setup first alternate repo' '
+ git init alt-repo1 &&
+ test_commit zero &&
+ git config odb.magic.command "$HELPER1"
+'
+
+test_expect_success 'setup other repo and its alternate repo' '
+ git init other-repo &&
+ git init alt-repo2 &&
+ (cd other-repo &&
+ git remote add origin .. &&
+ git pull origin master &&
+ git checkout master &&
+ git log)
+'
+
+test_expect_success 'new blobs are put in first object store' '
+ test_commit one &&
+ hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\ -f3) &&
+ content=$(cd alt-repo1 && git show "$hash1") &&
+ test "$content" = "one" &&
+ test_commit two &&
+ hash2=$(git ls-tree HEAD | grep two.t | cut -f1 | cut -d\ -f3) &&
+ content=$(cd alt-repo1 && git show "$hash2") &&
+ test "$content" = "two"
+'
+
+test_expect_success 'other repo gets the blobs from object store' '
+ (cd other-repo &&
+ git fetch origin "refs/odbs/magic/*:refs/odbs/magic/*" &&
+ test_must_fail git cat-file blob "$hash1" &&
+ test_must_fail git cat-file blob "$hash2" &&
+ git config odb.magic.command "$HELPER2" &&
+ git cat-file blob "$hash1" &&
+ git cat-file blob "$hash2"
+ )
+'
+
+test_expect_success 'other repo gets everything else' '
+ (cd other-repo &&
+ git fetch origin &&
+ content=$(git show "$hash1") &&
+ test "$content" = "one" &&
+ content=$(git show "$hash2") &&
+ test "$content" = "two")
+'
+
+test_done
--
2.9.0.rc2.11.g990c140
^ permalink raw reply related [flat|nested] 13+ messages in thread
* Re: [RFC/PATCH v2 00/10] Add initial experimental external ODB support
2016-06-28 18:19 [RFC/PATCH v2 00/10] Add initial experimental external ODB support Christian Couder
` (9 preceding siblings ...)
2016-06-28 18:19 ` [RFC/PATCH v2 10/10] Add t0410 to test external ODB transfer Christian Couder
@ 2016-06-29 20:01 ` Eric Wong
2016-06-30 9:49 ` Christian Couder
10 siblings, 1 reply; 13+ messages in thread
From: Eric Wong @ 2016-06-29 20:01 UTC (permalink / raw)
To: Christian Couder
Cc: git, Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
Christian Couder <christian.couder@gmail.com> wrote:
> Design discussion about performance
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>
> Yeah, it is not efficient to fork/exec a command to just read or write
> one object to or from the external ODB. Batch calls and/or using a
> daemon and/or RPC should be used instead to be able to store regular
> objects in an external ODB. But for now the external ODB would be all
> about really big files, where the cost of a fork+exec should not
> matter much. If we later want to extend usage of external ODBs, yeah
> we will probably need to design other mechanisms.
I would also investigate switching run_command to use vfork+exec
or posix_spawn for performance (keeping in mind vfork
caveats documented at https://ewontfix.com/7/ )
posix_spawn in future glibc (probably 2.24) will use CLONE_VFORK
in all cases under Linux, and posix_spawn may help with
portability, too. I think the only thing we can't support
with posix_spawn which run_command supports is chdir;
all the redirections/closing FDs should be fine.
With only 10MB malloc-ed, the following shows vfork performance
being noticeably faster than plain fork:
/* gcc -o vfork-test -Wall -O2 vfork-test.c */
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
int main(int argc, char *argv[], char *envp[])
{
int i;
int do_vfork = argc > 1 && !strcmp(argv[1], "vfork");
char * const cmd[] = { "/bin/true", 0 };
size_t n = 1024 * 1024 * 10;
char *mem = malloc(n);
memset(mem, 'a', n); /* make sure it's really allocated */
for (i = 0; i < 10000; i++) {
pid_t pid = do_vfork ? vfork() : fork();
if (pid == 0) {
execve(cmd[0], cmd, envp);
write(2, "exec error\n", 11);
_exit(1);
}
waitpid(pid, 0, 0);
}
return 0;
}
^ permalink raw reply [flat|nested] 13+ messages in thread
* Re: [RFC/PATCH v2 00/10] Add initial experimental external ODB support
2016-06-29 20:01 ` [RFC/PATCH v2 00/10] Add initial experimental external ODB support Eric Wong
@ 2016-06-30 9:49 ` Christian Couder
0 siblings, 0 replies; 13+ messages in thread
From: Christian Couder @ 2016-06-30 9:49 UTC (permalink / raw)
To: Eric Wong
Cc: git, Junio C Hamano, Jeff King, Nguyen Thai Ngoc Duy, Mike Hommey,
Lars Schneider, Christian Couder
On Wed, Jun 29, 2016 at 10:01 PM, Eric Wong <e@80x24.org> wrote:
> Christian Couder <christian.couder@gmail.com> wrote:
>> Design discussion about performance
>> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>>
>> Yeah, it is not efficient to fork/exec a command to just read or write
>> one object to or from the external ODB. Batch calls and/or using a
>> daemon and/or RPC should be used instead to be able to store regular
>> objects in an external ODB. But for now the external ODB would be all
>> about really big files, where the cost of a fork+exec should not
>> matter much. If we later want to extend usage of external ODBs, yeah
>> we will probably need to design other mechanisms.
>
> I would also investigate switching run_command to use vfork+exec
> or posix_spawn for performance (keeping in mind vfork
> caveats documented at https://ewontfix.com/7/ )
Thanks Eric for this idea and the test!
^ permalink raw reply [flat|nested] 13+ messages in thread