From: Christian Couder <christian.couder@gmail.com>
To: git@vger.kernel.org
Cc: Junio C Hamano <gitster@pobox.com>, Jeff King <peff@peff.net>,
Ben Peart <Ben.Peart@microsoft.com>,
Nguyen Thai Ngoc Duy <pclouds@gmail.com>,
Mike Hommey <mh@glandium.org>,
Lars Schneider <larsxschneider@gmail.com>,
Eric Wong <e@80x24.org>,
Christian Couder <chriscool@tuxfamily.org>,
Ben Peart <benpeart@microsoft.com>
Subject: [RFC/PATCH v4 30/49] odb-helper: add read_object_process()
Date: Tue, 20 Jun 2017 09:55:04 +0200 [thread overview]
Message-ID: <20170620075523.26961-31-chriscool@tuxfamily.org> (raw)
In-Reply-To: <20170620075523.26961-1-chriscool@tuxfamily.org>
From: Ben Peart <benpeart@microsoft.com>
Signed-off-by: Ben Peart <benpeart@microsoft.com>
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
odb-helper.c | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
odb-helper.h | 5 ++
sha1_file.c | 33 +++++++++-
3 files changed, 227 insertions(+), 13 deletions(-)
diff --git a/odb-helper.c b/odb-helper.c
index 5fb56c6135..20e83cb55a 100644
--- a/odb-helper.c
+++ b/odb-helper.c
@@ -4,6 +4,187 @@
#include "odb-helper.h"
#include "run-command.h"
#include "sha1-lookup.h"
+#include "sub-process.h"
+#include "pkt-line.h"
+#include "sigchain.h"
+
+struct read_object_process {
+ struct subprocess_entry subprocess;
+ unsigned int supported_capabilities;
+};
+
+static int subprocess_map_initialized;
+static struct hashmap subprocess_map;
+
+static void parse_capabilities(char *cap_buf,
+ unsigned int *supported_capabilities,
+ const char *process_name)
+{
+ struct string_list cap_list = STRING_LIST_INIT_NODUP;
+
+ string_list_split_in_place(&cap_list, cap_buf, '=', 1);
+
+ if (cap_list.nr == 2 && !strcmp(cap_list.items[0].string, "capability")) {
+ const char *cap_name = cap_list.items[1].string;
+
+ if (!strcmp(cap_name, "get")) {
+ *supported_capabilities |= ODB_HELPER_CAP_GET;
+ } else if (!strcmp(cap_name, "put")) {
+ *supported_capabilities |= ODB_HELPER_CAP_PUT;
+ } else if (!strcmp(cap_name, "have")) {
+ *supported_capabilities |= ODB_HELPER_CAP_HAVE;
+ } else {
+ warning("external process '%s' requested unsupported read-object capability '%s'",
+ process_name, cap_name);
+ }
+ }
+
+ string_list_clear(&cap_list, 0);
+}
+
+static int start_read_object_fn(struct subprocess_entry *subprocess)
+{
+ int err;
+ struct read_object_process *entry = (struct read_object_process *)subprocess;
+ struct child_process *process = &subprocess->process;
+ char *cap_buf;
+
+ sigchain_push(SIGPIPE, SIG_IGN);
+
+ err = packet_writel(process->in, "git-read-object-client", "version=1", NULL);
+ if (err)
+ goto done;
+
+ err = strcmp(packet_read_line(process->out, NULL), "git-read-object-server");
+ if (err) {
+ error("external process '%s' does not support read-object protocol version 1", subprocess->cmd);
+ goto done;
+ }
+ err = strcmp(packet_read_line(process->out, NULL), "version=1");
+ if (err)
+ goto done;
+ err = packet_read_line(process->out, NULL) != NULL;
+ if (err)
+ goto done;
+
+ err = packet_writel(process->in, "capability=get", NULL);
+ if (err)
+ goto done;
+
+ while ((cap_buf = packet_read_line(process->out, NULL)))
+ parse_capabilities(cap_buf, &entry->supported_capabilities, subprocess->cmd);
+
+done:
+ sigchain_pop(SIGPIPE);
+
+ return err;
+}
+
+static struct read_object_process *launch_read_object_process(const char *cmd)
+{
+ struct read_object_process *entry;
+
+ if (!subprocess_map_initialized) {
+ subprocess_map_initialized = 1;
+ hashmap_init(&subprocess_map, (hashmap_cmp_fn) cmd2process_cmp, 0);
+ entry = NULL;
+ } else {
+ entry = (struct read_object_process *)subprocess_find_entry(&subprocess_map, cmd);
+ }
+
+ fflush(NULL);
+
+ if (!entry) {
+ entry = xmalloc(sizeof(*entry));
+ entry->supported_capabilities = 0;
+
+ if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_read_object_fn)) {
+ free(entry);
+ return 0;
+ }
+ }
+
+ return entry;
+}
+
+static int check_object_process_error(int err,
+ const char *status,
+ struct read_object_process *entry,
+ const char *cmd,
+ unsigned int capability)
+{
+ if (!err)
+ return;
+
+ if (!strcmp(status, "error")) {
+ /* The process signaled a problem with the file. */
+ } else if (!strcmp(status, "notfound")) {
+ /* Object was not found */
+ err = -1;
+ } else if (!strcmp(status, "abort")) {
+ /*
+ * The process signaled a permanent problem. Don't try to read
+ * objects with the same command for the lifetime of the current
+ * Git process.
+ */
+ if (capability)
+ entry->supported_capabilities &= ~capability;
+ } else {
+ /*
+ * Something went wrong with the read-object process.
+ * Force shutdown and restart if needed.
+ */
+ error("external object process '%s' failed", cmd);
+ subprocess_stop(&subprocess_map, &entry->subprocess);
+ free(entry);
+ }
+
+ return err;
+}
+
+static int read_object_process(const unsigned char *sha1)
+{
+ int err;
+ struct read_object_process *entry;
+ struct child_process *process;
+ struct strbuf status = STRBUF_INIT;
+ const char *cmd = "read-object";
+ uint64_t start;
+
+ start = getnanotime();
+
+ entry = launch_read_object_process(cmd);
+ process = &entry->subprocess.process;
+
+ if (!(ODB_HELPER_CAP_GET & entry->supported_capabilities))
+ return -1;
+
+ sigchain_push(SIGPIPE, SIG_IGN);
+
+ err = packet_write_fmt_gently(process->in, "command=get\n");
+ if (err)
+ goto done;
+
+ err = packet_write_fmt_gently(process->in, "sha1=%s\n", sha1_to_hex(sha1));
+ if (err)
+ goto done;
+
+ err = packet_flush_gently(process->in);
+ if (err)
+ goto done;
+
+ subprocess_read_status(process->out, &status);
+ err = strcmp(status.buf, "success");
+
+done:
+ sigchain_pop(SIGPIPE);
+
+ err = check_object_process_error(err, status.buf, entry, cmd, ODB_HELPER_CAP_GET);
+
+ trace_performance_since(start, "read_object_process");
+
+ return err;
+}
struct odb_helper *odb_helper_new(const char *name, int namelen)
{
@@ -350,20 +531,21 @@ static int odb_helper_fetch_git_object(struct odb_helper *o,
int odb_helper_fault_in_object(struct odb_helper *o,
const unsigned char *sha1)
{
- struct odb_helper_object *obj;
- struct odb_helper_cmd cmd;
+ struct odb_helper_object *obj = odb_helper_lookup(o, sha1);
- obj = odb_helper_lookup(o, sha1);
if (!obj)
return -1;
- if (odb_helper_start(o, &cmd, 0, "get %s", sha1_to_hex(sha1)) < 0)
- return -1;
-
- if (odb_helper_finish(o, &cmd))
- return -1;
-
- return 0;
+ if (o->script_mode) {
+ struct odb_helper_cmd cmd;
+ if (odb_helper_start(o, &cmd, 0, "get %s", sha1_to_hex(sha1)) < 0)
+ return -1;
+ if (odb_helper_finish(o, &cmd))
+ return -1;
+ return 0;
+ } else {
+ return read_object_process(sha1);
+ }
}
int odb_helper_fetch_object(struct odb_helper *o,
diff --git a/odb-helper.h b/odb-helper.h
index 44c98bbf56..b23544aa4a 100644
--- a/odb-helper.h
+++ b/odb-helper.h
@@ -9,11 +9,16 @@ enum odb_helper_fetch_kind {
ODB_FETCH_KIND_FAULT_IN
};
+#define ODB_HELPER_CAP_GET (1u<<0)
+#define ODB_HELPER_CAP_PUT (1u<<1)
+#define ODB_HELPER_CAP_HAVE (1u<<2)
+
struct odb_helper {
const char *name;
const char *cmd;
enum odb_helper_fetch_kind fetch_kind;
int script_mode;
+ unsigned int supported_capabilities;
struct odb_helper_object {
unsigned char sha1[20];
diff --git a/sha1_file.c b/sha1_file.c
index 9d8e37432e..38a0404506 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -698,7 +698,17 @@ int check_and_freshen_file(const char *fn, int freshen)
static int check_and_freshen_local(const unsigned char *sha1, int freshen)
{
- return check_and_freshen_file(sha1_file_name(sha1), freshen);
+ int ret;
+ int tried_hook = 0;
+
+retry:
+ ret = check_and_freshen_file(sha1_file_name(sha1), freshen);
+ if (!ret && !tried_hook) {
+ tried_hook = 1;
+ if (!external_odb_fault_in_object(sha1))
+ goto retry;
+ }
+ return ret;
}
static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
@@ -3000,7 +3010,9 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,
int rtype;
enum object_type real_type;
const unsigned char *real = lookup_replace_object_extended(sha1, flags);
+ int tried_hook = 0;
+retry:
co = find_cached_object(real);
if (co) {
if (oi->typep)
@@ -3026,8 +3038,14 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,
/* Not a loose object; someone else may have just packed it. */
reprepare_packed_git();
- if (!find_pack_entry(real, &e))
+ if (!find_pack_entry(real, &e)) {
+ if (!tried_hook) {
+ tried_hook = 1;
+ if (!external_odb_fault_in_object(sha1))
+ goto retry;
+ }
return -1;
+ }
}
/*
@@ -3121,7 +3139,9 @@ static void *read_object(const unsigned char *sha1, enum object_type *type,
unsigned long mapsize;
void *map, *buf;
struct cached_object *co;
+ int tried_hook = 0;
+retry:
co = find_cached_object(sha1);
if (co) {
*type = co->type;
@@ -3139,7 +3159,14 @@ static void *read_object(const unsigned char *sha1, enum object_type *type,
return buf;
}
reprepare_packed_git();
- return read_packed_sha1(sha1, type, size);
+ buf = read_packed_sha1(sha1, type, size);
+ if (!buf && !tried_hook) {
+ tried_hook = 1;
+ if (!external_odb_fault_in_object(sha1))
+ goto retry;
+ }
+
+ return buf;
}
/*
--
2.13.1.565.gbfcd7a9048
next prev parent reply other threads:[~2017-06-20 7:56 UTC|newest]
Thread overview: 64+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-06-20 7:54 [RFC/PATCH v4 00/49] Add initial experimental external ODB support Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 01/49] builtin/clone: get rid of 'value' strbuf Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 02/49] t0021/rot13-filter: refactor packet reading functions Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 03/49] t0021/rot13-filter: improve 'if .. elsif .. else' style Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 04/49] Add Git/Packet.pm from parts of t0021/rot13-filter.pl Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 05/49] t0021/rot13-filter: use Git/Packet.pm Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 06/49] Git/Packet.pm: improve error message Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 07/49] Git/Packet.pm: add packet_initialize() Christian Couder
2017-06-23 18:55 ` Ben Peart
2017-06-20 7:54 ` [RFC/PATCH v4 08/49] Git/Packet: add capability functions Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 09/49] Add initial external odb support Christian Couder
2017-06-23 19:49 ` Ben Peart
2017-06-20 7:54 ` [RFC/PATCH v4 10/49] external odb foreach Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 11/49] t0400: add 'put' command to odb-helper script Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 12/49] external odb: add write support Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 13/49] external-odb: accept only blobs for now Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 14/49] t0400: add test for external odb write support Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 15/49] Add GIT_NO_EXTERNAL_ODB env variable Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 16/49] Add t0410 to test external ODB transfer Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 17/49] lib-httpd: pass config file to start_httpd() Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 18/49] lib-httpd: add upload.sh Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 19/49] lib-httpd: add list.sh Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 20/49] lib-httpd: add apache-e-odb.conf Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 21/49] odb-helper: add 'store_plain_objects' to 'struct odb_helper' Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 22/49] pack-objects: don't pack objects in external odbs Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 23/49] t0420: add test with HTTP external odb Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 24/49] odb-helper: start fault in implementation Christian Couder
2017-06-20 7:54 ` [RFC/PATCH v4 25/49] external-odb: add external_odb_fault_in_object() Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 26/49] odb-helper: add script_mode Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 27/49] Documentation: add read-object-protocol.txt Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 28/49] contrib: add long-running-read-object/example.pl Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 29/49] Add t0410 to test read object mechanism Christian Couder
2017-06-20 7:55 ` Christian Couder [this message]
2017-07-10 15:57 ` [RFC/PATCH v4 30/49] odb-helper: add read_object_process() Ben Peart
2017-06-20 7:55 ` [RFC/PATCH v4 31/49] external-odb: add external_odb_get_capabilities() Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 32/49] t04*: add 'get_cap' support to helpers Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 33/49] odb-helper: call odb_helper_lookup() with 'have' capability Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 34/49] odb-helper: fix odb_helper_fetch_object() for read_object Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 35/49] Add t0460 to test passing git objects Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 36/49] odb-helper: add read_packetized_git_object_to_fd() Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 37/49] odb-helper: add read_packetized_plain_object_to_fd() Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 38/49] Add t0470 to test passing plain objects Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 39/49] odb-helper: add write_object_process() Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 40/49] Add t0480 to test "have" capability and plain objects Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 41/49] external-odb: add external_odb_do_fetch_object() Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 42/49] odb-helper: advertise 'have' capability Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 43/49] odb-helper: advertise 'put' capability Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 44/49] odb-helper: add have_object_process() Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 45/49] clone: add initial param to write_remote_refs() Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 46/49] clone: add --initial-refspec option Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 47/49] clone: disable external odb before initial clone Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 48/49] Add test for 'clone --initial-refspec' Christian Couder
2017-06-20 7:55 ` [RFC/PATCH v4 49/49] t: add t0430 to test cloning using bundles Christian Couder
2017-06-20 13:48 ` [RFC/PATCH v4 00/49] Add initial experimental external ODB support Christian Couder
2017-06-23 18:24 ` Ben Peart
2017-07-01 19:41 ` Christian Couder
2017-07-01 20:12 ` Christian Couder
2017-07-01 20:33 ` Junio C Hamano
2017-07-02 4:25 ` Christian Couder
2017-07-03 16:56 ` Junio C Hamano
2017-07-06 17:36 ` Ben Peart
2017-09-15 12:56 ` Christian Couder
2017-07-12 19:06 ` Jonathan Tan
2017-09-15 13:16 ` Christian Couder
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20170620075523.26961-31-chriscool@tuxfamily.org \
--to=christian.couder@gmail.com \
--cc=Ben.Peart@microsoft.com \
--cc=benpeart@microsoft.com \
--cc=chriscool@tuxfamily.org \
--cc=e@80x24.org \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=larsxschneider@gmail.com \
--cc=mh@glandium.org \
--cc=pclouds@gmail.com \
--cc=peff@peff.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).