From: Christian Couder <christian.couder@gmail.com>
To: git@vger.kernel.org
Cc: Junio C Hamano <gitster@pobox.com>, Jeff King <peff@peff.net>,
Ben Peart <Ben.Peart@microsoft.com>,
Jonathan Tan <jonathantanmy@google.com>,
Nguyen Thai Ngoc Duy <pclouds@gmail.com>,
Mike Hommey <mh@glandium.org>,
Lars Schneider <larsxschneider@gmail.com>,
Eric Wong <e@80x24.org>,
Christian Couder <chriscool@tuxfamily.org>
Subject: [PATCH v6 21/40] odb-helper: add odb_helper_get_raw_object()
Date: Sat, 16 Sep 2017 10:07:12 +0200 [thread overview]
Message-ID: <20170916080731.13925-22-chriscool@tuxfamily.org> (raw)
In-Reply-To: <20170916080731.13925-1-chriscool@tuxfamily.org>
The existing odb_helper_get_object() is renamed
odb_helper_get_git_object() and a new odb_helper_get_raw_object()
is introduced to deal with external objects that are not in Git format.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
odb-helper.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 111 insertions(+), 2 deletions(-)
diff --git a/odb-helper.c b/odb-helper.c
index 39d20fdfd7..1f4666b349 100644
--- a/odb-helper.c
+++ b/odb-helper.c
@@ -221,8 +221,107 @@ int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1)
return !!odb_helper_lookup(o, sha1);
}
-int odb_helper_get_object(struct odb_helper *o, const unsigned char *sha1,
- int fd)
+static int odb_helper_get_raw_object(struct odb_helper *o,
+ const unsigned char *sha1,
+ int fd)
+{
+ struct odb_helper_object *obj;
+ struct odb_helper_cmd cmd;
+ unsigned long total_got = 0;
+
+ char hdr[32];
+ int hdrlen;
+
+ int ret = Z_STREAM_END;
+ unsigned char compressed[4096];
+ git_zstream stream;
+ git_SHA_CTX hash;
+ unsigned char real_sha1[20];
+
+ obj = odb_helper_lookup(o, sha1);
+ if (!obj)
+ return -1;
+
+ if (odb_helper_start(o, &cmd, 0, "get_raw_obj %s", sha1_to_hex(sha1)) < 0)
+ return -1;
+
+ /* Set it up */
+ git_deflate_init(&stream, zlib_compression_level);
+ stream.next_out = compressed;
+ stream.avail_out = sizeof(compressed);
+ git_SHA1_Init(&hash);
+
+ /* First header.. */
+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj->type), obj->size) + 1;
+ stream.next_in = (unsigned char *)hdr;
+ stream.avail_in = hdrlen;
+ while (git_deflate(&stream, 0) == Z_OK)
+ ; /* nothing */
+ git_SHA1_Update(&hash, hdr, hdrlen);
+
+ for (;;) {
+ unsigned char buf[4096];
+ int r;
+
+ r = xread(cmd.child.out, buf, sizeof(buf));
+ if (r < 0) {
+ error("unable to read from odb helper '%s': %s",
+ o->name, strerror(errno));
+ close(cmd.child.out);
+ odb_helper_finish(o, &cmd);
+ git_deflate_end(&stream);
+ return -1;
+ }
+ if (r == 0)
+ break;
+
+ total_got += r;
+
+ /* Then the data itself.. */
+ stream.next_in = (void *)buf;
+ stream.avail_in = r;
+ do {
+ unsigned char *in0 = stream.next_in;
+ ret = git_deflate(&stream, Z_FINISH);
+ git_SHA1_Update(&hash, in0, stream.next_in - in0);
+ write_or_die(fd, compressed, stream.next_out - compressed);
+ stream.next_out = compressed;
+ stream.avail_out = sizeof(compressed);
+ } while (ret == Z_OK);
+ }
+
+ close(cmd.child.out);
+ if (ret != Z_STREAM_END) {
+ warning("bad zlib data from odb helper '%s' for %s",
+ o->name, sha1_to_hex(sha1));
+ return -1;
+ }
+ ret = git_deflate_end_gently(&stream);
+ if (ret != Z_OK) {
+ warning("deflateEnd on object %s from odb helper '%s' failed (%d)",
+ sha1_to_hex(sha1), o->name, ret);
+ return -1;
+ }
+ git_SHA1_Final(real_sha1, &hash);
+ if (hashcmp(sha1, real_sha1)) {
+ warning("sha1 mismatch from odb helper '%s' for %s (got %s)",
+ o->name, sha1_to_hex(sha1), sha1_to_hex(real_sha1));
+ return -1;
+ }
+ if (odb_helper_finish(o, &cmd))
+ return -1;
+ if (total_got != obj->size) {
+ warning("size mismatch from odb helper '%s' for %s (%lu != %lu)",
+ o->name, sha1_to_hex(sha1), total_got, obj->size);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int odb_helper_get_git_object(struct odb_helper *o,
+ const unsigned char *sha1,
+ int fd)
{
struct odb_helper_object *obj;
struct odb_helper_cmd cmd;
@@ -326,6 +425,16 @@ int odb_helper_get_object(struct odb_helper *o, const unsigned char *sha1,
return 0;
}
+int odb_helper_get_object(struct odb_helper *o,
+ const unsigned char *sha1,
+ int fd)
+{
+ if (o->supported_capabilities & ODB_HELPER_CAP_GET_RAW_OBJ)
+ return odb_helper_get_raw_object(o, sha1, fd);
+ else
+ return odb_helper_get_git_object(o, sha1, fd);
+}
+
int odb_helper_put_object(struct odb_helper *o,
const void *buf, size_t len,
const char *type, unsigned char *sha1)
--
2.14.1.576.g3f707d88cd
next prev parent reply other threads:[~2017-09-16 8:09 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-09-16 8:06 [PATCH v6 00/40] Add initial experimental external ODB support Christian Couder
2017-09-16 8:06 ` [PATCH v6 01/40] builtin/clone: get rid of 'value' strbuf Christian Couder
2017-09-16 8:06 ` [PATCH v6 02/40] t0021/rot13-filter: refactor packet reading functions Christian Couder
2017-09-16 8:06 ` [PATCH v6 03/40] t0021/rot13-filter: improve 'if .. elsif .. else' style Christian Couder
2017-09-16 8:06 ` [PATCH v6 04/40] t0021/rot13-filter: improve error message Christian Couder
2017-09-16 8:06 ` [PATCH v6 05/40] t0021/rot13-filter: add packet_initialize() Christian Couder
2017-09-16 8:06 ` [PATCH v6 06/40] t0021/rot13-filter: add capability functions Christian Couder
2017-09-16 8:06 ` [PATCH v6 07/40] Add Git/Packet.pm from parts of t0021/rot13-filter.pl Christian Couder
2017-09-16 8:06 ` [PATCH v6 08/40] sha1_file: prepare for external odbs Christian Couder
2017-09-16 8:07 ` [PATCH v6 09/40] Add initial external odb support Christian Couder
2017-09-19 17:45 ` Jonathan Tan
2017-09-27 16:46 ` Christian Couder
2017-09-29 20:36 ` Jonathan Tan
2017-10-02 14:34 ` Ben Peart
2017-10-03 9:45 ` Christian Couder
2017-10-04 0:15 ` Jonathan Tan
2017-09-16 8:07 ` [PATCH v6 10/40] odb-helper: add odb_helper_init() to send 'init' instruction Christian Couder
2017-09-16 8:07 ` [PATCH v6 11/40] t0400: add 'put_raw_obj' instruction to odb-helper script Christian Couder
2017-09-16 8:07 ` [PATCH v6 12/40] external odb: add 'put_raw_obj' support Christian Couder
2017-09-16 8:07 ` [PATCH v6 13/40] external-odb: accept only blobs for now Christian Couder
2017-09-16 8:07 ` [PATCH v6 14/40] t0400: add test for external odb write support Christian Couder
2017-09-16 8:07 ` [PATCH v6 15/40] Add GIT_NO_EXTERNAL_ODB env variable Christian Couder
2017-09-16 8:07 ` [PATCH v6 16/40] Add t0410 to test external ODB transfer Christian Couder
2017-09-16 8:07 ` [PATCH v6 17/40] lib-httpd: pass config file to start_httpd() Christian Couder
2017-09-16 8:07 ` [PATCH v6 18/40] lib-httpd: add upload.sh Christian Couder
2017-09-16 8:07 ` [PATCH v6 19/40] lib-httpd: add list.sh Christian Couder
2017-09-16 8:07 ` [PATCH v6 20/40] lib-httpd: add apache-e-odb.conf Christian Couder
2017-09-16 8:07 ` Christian Couder [this message]
2017-09-16 8:07 ` [PATCH v6 22/40] pack-objects: don't pack objects in external odbs Christian Couder
2017-09-16 8:07 ` [PATCH v6 23/40] Add t0420 to test transfer to HTTP external odb Christian Couder
2017-09-16 8:07 ` [PATCH v6 24/40] external-odb: add 'get_direct' support Christian Couder
2017-09-16 8:07 ` [PATCH v6 25/40] odb-helper: add 'script_mode' to 'struct odb_helper' Christian Couder
2017-09-16 8:07 ` [PATCH v6 26/40] odb-helper: add init_object_process() Christian Couder
2017-09-16 8:07 ` [PATCH v6 27/40] Add t0450 to test 'get_direct' mechanism Christian Couder
2017-09-16 8:07 ` [PATCH v6 28/40] Add t0460 to test passing git objects Christian Couder
2017-09-16 8:07 ` [PATCH v6 29/40] odb-helper: add put_object_process() Christian Couder
2017-09-16 8:07 ` [PATCH v6 30/40] Add t0470 to test passing raw objects Christian Couder
2017-09-16 8:07 ` [PATCH v6 31/40] odb-helper: add have_object_process() Christian Couder
2017-09-16 8:07 ` [PATCH v6 32/40] Add t0480 to test "have" capability and raw objects Christian Couder
2017-09-16 8:07 ` [PATCH v6 33/40] external-odb: use 'odb=magic' attribute to mark odb blobs Christian Couder
2017-09-16 8:07 ` [PATCH v6 34/40] Add Documentation/technical/external-odb.txt Christian Couder
2017-09-16 8:07 ` [PATCH v6 35/40] clone: add 'initial' param to write_remote_refs() Christian Couder
2017-09-16 8:07 ` [PATCH v6 36/40] clone: add --initial-refspec option Christian Couder
2017-09-16 8:07 ` [PATCH v6 37/40] clone: disable external odb before initial clone Christian Couder
2017-09-16 8:07 ` [PATCH v6 38/40] Add tests for 'clone --initial-refspec' Christian Couder
2017-09-16 8:07 ` [PATCH v6 39/40] Add t0430 to test cloning using bundles Christian Couder
2017-09-16 8:07 ` [PATCH v6 40/40] Doc/external-odb: explain transfering objects and metadata Christian Couder
2017-10-02 14:18 ` [PATCH v6 00/40] Add initial experimental external ODB support Ben Peart
2017-10-03 6:32 ` Christian Couder
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20170916080731.13925-22-chriscool@tuxfamily.org \
--to=christian.couder@gmail.com \
--cc=Ben.Peart@microsoft.com \
--cc=chriscool@tuxfamily.org \
--cc=e@80x24.org \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=jonathantanmy@google.com \
--cc=larsxschneider@gmail.com \
--cc=mh@glandium.org \
--cc=pclouds@gmail.com \
--cc=peff@peff.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).