From: Christian Couder <christian.couder@gmail.com>
To: git@vger.kernel.org
Cc: Junio C Hamano <gitster@pobox.com>, Jeff King <peff@peff.net>,
Ben Peart <Ben.Peart@microsoft.com>,
Jonathan Tan <jonathantanmy@google.com>,
Nguyen Thai Ngoc Duy <pclouds@gmail.com>,
Mike Hommey <mh@glandium.org>,
Lars Schneider <larsxschneider@gmail.com>,
Eric Wong <e@80x24.org>,
Christian Couder <chriscool@tuxfamily.org>
Subject: [PATCH v5 34/40] external-odb: use 'odb=magic' attribute to mark odb blobs
Date: Thu, 3 Aug 2017 11:19:20 +0200 [thread overview]
Message-ID: <20170803091926.1755-35-chriscool@tuxfamily.org> (raw)
In-Reply-To: <20170803091926.1755-1-chriscool@tuxfamily.org>
To tell which blobs should be sent to the "magic" external odb,
let's require that the blobs be marked using the 'odb=magic'
attribute.
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
external-odb.c | 22 ++++++++++++++++++++--
external-odb.h | 3 ++-
sha1_file.c | 20 +++++++++++++++-----
t/t0400-external-odb.sh | 3 +++
t/t0410-transfer-e-odb.sh | 3 +++
t/t0420-transfer-http-e-odb.sh | 3 +++
t/t0470-read-object-http-e-odb.sh | 3 +++
t/t0480-read-object-have-http-e-odb.sh | 3 +++
8 files changed, 52 insertions(+), 8 deletions(-)
diff --git a/external-odb.c b/external-odb.c
index 084cd32e0b..e103514a46 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -1,6 +1,7 @@
#include "cache.h"
#include "external-odb.h"
#include "odb-helper.h"
+#include "attr.h"
static struct odb_helper *helpers;
static struct odb_helper **helpers_tail = &helpers;
@@ -155,8 +156,23 @@ int external_odb_get_object(const unsigned char *sha1)
return external_odb_do_get_object(sha1);
}
+static int has_odb_attrs(struct odb_helper *o, const char *path)
+{
+ static struct attr_check *check;
+
+ if (!check)
+ check = attr_check_initl("odb", NULL);
+
+ if (!git_check_attr(path, check)) {
+ const char *value = check->items[0].value;
+ return value ? !strcmp(o->name, value) : 0;
+ }
+ return 0;
+}
+
int external_odb_put_object(const void *buf, size_t len,
- const char *type, unsigned char *sha1)
+ const char *type, unsigned char *sha1,
+ const char *path)
{
struct odb_helper *o;
@@ -164,12 +180,14 @@ int external_odb_put_object(const void *buf, size_t len,
return 1;
/* For now accept only blobs */
- if (strcmp(type, "blob"))
+ if (!path || strcmp(type, "blob"))
return 1;
external_odb_init();
for (o = helpers; o; o = o->next) {
+ if (!has_odb_attrs(o, path))
+ continue;
int r = odb_helper_put_object(o, buf, len, type, sha1);
if (r <= 0)
return r;
diff --git a/external-odb.h b/external-odb.h
index 247b131fd5..9bd7856b60 100644
--- a/external-odb.h
+++ b/external-odb.h
@@ -6,6 +6,7 @@ int external_odb_has_object(const unsigned char *sha1);
int external_odb_get_object(const unsigned char *sha1);
int external_odb_get_direct(const unsigned char *sha1);
int external_odb_put_object(const void *buf, size_t len,
- const char *type, unsigned char *sha1);
+ const char *type, unsigned char *sha1,
+ const char *path);
#endif /* EXTERNAL_ODB_H */
diff --git a/sha1_file.c b/sha1_file.c
index fb34f0b18d..75188850da 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -3487,7 +3487,9 @@ static int freshen_packed_object(const unsigned char *sha1)
return 1;
}
-int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
+static int write_sha1_file_with_path(const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1,
+ const char *path)
{
char hdr[32];
int hdrlen = sizeof(hdr);
@@ -3496,13 +3498,19 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign
* it out into .git/objects/??/?{38} file.
*/
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
- if (!external_odb_put_object(buf, len, type, sha1))
+ if (!external_odb_put_object(buf, len, type, sha1, path))
return 0;
if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
return 0;
return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
}
+int write_sha1_file(const void *buf, unsigned long len,
+ const char *type, unsigned char *sha1)
+{
+ write_sha1_file_with_path(buf, len, type, sha1, NULL);
+}
+
int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
unsigned char *sha1, unsigned flags)
{
@@ -3637,7 +3645,8 @@ static int index_mem(unsigned char *sha1, void *buf, size_t size,
}
if (write_object)
- ret = write_sha1_file(buf, size, typename(type), sha1);
+ ret = write_sha1_file_with_path(buf, size, typename(type),
+ sha1, path);
else
ret = hash_sha1_file(buf, size, typename(type), sha1);
if (re_allocated)
@@ -3659,8 +3668,9 @@ static int index_stream_convert_blob(unsigned char *sha1, int fd,
write_object ? safe_crlf : SAFE_CRLF_FALSE);
if (write_object)
- ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
- sha1);
+ ret = write_sha1_file_with_path(sbuf.buf, sbuf.len,
+ typename(OBJ_BLOB),
+ sha1, path);
else
ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
sha1);
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index fa355bd7bb..f2c45a625c 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -73,6 +73,9 @@ test_expect_success 'helper can add objects to alt repo' '
test_expect_success 'commit adds objects to alt repo' '
test_config odb.magic.scriptCommand "$HELPER" &&
+ echo "* odb=magic" >.gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
test_commit three &&
hash3=$(git ls-tree HEAD | grep three.t | cut -f1 | cut -d\ -f3) &&
content=$(cd alt-repo && git show "$hash3") &&
diff --git a/t/t0410-transfer-e-odb.sh b/t/t0410-transfer-e-odb.sh
index 065ec7d759..fd3e37918c 100755
--- a/t/t0410-transfer-e-odb.sh
+++ b/t/t0410-transfer-e-odb.sh
@@ -111,6 +111,9 @@ test_expect_success 'setup other repo and its alternate repo' '
'
test_expect_success 'new blobs are put in first object store' '
+ echo "* odb=magic" >.gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
test_commit one &&
hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\ -f3) &&
content=$(cd alt-repo1 && git show "$hash1") &&
diff --git a/t/t0420-transfer-http-e-odb.sh b/t/t0420-transfer-http-e-odb.sh
index f84fe950ec..d307af0457 100755
--- a/t/t0420-transfer-http-e-odb.sh
+++ b/t/t0420-transfer-http-e-odb.sh
@@ -94,6 +94,9 @@ test_expect_success 'can delete uploaded files' '
FILES_DIR="httpd/www/files"
test_expect_success 'new blobs are transfered to the http server' '
+ echo "* odb=magic" >.gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
test_commit one &&
hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\ -f3) &&
echo "$hash1-4-blob" >expected &&
diff --git a/t/t0470-read-object-http-e-odb.sh b/t/t0470-read-object-http-e-odb.sh
index 774528c04f..d814a43d59 100755
--- a/t/t0470-read-object-http-e-odb.sh
+++ b/t/t0470-read-object-http-e-odb.sh
@@ -62,6 +62,9 @@ test_expect_success 'can delete uploaded files' '
FILES_DIR="httpd/www/files"
test_expect_success 'new blobs are transfered to the http server' '
+ echo "* odb=magic" >.gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
test_commit one &&
hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\ -f3) &&
echo "$hash1-4-blob" >expected &&
diff --git a/t/t0480-read-object-have-http-e-odb.sh b/t/t0480-read-object-have-http-e-odb.sh
index 056a40f2bb..fe1fac5ef3 100755
--- a/t/t0480-read-object-have-http-e-odb.sh
+++ b/t/t0480-read-object-have-http-e-odb.sh
@@ -62,6 +62,9 @@ test_expect_success 'can delete uploaded files' '
FILES_DIR="httpd/www/files"
test_expect_success 'new blobs are transfered to the http server' '
+ echo "* odb=magic" >.gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+ GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
test_commit one &&
hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\ -f3) &&
echo "$hash1-4-blob" >expected &&
--
2.14.0.rc1.52.gf02fb0ddac.dirty
next prev parent reply other threads:[~2017-08-03 9:20 UTC|newest]
Thread overview: 73+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-08-03 9:18 [PATCH v5 00/40] Add initial experimental external ODB support Christian Couder
2017-08-03 9:18 ` [PATCH v5 01/40] builtin/clone: get rid of 'value' strbuf Christian Couder
2017-08-03 9:18 ` [PATCH v5 02/40] t0021/rot13-filter: refactor packet reading functions Christian Couder
2017-08-03 9:18 ` [PATCH v5 03/40] t0021/rot13-filter: improve 'if .. elsif .. else' style Christian Couder
2017-08-03 9:18 ` [PATCH v5 04/40] Add Git/Packet.pm from parts of t0021/rot13-filter.pl Christian Couder
2017-08-03 19:11 ` Junio C Hamano
2017-08-04 6:32 ` Christian Couder
2017-08-03 9:18 ` [PATCH v5 05/40] t0021/rot13-filter: use Git/Packet.pm Christian Couder
2017-08-03 9:18 ` [PATCH v5 06/40] Git/Packet.pm: improve error message Christian Couder
2017-08-03 9:18 ` [PATCH v5 07/40] Git/Packet.pm: add packet_initialize() Christian Couder
2017-08-03 9:18 ` [PATCH v5 08/40] Git/Packet.pm: add capability functions Christian Couder
2017-08-03 19:14 ` Junio C Hamano
2017-08-04 20:34 ` Christian Couder
2017-08-03 9:18 ` [PATCH v5 09/40] sha1_file: prepare for external odbs Christian Couder
2017-08-03 9:18 ` [PATCH v5 10/40] Add initial external odb support Christian Couder
2017-08-03 19:34 ` Junio C Hamano
2017-08-03 20:17 ` Jeff King
2017-09-14 10:14 ` Christian Couder
2017-08-03 9:18 ` [PATCH v5 11/40] odb-helper: add odb_helper_init() to send 'init' instruction Christian Couder
2017-09-10 12:12 ` Lars Schneider
2017-09-14 7:18 ` Christian Couder
2017-08-03 9:18 ` [PATCH v5 12/40] t0400: add 'put_raw_obj' instruction to odb-helper script Christian Couder
2017-09-10 12:12 ` Lars Schneider
2017-09-14 7:09 ` Christian Couder
2017-08-03 9:18 ` [PATCH v5 13/40] external odb: add 'put_raw_obj' support Christian Couder
2017-08-03 19:50 ` Junio C Hamano
2017-09-14 9:17 ` Christian Couder
2017-08-03 9:19 ` [PATCH v5 14/40] external-odb: accept only blobs for now Christian Couder
2017-08-03 19:52 ` Junio C Hamano
2017-09-14 9:59 ` Christian Couder
2017-08-03 9:19 ` [PATCH v5 15/40] t0400: add test for external odb write support Christian Couder
2017-08-03 9:19 ` [PATCH v5 16/40] Add GIT_NO_EXTERNAL_ODB env variable Christian Couder
2017-08-03 9:19 ` [PATCH v5 17/40] Add t0410 to test external ODB transfer Christian Couder
2017-08-03 9:19 ` [PATCH v5 18/40] lib-httpd: pass config file to start_httpd() Christian Couder
2017-08-03 9:19 ` [PATCH v5 19/40] lib-httpd: add upload.sh Christian Couder
2017-08-03 20:07 ` Junio C Hamano
2017-09-14 7:43 ` Christian Couder
2017-08-03 9:19 ` [PATCH v5 20/40] lib-httpd: add list.sh Christian Couder
2017-08-03 9:19 ` [PATCH v5 21/40] lib-httpd: add apache-e-odb.conf Christian Couder
2017-08-03 9:19 ` [PATCH v5 22/40] odb-helper: add odb_helper_get_raw_object() Christian Couder
2017-08-03 9:19 ` [PATCH v5 23/40] pack-objects: don't pack objects in external odbs Christian Couder
2017-08-03 9:19 ` [PATCH v5 24/40] Add t0420 to test transfer to HTTP external odb Christian Couder
2017-08-03 9:19 ` [PATCH v5 25/40] external-odb: add 'get_direct' support Christian Couder
2017-08-03 21:40 ` Junio C Hamano
2017-09-14 8:39 ` Christian Couder
2017-09-14 18:19 ` Jonathan Tan
2017-09-15 11:24 ` Christian Couder
2017-09-15 20:54 ` Jonathan Tan
2017-08-03 9:19 ` [PATCH v5 26/40] odb-helper: add 'script_mode' to 'struct odb_helper' Christian Couder
2017-08-03 9:19 ` [PATCH v5 27/40] odb-helper: add init_object_process() Christian Couder
2017-08-03 9:19 ` [PATCH v5 28/40] Add t0450 to test 'get_direct' mechanism Christian Couder
2017-08-03 9:19 ` [PATCH v5 29/40] Add t0460 to test passing git objects Christian Couder
2017-08-03 9:19 ` [PATCH v5 30/40] odb-helper: add put_object_process() Christian Couder
2017-08-03 9:19 ` [PATCH v5 31/40] Add t0470 to test passing raw objects Christian Couder
2017-08-03 9:19 ` [PATCH v5 32/40] odb-helper: add have_object_process() Christian Couder
2017-08-03 9:19 ` [PATCH v5 33/40] Add t0480 to test "have" capability and raw objects Christian Couder
2017-08-03 9:19 ` Christian Couder [this message]
2017-08-03 9:19 ` [PATCH v5 35/40] Add Documentation/technical/external-odb.txt Christian Couder
2017-08-03 18:38 ` Stefan Beller
2017-08-25 6:14 ` Christian Couder
2017-08-25 21:23 ` Jonathan Tan
2017-08-29 9:37 ` Christian Couder
2017-08-28 18:59 ` Ben Peart
2017-08-29 15:43 ` Christian Couder
2017-08-30 12:50 ` Ben Peart
2017-08-30 14:15 ` Christian Couder
2017-08-03 9:19 ` [PATCH v5 36/40] clone: add 'initial' param to write_remote_refs() Christian Couder
2017-08-03 9:19 ` [PATCH v5 37/40] clone: add --initial-refspec option Christian Couder
2017-08-03 9:19 ` [PATCH v5 38/40] clone: disable external odb before initial clone Christian Couder
2017-08-03 9:19 ` [PATCH v5 39/40] Add tests for 'clone --initial-refspec' Christian Couder
2017-08-03 9:19 ` [PATCH v5 40/40] Add t0430 to test cloning using bundles Christian Couder
2017-09-10 12:30 ` [PATCH v5 00/40] Add initial experimental external ODB support Lars Schneider
2017-09-14 7:02 ` Christian Couder
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20170803091926.1755-35-chriscool@tuxfamily.org \
--to=christian.couder@gmail.com \
--cc=Ben.Peart@microsoft.com \
--cc=chriscool@tuxfamily.org \
--cc=e@80x24.org \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=jonathantanmy@google.com \
--cc=larsxschneider@gmail.com \
--cc=mh@glandium.org \
--cc=pclouds@gmail.com \
--cc=peff@peff.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).