From: Matthew DeVore <matvore@google.com>
To: git@vger.kernel.org
Cc: Matthew DeVore <matvore@google.com>,
jeffhost@microsoft.com, peff@peff.net, stefanbeller@gmail.com,
jonathantanmy@google.com
Subject: [PATCH 2/5] list-objects-filter: implement filter only:commits
Date: Thu, 9 Aug 2018 15:45:01 -0700 [thread overview]
Message-ID: <71619cbd1004d7acf991011fc0c941d4a5716079.1533854545.git.matvore@google.com> (raw)
In-Reply-To: <cover.1533854545.git.matvore@google.com>
Teach list-objects the "only:commits" filter which allows for filtering
out all non-commit and non-annotated tag objects (unless other objects
are explicitly specified by the user). The purpose of this patch is to
allow smaller partial clones.
The name of this filter - only:commits - is a bit inaccurate because it
still allows annotated tags to pass through. I chose it because it was
the only concise name I could think of that was pretty descriptive. I
considered and decided against "tree:none" because the code and
documentation for filters seems to lack the concept of "you're filtering
this, so we'll implicitly filter all referents of this." So "tree:none"
is vague, since some may think it filters blobs too, while some may not.
"only:commits" is specific and makes it easier to match it to a
potential use case.
Signed-off-by: Matthew DeVore <matvore@google.com>
---
Documentation/rev-list-options.txt | 2 ++
list-objects-filter-options.c | 4 +++
list-objects-filter-options.h | 1 +
list-objects-filter.c | 43 ++++++++++++++++++--------
t/t5317-pack-objects-filter-objects.sh | 30 ++++++++++++++++++
t/t6112-rev-list-filters-objects.sh | 13 ++++++++
6 files changed, 80 insertions(+), 13 deletions(-)
diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt
index 7b273635d..3a60a490a 100644
--- a/Documentation/rev-list-options.txt
+++ b/Documentation/rev-list-options.txt
@@ -743,6 +743,8 @@ specification contained in <path>.
A debug option to help with future "partial clone" development.
This option specifies how missing objects are handled.
+
+The form '--filter=only:commits' omits all blobs and trees.
++
The form '--missing=error' requests that rev-list stop with an error if
a missing object is encountered. This is the default action.
+
diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c
index c0e2bd6a0..aaaaae508 100644
--- a/list-objects-filter-options.c
+++ b/list-objects-filter-options.c
@@ -69,6 +69,10 @@ static int gently_parse_list_objects_filter(
filter_options->choice = LOFC_SPARSE_PATH;
filter_options->sparse_path_value = strdup(v0);
return 0;
+
+ } else if (!strcmp(arg, "only:commits")) {
+ filter_options->choice = LOFC_ONLY_COMMITS;
+ return 0;
}
if (errbuf) {
diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h
index 0000a61f8..a68df42c8 100644
--- a/list-objects-filter-options.h
+++ b/list-objects-filter-options.h
@@ -12,6 +12,7 @@ enum list_objects_filter_choice {
LOFC_BLOB_LIMIT,
LOFC_SPARSE_OID,
LOFC_SPARSE_PATH,
+ LOFC_ONLY_COMMITS,
LOFC__COUNT /* must be last */
};
diff --git a/list-objects-filter.c b/list-objects-filter.c
index a0ba78b20..f0a064b4b 100644
--- a/list-objects-filter.c
+++ b/list-objects-filter.c
@@ -26,38 +26,39 @@
#define FILTER_SHOWN_BUT_REVISIT (1<<21)
/*
- * A filter for list-objects to omit ALL blobs from the traversal.
- * And to OPTIONALLY collect a list of the omitted OIDs.
+ * A filter for list-objects to omit ALL blobs from the traversal, and possibly
+ * trees as well.
+ * Can OPTIONALLY collect a list of the omitted OIDs.
*/
-struct filter_blobs_none_data {
+struct filter_none_of_type_data {
+ unsigned omit_trees : 1;
struct oidset *omits;
};
-static enum list_objects_filter_result filter_blobs_none(
+static enum list_objects_filter_result filter_none_of_type(
enum list_objects_filter_situation filter_situation,
struct object *obj,
const char *pathname,
const char *filename,
void *filter_data_)
{
- struct filter_blobs_none_data *filter_data = filter_data_;
+ struct filter_none_of_type_data *filter_data = filter_data_;
switch (filter_situation) {
default:
die("unknown filter_situation");
return LOFR_ZERO;
- case LOFS_BEGIN_TREE:
- assert(obj->type == OBJ_TREE);
- /* always include all tree objects */
- return LOFR_MARK_SEEN | LOFR_DO_SHOW;
-
case LOFS_END_TREE:
assert(obj->type == OBJ_TREE);
return LOFR_ZERO;
+ case LOFS_BEGIN_TREE:
+ assert(obj->type == OBJ_TREE);
+ if (!filter_data->omit_trees)
+ return LOFR_MARK_SEEN | LOFR_DO_SHOW;
+
case LOFS_BLOB:
- assert(obj->type == OBJ_BLOB);
assert((obj->flags & SEEN) == 0);
if (filter_data->omits)
@@ -72,10 +73,25 @@ static void *filter_blobs_none__init(
filter_object_fn *filter_fn,
filter_free_fn *filter_free_fn)
{
- struct filter_blobs_none_data *d = xcalloc(1, sizeof(*d));
+ struct filter_none_of_type_data *d = xcalloc(1, sizeof(*d));
+ d->omits = omitted;
+
+ *filter_fn = filter_none_of_type;
+ *filter_free_fn = free;
+ return d;
+}
+
+static void* filter_only_commits__init(
+ struct oidset *omitted,
+ struct list_objects_filter_options *filter_options,
+ filter_object_fn *filter_fn,
+ filter_free_fn *filter_free_fn)
+{
+ struct filter_none_of_type_data *d = xcalloc(1, sizeof(*d));
+ d->omit_trees = 1;
d->omits = omitted;
- *filter_fn = filter_blobs_none;
+ *filter_fn = filter_none_of_type;
*filter_free_fn = free;
return d;
}
@@ -376,6 +392,7 @@ static filter_init_fn s_filters[] = {
filter_blobs_limit__init,
filter_sparse_oid__init,
filter_sparse_path__init,
+ filter_only_commits__init,
};
void *list_objects_filter__init(
diff --git a/t/t5317-pack-objects-filter-objects.sh b/t/t5317-pack-objects-filter-objects.sh
index 6710c8bc8..600d153f9 100755
--- a/t/t5317-pack-objects-filter-objects.sh
+++ b/t/t5317-pack-objects-filter-objects.sh
@@ -59,6 +59,36 @@ test_expect_success 'verify normal and blob:none packfiles have same commits/tre
test_cmp observed expected
'
+test_expect_success 'setup for tests of only:commits' '
+ mkdir r1/subtree &&
+ echo "This is a file in a subtree" > r1/subtree/file &&
+ git -C r1 add subtree/file &&
+ git -C r1 commit -m subtree
+'
+
+test_expect_success 'verify only:commits packfile has no blobs or trees' '
+ git -C r1 pack-objects --rev --stdout --filter=only:commits >commitsonly.pack <<-EOF &&
+ HEAD
+ EOF
+ git -C r1 index-pack ../commitsonly.pack &&
+ git -C r1 verify-pack -v ../commitsonly.pack \
+ | grep -E "tree|blob" \
+ | sort >observed &&
+ test_line_count = 0 observed
+'
+
+test_expect_success 'grab tree directly when using only:commits' '
+ # We should get the tree specified directly but not its blobs or subtrees.
+ git -C r1 pack-objects --rev --stdout --filter=only:commits >commitsonly.pack <<-EOF &&
+ HEAD:
+ EOF
+ git -C r1 index-pack ../commitsonly.pack &&
+ git -C r1 verify-pack -v ../commitsonly.pack \
+ | grep -E "tree|blob" \
+ | sort >observed &&
+ test_line_count = 1 observed
+'
+
# Test blob:limit=<n>[kmg] filter.
# We boundary test around the size parameter. The filter is strictly less than
# the value, so size 500 and 1000 should have the same results, but 1001 should
diff --git a/t/t6112-rev-list-filters-objects.sh b/t/t6112-rev-list-filters-objects.sh
index 0a37dd5f9..6dbd9477c 100755
--- a/t/t6112-rev-list-filters-objects.sh
+++ b/t/t6112-rev-list-filters-objects.sh
@@ -196,6 +196,19 @@ test_expect_success 'verify sparse:oid=oid-ish omits top-level files' '
test_cmp observed expected
'
+# Test only:commits filter.
+
+test_expect_success 'verify only:commits includes trees in "filtered" output' '
+ git -C r3 rev-list HEAD --quiet --objects --filter-print-omitted --filter=only:commits \
+ | awk -f print_1.awk \
+ | sed s/~// \
+ | xargs -n1 git -C r3 cat-file -t \
+ | sort -u >filtered_types &&
+ printf "blob\ntree\n" > expected &&
+ test_cmp filtered_types expected
+'
+
+
# Delete some loose objects and use rev-list, but WITHOUT any filtering.
# This models previously omitted objects that we did not receive.
--
2.18.0.597.ga71716f1ad-goog
next prev parent reply other threads:[~2018-08-09 22:45 UTC|newest]
Thread overview: 151+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-08-09 22:44 [RFC PATCH 0/5] filter: support for excluding all trees and blobs Matthew DeVore
2018-08-09 22:45 ` [PATCH 1/5] revision: invert meaning of the USER_GIVEN flag Matthew DeVore
2018-08-10 18:43 ` Jonathan Tan
2018-08-09 22:45 ` Matthew DeVore [this message]
2018-08-10 0:14 ` [PATCH 2/5] list-objects-filter: implement filter only:commits Jonathan Tan
2018-08-09 22:45 ` [PATCH 3/5] list-objects: store common func args in struct Matthew DeVore
2018-08-09 22:45 ` [PATCH 4/5] list-objects: refactor to process_tree_contents Matthew DeVore
2018-08-09 22:45 ` [PATCH 5/5] rev-list: handle missing tree objects properly Matthew DeVore
2018-08-10 0:24 ` Jonathan Tan
2018-08-10 19:03 ` [RFC PATCH 0/5] filter: support for excluding all trees and blobs Jonathan Tan
2018-08-10 23:06 ` [PATCH v2 " Matthew DeVore
2018-08-10 23:06 ` [PATCH v2 1/5] list-objects: store common func args in struct Matthew DeVore
2018-08-10 23:06 ` [PATCH v2 2/5] list-objects: refactor to process_tree_contents Matthew DeVore
2018-08-10 23:06 ` [PATCH v2 3/5] rev-list: handle missing tree objects properly Matthew DeVore
2018-08-13 18:20 ` Jonathan Tan
2018-08-14 0:22 ` Matthew DeVore
2018-08-14 16:03 ` Jonathan Tan
2018-08-10 23:06 ` [PATCH v2 4/5] revision: mark non-user-given objects instead Matthew DeVore
2018-08-10 23:06 ` [PATCH v2 5/5] list-objects-filter: implement filter tree:none Matthew DeVore
2018-08-13 16:38 ` Jeff Hostetler
2018-08-14 0:57 ` Matthew DeVore
2018-08-13 18:29 ` Jonathan Tan
2018-08-14 0:55 ` Matthew DeVore
2018-08-13 18:14 ` [PATCH v3 0/5] filter: support for excluding all trees and blobs Matthew DeVore
2018-08-13 18:14 ` [PATCH v3 1/5] list-objects: store common func args in struct Matthew DeVore
2018-08-13 18:14 ` [PATCH v3 2/5] list-objects: refactor to process_tree_contents Matthew DeVore
2018-08-13 18:14 ` [PATCH v3 3/5] rev-list: handle missing tree objects properly Matthew DeVore
2018-08-13 18:14 ` [PATCH v3 4/5] revision: mark non-user-given objects instead Matthew DeVore
2018-08-13 18:14 ` [PATCH v3 5/5] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-08-14 15:13 ` Jeff Hostetler
2018-08-14 17:25 ` Matthew DeVore
2018-10-03 19:00 ` Matthew DeVore
2018-08-14 17:28 ` [PATCH v4 0/6] filter: support for excluding all trees and blobs Matthew DeVore
2018-08-14 17:28 ` [PATCH v4 1/6] list-objects: store common func args in struct Matthew DeVore
2018-08-14 17:28 ` [PATCH v4 2/6] list-objects: refactor to process_tree_contents Matthew DeVore
2018-08-14 17:28 ` [PATCH v4 3/6] list-objects: always parse trees gently Matthew DeVore
2018-08-14 17:28 ` [PATCH v4 4/6] rev-list: handle missing tree objects properly Matthew DeVore
2018-08-14 18:06 ` Jonathan Tan
2018-08-14 22:43 ` Matthew DeVore
2018-08-14 22:56 ` Jonathan Tan
2018-08-14 23:14 ` Jonathan Tan
2018-08-14 17:28 ` [PATCH v4 5/6] revision: mark non-user-given objects instead Matthew DeVore
2018-08-14 17:28 ` [PATCH v4 6/6] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-08-14 18:18 ` Jonathan Tan
2018-08-14 20:00 ` Matthew DeVore
2018-08-14 20:19 ` Jonathan Tan
2018-08-14 20:55 ` Junio C Hamano
2018-08-14 23:30 ` Matthew DeVore
2018-08-15 16:14 ` Junio C Hamano
2018-08-15 16:37 ` Matthew DeVore
2018-08-14 20:01 ` Jeff King
2018-08-14 23:55 ` Matthew DeVore
2018-08-15 1:22 ` Jeff King
2018-08-15 16:17 ` Junio C Hamano
2018-08-15 17:54 ` Matthew DeVore
2018-08-15 0:22 ` [PATCH v5 0/6] filter: support for excluding all trees and blobs Matthew DeVore
2018-08-15 0:22 ` [PATCH v5 1/6] list-objects: store common func args in struct Matthew DeVore
2018-08-15 0:22 ` [PATCH v5 2/6] list-objects: refactor to process_tree_contents Matthew DeVore
2018-08-15 0:22 ` [PATCH v5 3/6] list-objects: always parse trees gently Matthew DeVore
2018-08-15 0:22 ` [PATCH v5 4/6] rev-list: handle missing tree objects properly Matthew DeVore
2018-08-15 0:22 ` [PATCH v5 5/6] revision: mark non-user-given objects instead Matthew DeVore
2018-08-15 0:22 ` [PATCH v5 6/6] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-08-15 23:19 ` [PATCH v6 0/6] filter: support for excluding all trees and blobs Matthew DeVore
2018-08-15 23:19 ` [PATCH v6 1/6] list-objects: store common func args in struct Matthew DeVore
2018-08-15 23:19 ` [PATCH v6 2/6] list-objects: refactor to process_tree_contents Matthew DeVore
2018-08-15 23:19 ` [PATCH v6 3/6] list-objects: always parse trees gently Matthew DeVore
2018-08-15 23:19 ` [PATCH v6 4/6] rev-list: handle missing tree objects properly Matthew DeVore
2018-08-15 23:19 ` [PATCH v6 5/6] revision: mark non-user-given objects instead Matthew DeVore
2018-08-15 23:19 ` [PATCH v6 6/6] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-08-17 21:42 ` Stefan Beller
2018-08-17 22:19 ` Matthew DeVore
2018-08-17 22:28 ` Stefan Beller
2018-08-20 23:30 ` Matthew DeVore
2018-08-21 0:29 ` Stefan Beller
2018-08-21 21:46 ` Junio C Hamano
2018-08-22 18:00 ` Stefan Beller
2018-08-18 16:17 ` Duy Nguyen
2018-08-20 13:04 ` Matthew DeVore
2018-08-20 18:38 ` Stefan Beller
2018-08-20 23:20 ` Matthew DeVore
2018-08-21 0:36 ` Stefan Beller
2018-08-21 15:50 ` Duy Nguyen
2018-09-04 18:05 ` [PATCH v7 0/7] filter: support for excluding all trees and blobs Matthew DeVore
2018-09-04 18:05 ` [PATCH v7 1/7] list-objects: store common func args in struct Matthew DeVore
2018-09-04 18:05 ` [PATCH v7 2/7] list-objects: refactor to process_tree_contents Matthew DeVore
2018-09-04 18:05 ` [PATCH v7 3/7] list-objects: always parse trees gently Matthew DeVore
2018-09-04 18:05 ` [PATCH v7 4/7] rev-list: handle missing tree objects properly Matthew DeVore
2018-09-04 18:05 ` [PATCH v7 5/7] revision: mark non-user-given objects instead Matthew DeVore
2018-09-04 20:31 ` Junio C Hamano
2018-09-05 18:00 ` Matthew DeVore
2018-09-04 18:05 ` [PATCH v7 6/7] list-objects-filter: use BUG rather than die Matthew DeVore
2018-09-04 20:32 ` Junio C Hamano
2018-09-04 18:05 ` [PATCH v7 7/7] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-09-04 20:44 ` Junio C Hamano
2018-09-06 0:08 ` Matthew DeVore
2018-09-04 18:41 ` [PATCH v7 0/7] filter: support for excluding all trees and blobs Stefan Beller
2018-09-14 0:55 ` [PATCH v8 " Matthew DeVore
2018-09-14 0:55 ` [PATCH v8 1/7] list-objects: store common func args in struct Matthew DeVore
2018-09-14 0:55 ` [PATCH v8 2/7] list-objects: refactor to process_tree_contents Matthew DeVore
2018-09-14 0:55 ` [PATCH v8 3/7] list-objects: always parse trees gently Matthew DeVore
2018-09-14 0:55 ` [PATCH v8 4/7] rev-list: handle missing tree objects properly Matthew DeVore
2018-09-14 0:55 ` [PATCH v8 5/7] revision: mark non-user-given objects instead Matthew DeVore
2018-09-14 17:23 ` Junio C Hamano
2018-09-14 20:08 ` Matthew DeVore
2018-09-14 0:55 ` [PATCH v8 6/7] list-objects-filter: use BUG rather than die Matthew DeVore
2018-09-14 0:55 ` [PATCH v8 7/7] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-09-14 17:39 ` Junio C Hamano
2018-09-14 17:47 ` Junio C Hamano
2018-09-15 0:41 ` Matthew DeVore
2018-09-21 20:31 ` [PATCH v9 0/8] filter: support for excluding all trees and blobs Matthew DeVore
2018-09-21 20:31 ` [PATCH v9 1/8] list-objects: store common func args in struct Matthew DeVore
2018-09-21 20:31 ` [PATCH v9 2/8] list-objects: refactor to process_tree_contents Matthew DeVore
2018-09-21 20:31 ` [PATCH v9 3/8] list-objects: always parse trees gently Matthew DeVore
2018-09-21 20:32 ` [PATCH v9 4/8] rev-list: handle missing tree objects properly Matthew DeVore
2018-09-21 20:32 ` [PATCH v9 5/8] revision: mark non-user-given objects instead Matthew DeVore
2018-09-21 20:32 ` [PATCH v9 6/8] list-objects-filter: use BUG rather than die Matthew DeVore
2018-09-21 20:32 ` [PATCH v9 7/8] list-objects-filter-options: do not over-strbuf_init Matthew DeVore
2018-09-21 20:32 ` [PATCH v9 8/8] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 0/8] filter: support for excluding all trees and blobs Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 1/8] list-objects: store common func args in struct Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 2/8] list-objects: refactor to process_tree_contents Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 3/8] list-objects: always parse trees gently Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 4/8] rev-list: handle missing tree objects properly Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 5/8] revision: mark non-user-given objects instead Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 6/8] list-objects-filter: use BUG rather than die Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 7/8] list-objects-filter-options: do not over-strbuf_init Matthew DeVore
2018-10-03 19:52 ` [PATCH v10 8/8] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-10-03 23:08 ` [PATCH v10 0/8] filter: support for excluding all trees and blobs Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 " Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 1/8] list-objects: store common func args in struct Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 2/8] list-objects: refactor to process_tree_contents Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 3/8] list-objects: always parse trees gently Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 4/8] rev-list: handle missing tree objects properly Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 5/8] revision: mark non-user-given objects instead Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 6/8] list-objects-filter: use BUG rather than die Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 7/8] list-objects-filter-options: do not over-strbuf_init Matthew DeVore
2018-10-05 21:31 ` [PATCH v11 8/8] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-10-07 0:10 ` Junio C Hamano
2018-10-08 17:23 ` Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 0/8] filter: support for excluding all trees and blobs Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 1/8] list-objects: store common func args in struct Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 2/8] list-objects: refactor to process_tree_contents Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 3/8] list-objects: always parse trees gently Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 4/8] rev-list: handle missing tree objects properly Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 5/8] revision: mark non-user-given objects instead Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 6/8] list-objects-filter: use BUG rather than die Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 7/8] list-objects-filter-options: do not over-strbuf_init Matthew DeVore
2018-10-12 20:01 ` [PATCH v12 8/8] list-objects-filter: implement filter tree:0 Matthew DeVore
2018-10-15 2:37 ` [PATCH v12 0/8] filter: support for excluding all trees and blobs Junio C Hamano
2018-10-15 3:42 ` Junio C Hamano
2018-10-16 15:00 ` Matthew DeVore
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=71619cbd1004d7acf991011fc0c941d4a5716079.1533854545.git.matvore@google.com \
--to=matvore@google.com \
--cc=git@vger.kernel.org \
--cc=jeffhost@microsoft.com \
--cc=jonathantanmy@google.com \
--cc=peff@peff.net \
--cc=stefanbeller@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).