From: Jeff Hostetler <git@jeffhostetler.com>
To: git@vger.kernel.org
Cc: gitster@pobox.com, peff@peff.net, jonathantanmy@google.com,
Jeff Hostetler <jeffhost@microsoft.com>
Subject: [PATCH 05/13] list-objects-filter-blobs-limit: add large blob filtering
Date: Tue, 24 Oct 2017 18:53:24 +0000 [thread overview]
Message-ID: <20171024185332.57261-6-git@jeffhostetler.com> (raw)
In-Reply-To: <20171024185332.57261-1-git@jeffhostetler.com>
From: Jeff Hostetler <jeffhost@microsoft.com>
Create a filter for traverse_commit_list_worker() to omit blobs
larger than a requested size from the result, but always include
".git*" special files.
Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>
---
Makefile | 1 +
list-objects-filter-blobs-limit.c | 146 ++++++++++++++++++++++++++++++++++++++
list-objects-filter-blobs-limit.h | 18 +++++
3 files changed, 165 insertions(+)
create mode 100644 list-objects-filter-blobs-limit.c
create mode 100644 list-objects-filter-blobs-limit.h
diff --git a/Makefile b/Makefile
index 7e9d1f4..0fdeabb 100644
--- a/Makefile
+++ b/Makefile
@@ -807,6 +807,7 @@ LIB_OBJS += levenshtein.o
LIB_OBJS += line-log.o
LIB_OBJS += line-range.o
LIB_OBJS += list-objects.o
+LIB_OBJS += list-objects-filter-blobs-limit.o
LIB_OBJS += list-objects-filter-blobs-none.o
LIB_OBJS += list-objects-filter-map.o
LIB_OBJS += ll-merge.o
diff --git a/list-objects-filter-blobs-limit.c b/list-objects-filter-blobs-limit.c
new file mode 100644
index 0000000..f68963d
--- /dev/null
+++ b/list-objects-filter-blobs-limit.c
@@ -0,0 +1,146 @@
+#include "cache.h"
+#include "dir.h"
+#include "tag.h"
+#include "commit.h"
+#include "tree.h"
+#include "blob.h"
+#include "diff.h"
+#include "tree-walk.h"
+#include "revision.h"
+#include "list-objects.h"
+#include "list-objects-filter-blobs-limit.h"
+
+#define DEFAULT_MAP_SIZE (16*1024)
+
+/*
+ * A filter for list-objects to omit large blobs,
+ * but always include ".git*" special files.
+ * And to OPTIONALLY collect a list of the omitted OIDs.
+ */
+struct filter_blobs_limit_data {
+ struct oidmap *omits;
+ unsigned long max_bytes;
+};
+
+static list_objects_filter_result filter_blobs_limit(
+ list_objects_filter_type filter_type,
+ struct object *obj,
+ const char *pathname,
+ const char *filename,
+ void *filter_data_)
+{
+ struct filter_blobs_limit_data *filter_data = filter_data_;
+ struct list_objects_filter_data_entry *entry;
+ unsigned long object_length;
+ enum object_type t;
+ int is_special_filename;
+
+ switch (filter_type) {
+ default:
+ die("unkown filter_type");
+ return LOFR_ZERO;
+
+ case LOFT_BEGIN_TREE:
+ assert(obj->type == OBJ_TREE);
+ /* always include all tree objects */
+ return LOFR_MARK_SEEN | LOFR_SHOW;
+
+ case LOFT_END_TREE:
+ assert(obj->type == OBJ_TREE);
+ return LOFR_ZERO;
+
+ case LOFT_BLOB:
+ assert(obj->type == OBJ_BLOB);
+ assert((obj->flags & SEEN) == 0);
+
+ is_special_filename = ((strncmp(filename, ".git", 4) == 0) &&
+ filename[4]);
+
+ /*
+ * If we are keeping a list of the omitted objects
+ * for the caller *AND* we previously "provisionally"
+ * omitted this object (because of size) *AND* it now
+ * has a special filename, make it not-omitted.
+ * Otherwise, continue to provisionally omit it.
+ */
+ if (filter_data->omits &&
+ oidmap_get(filter_data->omits, &obj->oid)) {
+ if (!is_special_filename)
+ return LOFR_ZERO;
+ entry = oidmap_remove(filter_data->omits, &obj->oid);
+ free(entry);
+ return LOFR_MARK_SEEN | LOFR_SHOW;
+ }
+
+ /*
+ * If filename matches ".git*", always include it (regardless
+ * of size). (This may include blobs that we do not have
+ * locally.)
+ */
+ if (is_special_filename)
+ return LOFR_MARK_SEEN | LOFR_SHOW;
+
+ t = sha1_object_info(obj->oid.hash, &object_length);
+ if (t != OBJ_BLOB) { /* probably OBJ_NONE */
+ /*
+ * We DO NOT have the blob locally, so we cannot
+ * apply the size filter criteria. Be conservative
+ * and force show it (and let the caller deal with
+ * the ambiguity). (This matches the behavior above
+ * when the special filename matches.)
+ */
+ return LOFR_MARK_SEEN | LOFR_SHOW;
+ }
+
+ if (object_length < filter_data->max_bytes)
+ return LOFR_MARK_SEEN | LOFR_SHOW;
+
+ /*
+ * Provisionally omit it. We've already established
+ * that this blob is too big and doesn't have a special
+ * filename, so we *WANT* to omit it. However, there
+ * may be a special file elsewhere in the tree that
+ * references this same blob, so we cannot reject it
+ * just yet. Leave the LOFR_ bits unset so that *IF*
+ * the blob appears again in the traversal, we will
+ * be asked again.
+ *
+ * If we are keeping a list of the ommitted objects,
+ * provisionally add it to the list.
+ */
+
+ if (filter_data->omits)
+ list_objects_filter_map_insert(filter_data->omits,
+ &obj->oid, pathname,
+ obj->type);
+
+ return LOFR_ZERO;
+ }
+}
+
+void traverse_commit_list__blobs_limit(
+ struct rev_info *revs,
+ show_commit_fn show_commit,
+ show_object_fn show_object,
+ list_objects_filter_map_foreach_cb print_omitted_object,
+ void *ctx_data,
+ unsigned long large_byte_limit)
+{
+ struct filter_blobs_limit_data d;
+
+ memset(&d, 0, sizeof(d));
+ if (print_omitted_object) {
+ d.omits = xcalloc(1, sizeof(*d.omits));
+ oidmap_init(d.omits, DEFAULT_MAP_SIZE);
+ }
+ d.max_bytes = large_byte_limit;
+
+ traverse_commit_list_worker(revs, show_commit, show_object, ctx_data,
+ filter_blobs_limit, &d);
+
+ if (print_omitted_object) {
+ list_objects_filter_map_foreach(d.omits, print_omitted_object,
+ ctx_data);
+ oidmap_free(d.omits, 1);
+ }
+}
diff --git a/list-objects-filter-blobs-limit.h b/list-objects-filter-blobs-limit.h
new file mode 100644
index 0000000..ea05088
--- /dev/null
+++ b/list-objects-filter-blobs-limit.h
@@ -0,0 +1,18 @@
+#ifndef LIST_OBJECTS_FILTER_BLOBS_LIMIT_H
+#define LIST_OBJECTS_FILTER_BLOBS_LIMIT_H
+
+#include "list-objects-filter-map.h"
+
+/*
+ * A filter for list-objects to omit large blobs,
+ * but always include ".git*" special files.
+ */
+void traverse_commit_list__blobs_limit(
+ struct rev_info *revs,
+ show_commit_fn show_commit,
+ show_object_fn show_object,
+ list_objects_filter_map_foreach_cb print_omitted_object,
+ void *ctx_data,
+ unsigned long large_byte_limit);
+
+#endif /* LIST_OBJECTS_FILTER_BLOBS_LIMIT_H */
--
2.9.3
next prev parent reply other threads:[~2017-10-24 18:54 UTC|newest]
Thread overview: 37+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-10-24 18:53 [PATCH 00/13] WIP Partial clone part 1: object filtering Jeff Hostetler
2017-10-24 18:53 ` [PATCH 01/13] dir: allow exclusions from blob in addition to file Jeff Hostetler
2017-10-25 4:05 ` Eric Sunshine
2017-10-25 6:43 ` Junio C Hamano
2017-10-25 14:54 ` Jeff Hostetler
2017-10-26 3:47 ` Junio C Hamano
2017-10-26 18:11 ` Jeff Hostetler
2017-10-24 18:53 ` [PATCH 02/13] list-objects-filter-map: extend oidmap to collect omitted objects Jeff Hostetler
2017-10-25 7:10 ` Junio C Hamano
2017-10-25 19:22 ` Jeff Hostetler
2017-10-26 4:12 ` Junio C Hamano
2017-10-24 18:53 ` [PATCH 03/13] list-objects: filter objects in traverse_commit_list Jeff Hostetler
2017-10-25 4:05 ` Jonathan Tan
2017-10-25 19:25 ` Jeff Hostetler
2017-10-24 18:53 ` [PATCH 04/13] list-objects-filter-blobs-none: add filter to omit all blobs Jeff Hostetler
2017-10-24 18:53 ` Jeff Hostetler [this message]
2017-10-24 18:53 ` [PATCH 06/13] list-objects-filter-sparse: add sparse filter Jeff Hostetler
2017-10-24 18:53 ` [PATCH 07/13] list-objects-filter-options: common argument parsing Jeff Hostetler
2017-10-25 4:14 ` Jonathan Tan
2017-10-25 19:28 ` Jeff Hostetler
2017-10-24 18:53 ` [PATCH 08/13] list-objects: add traverse_commit_list_filtered method Jeff Hostetler
2017-10-25 4:24 ` Jonathan Tan
2017-10-25 19:29 ` Jeff Hostetler
2017-10-24 18:53 ` [PATCH 09/13] extension.partialclone: introduce partial clone extension Jeff Hostetler
2017-10-24 18:53 ` [PATCH 10/13] rev-list: add list-objects filtering support Jeff Hostetler
2017-10-25 4:41 ` Jonathan Tan
2017-10-25 19:37 ` Jeff Hostetler
2017-10-24 18:53 ` [PATCH 11/13] t6112: rev-list object filtering test Jeff Hostetler
2017-10-24 18:53 ` [PATCH 12/13] pack-objects: add list-objects filtering Jeff Hostetler
2017-10-24 18:53 ` [PATCH 13/13] t5317: pack-objects object filtering test Jeff Hostetler
2017-10-25 4:57 ` [PATCH 00/13] WIP Partial clone part 1: object filtering Jonathan Tan
2017-10-25 5:00 ` Junio C Hamano
2017-10-25 6:46 ` Jonathan Tan
2017-10-25 15:39 ` Jeff Hostetler
2017-10-26 2:09 ` Junio C Hamano
2017-10-26 2:01 ` Junio C Hamano
2017-10-30 22:27 ` Jonathan Nieder
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20171024185332.57261-6-git@jeffhostetler.com \
--to=git@jeffhostetler.com \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=jeffhost@microsoft.com \
--cc=jonathantanmy@google.com \
--cc=peff@peff.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).