From: Jameson Miller <jamill@microsoft.com>
To: Jameson Miller <jamill@microsoft.com>
Cc: "git@vger.kernel.org" <git@vger.kernel.org>,
"gitster@pobox.com" <gitster@pobox.com>,
"jonathantanmy@google.com" <jonathantanmy@google.com>,
"pclouds@gmail.com" <pclouds@gmail.com>,
"peartben@gmail.com" <peartben@gmail.com>,
"peff@peff.net" <peff@peff.net>,
"sbeller@google.com" <sbeller@google.com>
Subject: [PATCH v5 8/8] block alloc: add validations around cache_entry lifecyle
Date: Thu, 28 Jun 2018 14:00:16 +0000 [thread overview]
Message-ID: <20180628135932.225288-9-jamill@microsoft.com> (raw)
In-Reply-To: <20180628135932.225288-1-jamill@microsoft.com>
Add an option (controlled by an environment variable) perform extra
validations on mem_pool allocated cache entries. When set:
1) Invalidate cache_entry memory when discarding cache_entry.
2) When discarding index_state struct, verify that all cache_entries
were allocated from expected mem_pool.
3) When discarding mem_pools, invalidate mem_pool memory.
This should provide extra checks that mem_pools and their allocated
cache_entries are being used as expected.
Signed-off-by: Jameson Miller <jamill@microsoft.com>
---
cache.h | 6 ++++++
git.c | 3 +++
mem-pool.c | 6 +++++-
mem-pool.h | 2 +-
read-cache.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++--
5 files changed, 68 insertions(+), 4 deletions(-)
diff --git a/cache.h b/cache.h
index 8fd89ae51f..b3faef3efc 100644
--- a/cache.h
+++ b/cache.h
@@ -380,6 +380,12 @@ struct cache_entry *make_empty_transient_cache_entry(size_t name_len);
*/
void discard_cache_entry(struct cache_entry *ce);
+/*
+ * Check configuration if we should perform extra validation on cache
+ * entries.
+ */
+int should_validate_cache_entries(void);
+
/*
* Duplicate a cache_entry. Allocate memory for the new entry from a
* memory_pool. Takes into account cache_entry fields that are meant
diff --git a/git.c b/git.c
index 9dbe6ffaa7..c7e4f0351a 100644
--- a/git.c
+++ b/git.c
@@ -414,7 +414,10 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv)
trace_argv_printf(argv, "trace: built-in: git");
+ validate_cache_entries(&the_index);
status = p->fn(argc, argv, prefix);
+ validate_cache_entries(&the_index);
+
if (status)
return status;
diff --git a/mem-pool.c b/mem-pool.c
index 139617cb23..a2841a4a9a 100644
--- a/mem-pool.c
+++ b/mem-pool.c
@@ -50,7 +50,7 @@ void mem_pool_init(struct mem_pool **mem_pool, size_t initial_size)
*mem_pool = pool;
}
-void mem_pool_discard(struct mem_pool *mem_pool)
+void mem_pool_discard(struct mem_pool *mem_pool, int invalidate_memory)
{
struct mp_block *block, *block_to_free;
@@ -59,6 +59,10 @@ void mem_pool_discard(struct mem_pool *mem_pool)
{
block_to_free = block;
block = block->next_block;
+
+ if (invalidate_memory)
+ memset(block_to_free->space, 0xDD, ((char *)block_to_free->end) - ((char *)block_to_free->space));
+
free(block_to_free);
}
diff --git a/mem-pool.h b/mem-pool.h
index adeefdcb28..999d3c3a52 100644
--- a/mem-pool.h
+++ b/mem-pool.h
@@ -29,7 +29,7 @@ void mem_pool_init(struct mem_pool **mem_pool, size_t initial_size);
/*
* Discard a memory pool and free all the memory it is responsible for.
*/
-void mem_pool_discard(struct mem_pool *mem_pool);
+void mem_pool_discard(struct mem_pool *mem_pool, int invalidate_memory);
/*
* Alloc memory from the mem_pool.
diff --git a/read-cache.c b/read-cache.c
index f346437800..ba31e929e8 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -2050,8 +2050,10 @@ int discard_index(struct index_state *istate)
* Cache entries in istate->cache[] should have been allocated
* from the memory pool associated with this index, or from an
* associated split_index. There is no need to free individual
- * cache entries.
+ * cache entries. validate_cache_entries can detect when this
+ * assertion does not hold.
*/
+ validate_cache_entries(istate);
resolve_undo_clear_index(istate);
istate->cache_nr = 0;
@@ -2068,13 +2070,45 @@ int discard_index(struct index_state *istate)
istate->untracked = NULL;
if (istate->ce_mem_pool) {
- mem_pool_discard(istate->ce_mem_pool);
+ mem_pool_discard(istate->ce_mem_pool, should_validate_cache_entries());
istate->ce_mem_pool = NULL;
}
return 0;
}
+/*
+ * Validate the cache entries of this index.
+ * All cache entries associated with this index
+ * should have been allocated by the memory pool
+ * associated with this index, or by a referenced
+ * split index.
+ */
+void validate_cache_entries(const struct index_state *istate)
+{
+ int i;
+
+ if (!should_validate_cache_entries() ||!istate || !istate->initialized)
+ return;
+
+ for (i = 0; i < istate->cache_nr; i++) {
+ if (!istate) {
+ die("internal error: cache entry is not allocated from expected memory pool");
+ } else if (!istate->ce_mem_pool ||
+ !mem_pool_contains(istate->ce_mem_pool, istate->cache[i])) {
+ if (!istate->split_index ||
+ !istate->split_index->base ||
+ !istate->split_index->base->ce_mem_pool ||
+ !mem_pool_contains(istate->split_index->base->ce_mem_pool, istate->cache[i])) {
+ die("internal error: cache entry is not allocated from expected memory pool");
+ }
+ }
+ }
+
+ if (istate->split_index)
+ validate_cache_entries(istate->split_index->base);
+}
+
int unmerged_index(const struct index_state *istate)
{
int i;
@@ -2878,8 +2912,25 @@ struct cache_entry *dup_cache_entry(const struct cache_entry *ce,
void discard_cache_entry(struct cache_entry *ce)
{
+ if (ce && should_validate_cache_entries())
+ memset(ce, 0xCD, cache_entry_size(ce->ce_namelen));
+
if (ce && ce->mem_pool_allocated)
return;
free(ce);
}
+
+int should_validate_cache_entries(void)
+{
+ static int validate_index_cache_entries = -1;
+
+ if (validate_index_cache_entries < 0) {
+ if (getenv("GIT_TEST_VALIDATE_INDEX_CACHE_ENTRIES"))
+ validate_index_cache_entries = 1;
+ else
+ validate_index_cache_entries = 0;
+ }
+
+ return validate_index_cache_entries;
+}
--
2.17.1
next prev parent reply other threads:[~2018-06-28 14:00 UTC|newest]
Thread overview: 100+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-04-17 16:34 [PATCH v1 0/5] Allocate cache entries from memory pool Jameson Miller
2018-04-17 16:34 ` Jameson Miller
2018-04-17 16:34 ` [PATCH v1 1/5] read-cache: teach refresh_cache_entry to take istate Jameson Miller
2018-04-17 19:00 ` Ben Peart
2018-04-17 16:34 ` [PATCH v1 2/5] Add an API creating / discarding cache_entry structs Jameson Miller
2018-04-17 23:11 ` Ben Peart
2018-04-17 16:34 ` [PATCH v1 4/5] Allocate cache entries from memory pools Jameson Miller
2018-04-17 16:34 ` [PATCH v1 3/5] mem-pool: fill out functionality Jameson Miller
2018-04-20 23:21 ` Jonathan Tan
2018-04-23 17:27 ` Jameson Miller
2018-04-23 17:49 ` Jonathan Tan
2018-04-23 18:20 ` Jameson Miller
2018-04-17 16:34 ` [PATCH v1 5/5] Add optional memory validations around cache_entry lifecyle Jameson Miller
2018-04-17 18:39 ` [PATCH v1 0/5] Allocate cache entries from memory pool Ben Peart
2018-04-23 14:09 ` Jameson Miller
2018-04-18 4:49 ` Junio C Hamano
2018-04-20 17:49 ` Stefan Beller
2018-04-23 16:44 ` Jameson Miller
2018-04-23 17:18 ` Stefan Beller
2018-04-23 16:19 ` Jameson Miller
2018-04-20 23:34 ` Jonathan Tan
2018-04-23 17:14 ` Jameson Miller
2018-04-30 15:31 ` [PATCH v2 " Jameson Miller
2018-04-30 15:31 ` [PATCH v2 1/5] read-cache: teach refresh_cache_entry() to take istate Jameson Miller
2018-04-30 15:31 ` [PATCH v2 2/5] block alloc: add lifecycle APIs for cache_entry structs Jameson Miller
2018-04-30 15:31 ` [PATCH v2 3/5] mem-pool: fill out functionality Jameson Miller
2018-04-30 21:42 ` Stefan Beller
2018-05-01 15:43 ` Jameson Miller
2018-05-03 16:18 ` Duy Nguyen
2018-04-30 15:31 ` [PATCH v2 4/5] block alloc: allocate cache entries from mem_pool Jameson Miller
2018-04-30 15:31 ` [PATCH v2 5/5] block alloc: add validations around cache_entry lifecyle Jameson Miller
2018-05-03 16:28 ` Duy Nguyen
2018-05-03 16:35 ` [PATCH v2 0/5] Allocate cache entries from memory pool Duy Nguyen
2018-05-03 17:21 ` Stefan Beller
2018-05-03 19:17 ` Duy Nguyen
2018-05-03 20:58 ` Stefan Beller
2018-05-03 21:13 ` Jameson Miller
2018-05-03 22:18 ` [PATCH] alloc.c: replace alloc by mempool Stefan Beller
2018-05-04 16:33 ` Duy Nguyen
2018-05-08 0:37 ` Junio C Hamano
2018-05-08 0:44 ` Stefan Beller
2018-05-08 1:07 ` Junio C Hamano
2018-05-23 14:47 ` [PATCH v3 0/7] allocate cache entries from memory pool Jameson Miller
2018-05-23 14:47 ` [PATCH v3 1/7] read-cache: teach refresh_cache_entry() to take istate Jameson Miller
2018-05-25 22:54 ` Stefan Beller
2018-05-23 14:47 ` [PATCH v3 2/7] block alloc: add lifecycle APIs for cache_entry structs Jameson Miller
2018-05-24 4:52 ` Junio C Hamano
2018-05-24 14:47 ` Jameson Miller
2018-05-23 14:47 ` [PATCH v3 3/7] mem-pool: only search head block for available space Jameson Miller
2018-05-23 14:47 ` [PATCH v3 4/7] mem-pool: add lifecycle management functions Jameson Miller
2018-05-23 14:47 ` [PATCH v3 5/7] mem-pool: fill out functionality Jameson Miller
2018-06-01 19:28 ` Stefan Beller
2018-05-23 14:47 ` [PATCH v3 6/7] block alloc: allocate cache entries from mem_pool Jameson Miller
2018-05-23 14:47 ` [PATCH v3 7/7] block alloc: add validations around cache_entry lifecyle Jameson Miller
2018-05-24 4:55 ` [PATCH v3 0/7] allocate cache entries from memory pool Junio C Hamano
2018-05-24 14:44 ` Jameson Miller
2018-05-25 22:53 ` Stefan Beller
2018-06-20 20:41 ` Jameson Miller
2018-05-25 22:41 ` Stefan Beller
2018-06-20 20:17 ` [PATCH v4 0/8] Allocate cache entries from mem_pool Jameson Miller
2018-06-20 20:17 ` [PATCH v4 1/8] read-cache: teach refresh_cache_entry() to take istate Jameson Miller
2018-06-20 20:17 ` [PATCH v4 2/8] block alloc: add lifecycle APIs for cache_entry structs Jameson Miller
2018-06-21 21:14 ` Stefan Beller
2018-06-28 14:07 ` Jameson Miller
2018-06-20 20:17 ` [PATCH v4 3/8] mem-pool: only search head block for available space Jameson Miller
2018-06-21 21:33 ` Stefan Beller
2018-06-28 14:12 ` Jameson Miller
2018-06-20 20:17 ` [PATCH v4 4/8] mem-pool: tweak math on mp_block allocation size Jameson Miller
2018-06-20 20:17 ` [PATCH v4 5/8] mem-pool: add lifecycle management functions Jameson Miller
2018-06-20 20:17 ` [PATCH v4 6/8] mem-pool: fill out functionality Jameson Miller
2018-06-20 20:17 ` [PATCH v4 7/8] block alloc: allocate cache entries from mem_pool Jameson Miller
2018-06-20 20:17 ` [PATCH v4 8/8] block alloc: add validations around cache_entry lifecyle Jameson Miller
2018-06-28 14:00 ` [PATCH v5 0/8] Allocate cache entries from mem_pool Jameson Miller
2018-06-28 14:00 ` [PATCH v5 1/8] read-cache: teach refresh_cache_entry() to take istate Jameson Miller
2018-06-28 14:00 ` [PATCH v5 2/8] read-cache: make_cache_entry should take object_id struct Jameson Miller
2018-06-28 17:14 ` Junio C Hamano
2018-06-28 22:27 ` SZEDER Gábor
2018-06-28 14:00 ` [PATCH v5 3/8] block alloc: add lifecycle APIs for cache_entry structs Jameson Miller
2018-06-28 18:43 ` Junio C Hamano
2018-06-28 22:28 ` SZEDER Gábor
2018-06-28 14:00 ` [PATCH v5 4/8] mem-pool: only search head block for available space Jameson Miller
2018-06-28 14:00 ` [PATCH v5 5/8] mem-pool: add life cycle management functions Jameson Miller
2018-06-28 17:15 ` Junio C Hamano
2018-06-28 14:00 ` [PATCH v5 6/8] mem-pool: fill out functionality Jameson Miller
2018-06-28 19:09 ` Junio C Hamano
2018-07-02 18:28 ` Jameson Miller
2018-06-28 14:00 ` [PATCH v5 7/8] block alloc: allocate cache entries from mem-pool Jameson Miller
2018-06-28 14:00 ` Jameson Miller [this message]
2018-07-02 19:49 ` [PATCH v6 0/8] Allocate cache entries from mem_pool Jameson Miller
2018-07-02 19:49 ` [PATCH v6 1/8] read-cache: teach refresh_cache_entry to take istate Jameson Miller
2018-07-02 19:49 ` [PATCH v6 2/8] read-cache: teach make_cache_entry to take object_id Jameson Miller
2018-07-02 21:23 ` Stefan Beller
2018-07-05 15:20 ` Jameson Miller
2018-07-02 19:49 ` [PATCH v6 3/8] block alloc: add lifecycle APIs for cache_entry structs Jameson Miller
2018-07-22 9:23 ` Duy Nguyen
2018-07-02 19:49 ` [PATCH v6 4/8] mem-pool: only search head block for available space Jameson Miller
2018-07-02 19:49 ` [PATCH v6 5/8] mem-pool: add life cycle management functions Jameson Miller
2018-07-02 19:49 ` [PATCH v6 6/8] mem-pool: fill out functionality Jameson Miller
2018-07-02 19:49 ` [PATCH v6 7/8] block alloc: allocate cache entries from mem_pool Jameson Miller
2018-07-02 19:49 ` [PATCH v6 8/8] block alloc: add validations around cache_entry lifecyle Jameson Miller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180628135932.225288-9-jamill@microsoft.com \
--to=jamill@microsoft.com \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=jonathantanmy@google.com \
--cc=pclouds@gmail.com \
--cc=peartben@gmail.com \
--cc=peff@peff.net \
--cc=sbeller@google.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).