From 927f611aad0c71fef3d06c4130b403b5bfc56ec1 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 11 Dec 2013 17:00:57 +0000 Subject: [PATCH] remove btree_checker, and add btree_counter --- .../data-structures/btree_checker.h | 277 ------------------ .../data-structures/btree_counter.h | 81 +++++ unit-tests/Makefile.in | 1 + unit-tests/btree_counter_t.cc | 84 ++++++ 4 files changed, 166 insertions(+), 277 deletions(-) delete mode 100644 persistent-data/data-structures/btree_checker.h create mode 100644 persistent-data/data-structures/btree_counter.h create mode 100644 unit-tests/btree_counter_t.cc diff --git a/persistent-data/data-structures/btree_checker.h b/persistent-data/data-structures/btree_checker.h deleted file mode 100644 index d78505f..0000000 --- a/persistent-data/data-structures/btree_checker.h +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright (C) 2011 Red Hat, Inc. All rights reserved. -// -// This file is part of the thin-provisioning-tools source. -// -// thin-provisioning-tools is free software: you can redistribute it -// and/or modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation, either version 3 of -// the License, or (at your option) any later version. -// -// thin-provisioning-tools is distributed in the hope that it will be -// useful, but WITHOUT ANY WARRANTY; without even the implied warranty -// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with thin-provisioning-tools. If not, see -// . - -#ifndef BTREE_CHECKER_H -#define BTREE_CHECKER_H - -#include "btree.h" - -#include "persistent-data/block_counter.h" -#include "persistent-data/checksum.h" -#include "persistent-data/error_set.h" - -#include -#include -#include - -using namespace persistent_data; -using namespace std; - -//---------------------------------------------------------------- - -namespace persistent_data { - //---------------------------------------------------------------- - // This class implements consistency checking for the btrees in - // general. Derive from this if you want some additional checks. - // It's worth summarising what is checked: - // - // Implemented - // ----------- - // - // - block_nr - // - nr_entries < max_entries - // - max_entries fits in block - // - max_entries is divisible by 3 - // - nr_entries > minimum (except for root nodes) - // - // Not implemented - // --------------- - // - // - leaf | internal flags (this can be inferred from siblings) - //---------------------------------------------------------------- - template - class btree_checker : public btree::visitor { - public: - typedef btree_detail::node_location node_location; - - btree_checker(block_counter &counter, bool avoid_repeated_visits = true) - : counter_(counter), - errs_(new error_set("btree errors")), - avoid_repeated_visits_(avoid_repeated_visits) { - } - - bool visit_internal(node_location const &loc, - btree_detail::node_ref const &n) { - return check_internal(loc, n); - } - - bool visit_internal_leaf(node_location const &loc, - btree_detail::node_ref const &n) { - return check_leaf(loc, n); - } - - bool visit_leaf(node_location const &loc, - btree_detail::node_ref const &n) { - return check_leaf(loc, n); - } - - error_set::ptr get_errors() const { - return errs_; - } - - protected: - block_counter &get_counter() { - return counter_; - } - - private: - bool check_internal(node_location const &loc, - btree_detail::node_ref const &n) { - if (!already_visited(n) && - check_block_nr(n) && - check_max_entries(n) && - check_nr_entries(n, loc.is_sub_root()) && - check_ordered_keys(n) && - check_parent_key(loc.is_sub_root() ? boost::optional() : loc.key, n)) { - if (loc.is_sub_root()) - new_root(loc.level()); - - return true; - } - - return false; - } - - template - bool check_leaf(node_location const &loc, - btree_detail::node_ref const &n) { - if (!already_visited(n) && - check_block_nr(n) && - check_max_entries(n) && - check_nr_entries(n, loc.is_sub_root()) && - check_ordered_keys(n) && - check_parent_key(loc.is_sub_root() ? boost::optional() : loc.key, n)) { - if (loc.is_sub_root()) - new_root(loc.level()); - - return check_leaf_key(loc.level(), n); - } - - return false; - } - - - template - bool already_visited(node const &n) { - block_address b = n.get_location(); - - counter_.inc(b); - - if (avoid_repeated_visits_) { - if (seen_.count(b) > 0) - return true; - - seen_.insert(b); - } - - return false; - } - - template - bool check_block_nr(node const &n) const { - if (n.get_location() != n.get_block_nr()) { - std::ostringstream out; - out << "block number mismatch: actually " - << n.get_location() - << ", claims " << n.get_block_nr(); - errs_->add_child(out.str()); - return false; - } - - return true; - } - - template - bool check_max_entries(node const &n) const { - size_t elt_size = sizeof(uint64_t) + n.get_value_size(); - if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { - std::ostringstream out; - out << "max entries too large: " << n.get_max_entries(); - errs_->add_child(out.str()); - return false; - } - - if (n.get_max_entries() % 3) { - std::ostringstream out; - out << "max entries is not divisible by 3: " << n.get_max_entries(); - errs_->add_child(out.str()); - return false; - } - - return true; - } - - template - bool check_nr_entries(node const &n, bool is_root) const { - if (n.get_nr_entries() > n.get_max_entries()) { - std::ostringstream out; - out << "bad nr_entries: " - << n.get_nr_entries() << " < " - << n.get_max_entries(); - errs_->add_child(out.str()); - return false; - } - - block_address min = n.get_max_entries() / 3; - if (!is_root && (n.get_nr_entries() < min)) { - ostringstream out; - out << "too few entries in btree: " - << n.get_nr_entries() - << ", expected at least " - << min - << "(max_entries = " << n.get_max_entries() << ")"; - errs_->add_child(out.str()); - return false; - } - - return true; - } - - template - bool check_ordered_keys(node const &n) const { - unsigned nr_entries = n.get_nr_entries(); - - if (nr_entries == 0) - return true; // can only happen if a root node - - uint64_t last_key = n.key_at(0); - - for (unsigned i = 1; i < nr_entries; i++) { - uint64_t k = n.key_at(i); - if (k <= last_key) { - ostringstream out; - out << "keys are out of order, " << k << " <= " << last_key; - errs_->add_child(out.str()); - return false; - } - last_key = k; - } - - return true; - } - - template - bool check_parent_key(boost::optional key, node const &n) const { - if (!key) - return true; - - if (*key > n.key_at(0)) { - ostringstream out; - out << "parent key mismatch: parent was " << *key - << ", but lowest in node was " << n.key_at(0); - errs_->add_child(out.str()); - return false; - } - - return true; - } - - template - bool check_leaf_key(unsigned level, node const &n) { - if (n.get_nr_entries() == 0) - return true; // can only happen if a root node - - if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) { - ostringstream out; - out << "the last key of the previous leaf was " << *last_leaf_key_[level] - << " and the first key of this leaf is " << n.key_at(0); - errs_->add_child(out.str()); - return false; - } - - last_leaf_key_[level] = n.key_at(n.get_nr_entries() - 1); - return true; - } - - void new_root(unsigned level) { - // we're starting a new subtree, so should - // reset the last_leaf value. - last_leaf_key_[level] = boost::optional(); - } - - block_counter &counter_; - std::set seen_; - error_set::ptr errs_; - boost::optional last_leaf_key_[Levels]; - bool avoid_repeated_visits_; - }; -} - -//---------------------------------------------------------------- - -#endif diff --git a/persistent-data/data-structures/btree_counter.h b/persistent-data/data-structures/btree_counter.h new file mode 100644 index 0000000..2f00173 --- /dev/null +++ b/persistent-data/data-structures/btree_counter.h @@ -0,0 +1,81 @@ +#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H +#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H + +#include "persistent-data/block_counter.h" + +//---------------------------------------------------------------- + +namespace persistent_data { + namespace btree_count_detail { + template + class counting_visitor : public btree::visitor { + public: + typedef btree tree; + + counting_visitor(block_counter &bc, ValueCounter &vc) + : bc_(bc), + vc_(vc) { + } + + virtual bool visit_internal(node_location const &l, + typename tree::internal_node const &n) { + return visit_node(n); + } + + virtual bool visit_internal_leaf(node_location const &l, + typename tree::internal_node const &n) { + return visit_node(n); + } + + virtual bool visit_leaf(node_location const &l, + typename tree::leaf_node const &n) { + if (visit_node(n)) { + unsigned nr = n.get_nr_entries(); + + for (unsigned i = 0; i < nr; i++) { + // FIXME: confirm l2 is correct + node_location l2(l); + l2.push_key(i); + vc_.visit(l2, n.value_at(i)); + } + + return true; + } + + return false; + } + + private: + template + bool visit_node(Node const &n) { + block_address b = n.get_location(); + bool seen = bc_.get_count(b); + bc_.inc(b); + return !seen; + } + + block_counter &bc_; + ValueCounter &vc_; + }; + } + + template + struct noop_value_counter { + void visit(btree_detail::node_location const &loc, T const &v) { + } + }; + + // Counts how many times each metadata block is referenced in the + // tree. Blocks already referenced in the block counter are not + // walked. This walk should only be done once you're sure the tree + // is not corrupt. + template + void count_btree_blocks(btree const &tree, block_counter &bc, ValueCounter &vc) { + btree_count_detail::counting_visitor v(bc, vc); + tree.visit_depth_first(v); + } +} + +//---------------------------------------------------------------- + +#endif diff --git a/unit-tests/Makefile.in b/unit-tests/Makefile.in index db790e6..d2506ec 100644 --- a/unit-tests/Makefile.in +++ b/unit-tests/Makefile.in @@ -51,6 +51,7 @@ TEST_SOURCE=\ unit-tests/bitset_t.cc \ unit-tests/block_t.cc \ unit-tests/btree_t.cc \ + unit-tests/btree_counter_t.cc \ unit-tests/btree_damage_visitor_t.cc \ unit-tests/buffer_t.cc \ unit-tests/cache_t.cc \ diff --git a/unit-tests/btree_counter_t.cc b/unit-tests/btree_counter_t.cc new file mode 100644 index 0000000..5f4a7d0 --- /dev/null +++ b/unit-tests/btree_counter_t.cc @@ -0,0 +1,84 @@ +#include "gmock/gmock.h" + +#include "test_utils.h" + +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/btree_counter.h" +#include "persistent-data/space-maps/core.h" + +using namespace base; +using namespace std; +using namespace persistent_data; +using namespace test; +using namespace testing; + +//---------------------------------------------------------------- + +namespace { + block_address const BLOCK_SIZE = 4096; + block_address const NR_BLOCKS = 102400; + block_address const SUPERBLOCK = 0; + + class BTreeCounterTests : public Test { + public: + BTreeCounterTests() + : bm_(create_bm(NR_BLOCKS)), + sm_(setup_core_map()), + tm_(new transaction_manager(bm_, sm_)) { + } + + void check_nr_metadata_blocks_is_ge(unsigned n) { + block_counter bc; + noop_value_counter vc; + count_btree_blocks(*tree_, bc, vc); + ASSERT_THAT(bc.get_counts().size(), Ge(n)); + } + + with_temp_directory dir_; + block_manager<>::ptr bm_; + space_map::ptr sm_; + transaction_manager::ptr tm_; + uint64_traits::ref_counter rc_; + + btree<1, uint64_traits>::ptr tree_; + + private: + space_map::ptr setup_core_map() { + space_map::ptr sm(new core_map(NR_BLOCKS)); + sm->inc(SUPERBLOCK); + return sm; + } + + void commit() { + block_manager<>::write_ref superblock(bm_->superblock(SUPERBLOCK)); + } + }; + + void dump_counts(block_counter const &bc) { + block_counter::count_map::const_iterator it, end = bc.get_counts().end(); + for (it = bc.get_counts().begin(); it != end; ++it) + cout << it->first << " -> " << it->second << endl; + } +} + +//---------------------------------------------------------------- + +TEST_F(BTreeCounterTests, count_empty_tree) +{ + tree_.reset(new btree<1, uint64_traits>(tm_, rc_)); + check_nr_metadata_blocks_is_ge(1); +} + +TEST_F(BTreeCounterTests, count_populated_tree) +{ + tree_.reset(new btree<1, uint64_traits>(tm_, rc_)); + + for (unsigned i = 0; i < 10000; i++) { + uint64_t key[1] = {i}; + tree_->insert(key, 0ull); + } + + check_nr_metadata_blocks_is_ge(40); +} + +//----------------------------------------------------------------