emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove btree_checker, and add btree_counter

Browse Source
This commit is contained in:
Joe Thornber 2013-12-11 17:00:57 +00:00
parent 2de1962151
commit 927f611aad
4 changed files with 166 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
persistent-data/data-structures/btree_checker.h
View File

@ -1,277 +0,0 @@
// Copyright (C) 2011 Red Hat, Inc. All rights reserved.
//
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#ifndef BTREE_CHECKER_H
#define BTREE_CHECKER_H
#include "btree.h"
#include "persistent-data/block_counter.h"
#include "persistent-data/checksum.h"
#include "persistent-data/error_set.h"
#include <sstream>
#include <map>
#include <set>
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
namespace persistent_data {
//----------------------------------------------------------------
// This class implements consistency checking for the btrees in
// general. Derive from this if you want some additional checks.
// It's worth summarising what is checked:
//
// Implemented
// -----------
//
// - block_nr
// - nr_entries < max_entries
// - max_entries fits in block
// - max_entries is divisible by 3
// - nr_entries > minimum (except for root nodes)
//
// Not implemented
// ---------------
//
// - leaf | internal flags (this can be inferred from siblings)
//----------------------------------------------------------------
template <uint32_t Levels, typename ValueTraits>
class btree_checker : public btree<Levels, ValueTraits>::visitor {
public:
typedef btree_detail::node_location node_location;
btree_checker(block_counter &counter, bool avoid_repeated_visits = true)
: counter_(counter),
errs_(new error_set("btree errors")),
avoid_repeated_visits_(avoid_repeated_visits) {
}
bool visit_internal(node_location const &loc,
btree_detail::node_ref<block_traits> const &n) {
return check_internal(loc, n);
}
bool visit_internal_leaf(node_location const &loc,
btree_detail::node_ref<block_traits> const &n) {
return check_leaf(loc, n);
}
bool visit_leaf(node_location const &loc,
btree_detail::node_ref<ValueTraits> const &n) {
return check_leaf(loc, n);
}
error_set::ptr get_errors() const {
return errs_;
}
protected:
block_counter &get_counter() {
return counter_;
}
private:
bool check_internal(node_location const &loc,
btree_detail::node_ref<block_traits> const &n) {
if (!already_visited(n) &&
check_block_nr(n) &&
check_max_entries(n) &&
check_nr_entries(n, loc.is_sub_root()) &&
check_ordered_keys(n) &&
check_parent_key(loc.is_sub_root() ? boost::optional<uint64_t>() : loc.key, n)) {
if (loc.is_sub_root())
new_root(loc.level());
return true;
}
return false;
}
template <typename ValueTraits2>
bool check_leaf(node_location const &loc,
btree_detail::node_ref<ValueTraits2> const &n) {
if (!already_visited(n) &&
check_block_nr(n) &&
check_max_entries(n) &&
check_nr_entries(n, loc.is_sub_root()) &&
check_ordered_keys(n) &&
check_parent_key(loc.is_sub_root() ? boost::optional<uint64_t>() : loc.key, n)) {
if (loc.is_sub_root())
new_root(loc.level());
return check_leaf_key(loc.level(), n);
}
return false;
}
template <typename node>
bool already_visited(node const &n) {
block_address b = n.get_location();
counter_.inc(b);
if (avoid_repeated_visits_) {
if (seen_.count(b) > 0)
return true;
seen_.insert(b);
}
return false;
}
template <typename node>
bool check_block_nr(node const &n) const {
if (n.get_location() != n.get_block_nr()) {
std::ostringstream out;
out << "block number mismatch: actually "
<< n.get_location()
<< ", claims " << n.get_block_nr();
errs_->add_child(out.str());
return false;
}
return true;
}
template <typename node>
bool check_max_entries(node const &n) const {
size_t elt_size = sizeof(uint64_t) + n.get_value_size();
if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) {
std::ostringstream out;
out << "max entries too large: " << n.get_max_entries();
errs_->add_child(out.str());
return false;
}
if (n.get_max_entries() % 3) {
std::ostringstream out;
out << "max entries is not divisible by 3: " << n.get_max_entries();
errs_->add_child(out.str());
return false;
}
return true;
}
template <typename node>
bool check_nr_entries(node const &n, bool is_root) const {
if (n.get_nr_entries() > n.get_max_entries()) {
std::ostringstream out;
out << "bad nr_entries: "
<< n.get_nr_entries() << " < "
<< n.get_max_entries();
errs_->add_child(out.str());
return false;
}
block_address min = n.get_max_entries() / 3;
if (!is_root && (n.get_nr_entries() < min)) {
ostringstream out;
out << "too few entries in btree: "
<< n.get_nr_entries()
<< ", expected at least "
<< min
<< "(max_entries = " << n.get_max_entries() << ")";
errs_->add_child(out.str());
return false;
}
return true;
}
template <typename node>
bool check_ordered_keys(node const &n) const {
unsigned nr_entries = n.get_nr_entries();
if (nr_entries == 0)
return true; // can only happen if a root node
uint64_t last_key = n.key_at(0);
for (unsigned i = 1; i < nr_entries; i++) {
uint64_t k = n.key_at(i);
if (k <= last_key) {
ostringstream out;
out << "keys are out of order, " << k << " <= " << last_key;
errs_->add_child(out.str());
return false;
}
last_key = k;
}
return true;
}
template <typename node>
bool check_parent_key(boost::optional<uint64_t> key, node const &n) const {
if (!key)
return true;
if (*key > n.key_at(0)) {
ostringstream out;
out << "parent key mismatch: parent was " << *key
<< ", but lowest in node was " << n.key_at(0);
errs_->add_child(out.str());
return false;
}
return true;
}
template <typename node>
bool check_leaf_key(unsigned level, node const &n) {
if (n.get_nr_entries() == 0)
return true; // can only happen if a root node
if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) {
ostringstream out;
out << "the last key of the previous leaf was " << *last_leaf_key_[level]
<< " and the first key of this leaf is " << n.key_at(0);
errs_->add_child(out.str());
return false;
}
last_leaf_key_[level] = n.key_at(n.get_nr_entries() - 1);
return true;
}
void new_root(unsigned level) {
// we're starting a new subtree, so should
// reset the last_leaf value.
last_leaf_key_[level] = boost::optional<uint64_t>();
}
block_counter &counter_;
std::set<block_address> seen_;
error_set::ptr errs_;
boost::optional<uint64_t> last_leaf_key_[Levels];
bool avoid_repeated_visits_;
};
}
//----------------------------------------------------------------
#endif

81
persistent-data/data-structures/btree_counter.h Normal file
View File

@ -0,0 +1,81 @@
#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H
#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H
#include "persistent-data/block_counter.h"
//----------------------------------------------------------------
namespace persistent_data {
namespace btree_count_detail {
template <unsigned Levels, typename ValueTraits, typename ValueCounter>
class counting_visitor : public btree<Levels, ValueTraits>::visitor {
public:
typedef btree<Levels, ValueTraits> tree;
counting_visitor(block_counter &bc, ValueCounter &vc)
: bc_(bc),
vc_(vc) {
}
virtual bool visit_internal(node_location const &l,
typename tree::internal_node const &n) {
return visit_node(n);
}
virtual bool visit_internal_leaf(node_location const &l,
typename tree::internal_node const &n) {
return visit_node(n);
}
virtual bool visit_leaf(node_location const &l,
typename tree::leaf_node const &n) {
if (visit_node(n)) {
unsigned nr = n.get_nr_entries();
for (unsigned i = 0; i < nr; i++) {
// FIXME: confirm l2 is correct
node_location l2(l);
l2.push_key(i);
vc_.visit(l2, n.value_at(i));
}
return true;
}
return false;
}
private:
template <typename Node>
bool visit_node(Node const &n) {
block_address b = n.get_location();
bool seen = bc_.get_count(b);
bc_.inc(b);
return !seen;
}
block_counter &bc_;
ValueCounter &vc_;
};
}
template <typename T>
struct noop_value_counter {
void visit(btree_detail::node_location const &loc, T const &v) {
}
};
// Counts how many times each metadata block is referenced in the
// tree. Blocks already referenced in the block counter are not
// walked. This walk should only be done once you're sure the tree
// is not corrupt.
template <unsigned Levels, typename ValueTraits, typename ValueCounter>
void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc, ValueCounter &vc) {
btree_count_detail::counting_visitor<Levels, ValueTraits, ValueCounter> v(bc, vc);
tree.visit_depth_first(v);
}
}
//----------------------------------------------------------------
#endif

1
unit-tests/Makefile.in
View File

@ -51,6 +51,7 @@ TEST_SOURCE=\
unit-tests/bitset_t.cc \
unit-tests/block_t.cc \
unit-tests/btree_t.cc \
unit-tests/btree_counter_t.cc \
unit-tests/btree_damage_visitor_t.cc \
unit-tests/buffer_t.cc \
unit-tests/cache_t.cc \

84
unit-tests/btree_counter_t.cc Normal file
View File

@ -0,0 +1,84 @@
#include "gmock/gmock.h"
#include "test_utils.h"
#include "persistent-data/data-structures/btree.h"
#include "persistent-data/data-structures/btree_counter.h"
#include "persistent-data/space-maps/core.h"
using namespace base;
using namespace std;
using namespace persistent_data;
using namespace test;
using namespace testing;
//----------------------------------------------------------------
namespace {
block_address const BLOCK_SIZE = 4096;
block_address const NR_BLOCKS = 102400;
block_address const SUPERBLOCK = 0;
class BTreeCounterTests : public Test {
public:
BTreeCounterTests()
: bm_(create_bm<BLOCK_SIZE>(NR_BLOCKS)),
sm_(setup_core_map()),
tm_(new transaction_manager(bm_, sm_)) {
}
void check_nr_metadata_blocks_is_ge(unsigned n) {
block_counter bc;
noop_value_counter<uint64_t> vc;
count_btree_blocks(*tree_, bc, vc);
ASSERT_THAT(bc.get_counts().size(), Ge(n));
}
with_temp_directory dir_;
block_manager<>::ptr bm_;
space_map::ptr sm_;
transaction_manager::ptr tm_;
uint64_traits::ref_counter rc_;
btree<1, uint64_traits>::ptr tree_;
private:
space_map::ptr setup_core_map() {
space_map::ptr sm(new core_map(NR_BLOCKS));
sm->inc(SUPERBLOCK);
return sm;
}
void commit() {
block_manager<>::write_ref superblock(bm_->superblock(SUPERBLOCK));
}
};
void dump_counts(block_counter const &bc) {
block_counter::count_map::const_iterator it, end = bc.get_counts().end();
for (it = bc.get_counts().begin(); it != end; ++it)
cout << it->first << " -> " << it->second << endl;
}
}
//----------------------------------------------------------------
TEST_F(BTreeCounterTests, count_empty_tree)
{
tree_.reset(new btree<1, uint64_traits>(tm_, rc_));
check_nr_metadata_blocks_is_ge(1);
}
TEST_F(BTreeCounterTests, count_populated_tree)
{
tree_.reset(new btree<1, uint64_traits>(tm_, rc_));
for (unsigned i = 0; i < 10000; i++) {
uint64_t key[1] = {i};
tree_->insert(key, 0ull);
}
check_nr_metadata_blocks_is_ge(40);
}
//----------------------------------------------------------------