From f016161e9a7a5837fef057ac07ecfa0986762be8 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 17 May 2013 11:29:34 +0100 Subject: [PATCH] [btree_damage_visitor] Hide this behind a utility function btree_visit_values(). --- .../data-structures/btree_damage_visitor.h | 612 +++++++++--------- unit-tests/btree_damage_visitor_t.cc | 8 +- 2 files changed, 313 insertions(+), 307 deletions(-) diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index 9cd2d0a..3c8a9c7 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -91,336 +91,346 @@ namespace persistent_data { bool damaged_; block_address damage_begin_; }; - } - //---------------------------------------------------------------- + //---------------------------------------------------------------- - // This class implements consistency checking for the btrees. It - // also allows the caller to visit all accessible values. + // This class implements consistency checking for the btrees. It + // also allows the caller to visit all accessible values. - // Derive from this if you want some additional checks. It's worth - // summarising what is checked: + // Derive from this if you want some additional checks. It's worth + // summarising what is checked: - // - // Implemented - // ----------- - // - // - block_nr - // - nr_entries < max_entries - // - max_entries fits in block - // - max_entries is divisible by 3 - // - nr_entries > minimum (except for root nodes) - // - // Not implemented - // --------------- - // - // - leaf | internal flags (this can be inferred from siblings) + // + // Implemented + // ----------- + // + // - block_nr + // - nr_entries < max_entries + // - max_entries fits in block + // - max_entries is divisible by 3 + // - nr_entries > minimum (except for root nodes) + // + // Not implemented + // --------------- + // + // - leaf | internal flags (this can be inferred from siblings) - //---------------------------------------------------------------- + //---------------------------------------------------------------- - template - class btree_damage_visitor : public btree::visitor { - public: - typedef btree_detail::node_location node_location; - typedef range range64; - typedef boost::optional maybe_range64; + template + class btree_damage_visitor : public btree::visitor { + public: + typedef btree_detail::node_location node_location; + typedef range range64; + typedef boost::optional maybe_range64; - btree_damage_visitor(block_counter &counter, - ValueVisitor &value_visitor, - DamageVisitor &damage_visitor) - : counter_(counter), - avoid_repeated_visits_(true), - value_visitor_(value_visitor), - damage_visitor_(damage_visitor) { - } + btree_damage_visitor(block_counter &counter, + ValueVisitor &value_visitor, + DamageVisitor &damage_visitor) + : counter_(counter), + avoid_repeated_visits_(true), + value_visitor_(value_visitor), + damage_visitor_(damage_visitor) { + } - bool visit_internal(node_location const &loc, - btree_detail::node_ref const &n) { - return check_internal(loc, n); - } + bool visit_internal(node_location const &loc, + btree_detail::node_ref const &n) { + return check_internal(loc, n); + } - bool visit_internal_leaf(node_location const &loc, - btree_detail::node_ref const &n) { - return check_leaf(loc, n); - } + bool visit_internal_leaf(node_location const &loc, + btree_detail::node_ref const &n) { + return check_leaf(loc, n); + } - bool visit_leaf(node_location const &loc, - btree_detail::node_ref const &n) { - bool r = check_leaf(loc, n); + bool visit_leaf(node_location const &loc, + btree_detail::node_ref const &n) { + bool r = check_leaf(loc, n); - // If anything goes wrong with the checks, we skip - // the value visiting. - if (!r) - return false; + // If anything goes wrong with the checks, we skip + // the value visiting. + if (!r) + return false; - visit_values(n); + visit_values(n); - return true; - } - - void visit_complete() { - end_walk(); - } - - typedef typename btree::visitor::error_outcome error_outcome; - - error_outcome error_accessing_node(node_location const &l, block_address b, - std::string const &what) { - report_damage(what); - return btree::visitor::EXCEPTION_HANDLED; - } - - private: - void visit_values(btree_detail::node_ref const &n) { - unsigned nr = n.get_nr_entries(); - for (unsigned i = 0; i < nr; i++) - value_visitor_.visit(n.value_at(i)); - } - - bool check_internal(node_location const &loc, - btree_detail::node_ref const &n) { - if (!already_visited(n) && - check_block_nr(n) && - check_max_entries(n) && - check_nr_entries(n, loc.is_sub_root()) && - check_ordered_keys(n) && - check_parent_key(loc.is_sub_root() ? optional() : loc.key, n)) { - if (loc.is_sub_root()) - new_root(loc.level()); - - good_internal(n.key_at(0)); return true; } - return false; - } + void visit_complete() { + end_walk(); + } - template - bool check_leaf(node_location const &loc, - btree_detail::node_ref const &n) { - if (!already_visited(n) && - check_block_nr(n) && - check_max_entries(n) && - check_nr_entries(n, loc.is_sub_root()) && - check_ordered_keys(n) && - check_parent_key(loc.is_sub_root() ? optional() : loc.key, n)) { - if (loc.is_sub_root()) - new_root(loc.level()); + typedef typename btree::visitor::error_outcome error_outcome; - bool r = check_leaf_key(loc.level(), n); - if (r && n.get_nr_entries() > 0) - good_leaf(n.key_at(0), n.key_at(n.get_nr_entries() - 1) + 1); + error_outcome error_accessing_node(node_location const &l, block_address b, + std::string const &what) { + report_damage(what); + return btree::visitor::EXCEPTION_HANDLED; + } + + private: + void visit_values(btree_detail::node_ref const &n) { + unsigned nr = n.get_nr_entries(); + for (unsigned i = 0; i < nr; i++) + value_visitor_.visit(n.value_at(i)); + } + + bool check_internal(node_location const &loc, + btree_detail::node_ref const &n) { + if (!already_visited(n) && + check_block_nr(n) && + check_max_entries(n) && + check_nr_entries(n, loc.is_sub_root()) && + check_ordered_keys(n) && + check_parent_key(loc.is_sub_root() ? optional() : loc.key, n)) { + if (loc.is_sub_root()) + new_root(loc.level()); + + good_internal(n.key_at(0)); + return true; + } + + return false; + } + + template + bool check_leaf(node_location const &loc, + btree_detail::node_ref const &n) { + if (!already_visited(n) && + check_block_nr(n) && + check_max_entries(n) && + check_nr_entries(n, loc.is_sub_root()) && + check_ordered_keys(n) && + check_parent_key(loc.is_sub_root() ? optional() : loc.key, n)) { + if (loc.is_sub_root()) + new_root(loc.level()); + + bool r = check_leaf_key(loc.level(), n); + if (r && n.get_nr_entries() > 0) + good_leaf(n.key_at(0), n.key_at(n.get_nr_entries() - 1) + 1); + + return r; + } + + return false; + } + + + template + bool already_visited(node const &n) { + block_address b = n.get_location(); + + counter_.inc(b); + + if (avoid_repeated_visits_) { + if (seen_.count(b) > 0) + return true; + + seen_.insert(b); + } + + return false; + } + + template + bool check_block_nr(node const &n) { + if (n.get_location() != n.get_block_nr()) { + std::ostringstream out; + out << "block number mismatch: actually " + << n.get_location() + << ", claims " << n.get_block_nr(); + + report_damage(out.str()); + return false; + } + + return true; + } + + template + bool check_max_entries(node const &n) { + size_t elt_size = sizeof(uint64_t) + n.get_value_size(); + if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { + std::ostringstream out; + out << "max entries too large: " << n.get_max_entries(); + report_damage(out.str()); + return false; + } + + if (n.get_max_entries() % 3) { + std::ostringstream out; + out << "max entries is not divisible by 3: " << n.get_max_entries(); + report_damage(out.str()); + return false; + } + + return true; + } + + template + bool check_nr_entries(node const &n, bool is_root) { + if (n.get_nr_entries() > n.get_max_entries()) { + std::ostringstream out; + out << "bad nr_entries: " + << n.get_nr_entries() << " < " + << n.get_max_entries(); + report_damage(out.str()); + return false; + } + + block_address min = n.get_max_entries() / 3; + if (!is_root && (n.get_nr_entries() < min)) { + ostringstream out; + out << "too few entries in btree_node: " + << n.get_nr_entries() + << ", expected at least " + << min + << "(max_entries = " << n.get_max_entries() << ")"; + report_damage(out.str()); + return false; + } + + return true; + } + + template + bool check_ordered_keys(node const &n) { + unsigned nr_entries = n.get_nr_entries(); + + if (nr_entries == 0) + return true; // can only happen if a root node + + uint64_t last_key = n.key_at(0); + + for (unsigned i = 1; i < nr_entries; i++) { + uint64_t k = n.key_at(i); + if (k <= last_key) { + ostringstream out; + out << "keys are out of order, " << k << " <= " << last_key; + report_damage(out.str()); + return false; + } + last_key = k; + } + + return true; + } + + template + bool check_parent_key(boost::optional key, node const &n) { + if (!key) + return true; + + if (*key > n.key_at(0)) { + ostringstream out; + out << "parent key mismatch: parent was " << *key + << ", but lowest in node was " << n.key_at(0); + report_damage(out.str()); + return false; + } + + return true; + } + + template + bool check_leaf_key(unsigned level, node const &n) { + if (n.get_nr_entries() == 0) + return true; // can only happen if a root node + + if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) { + ostringstream out; + out << "the last key of the previous leaf was " << *last_leaf_key_[level] + << " and the first key of this leaf is " << n.key_at(0); + report_damage(out.str()); + return false; + } + + last_leaf_key_[level] = n.key_at(n.get_nr_entries() - 1); + return true; + } + + void new_root(unsigned level) { + // we're starting a new subtree, so should + // reset the last_leaf value. + last_leaf_key_[level] = boost::optional(); + } + + //-------------------------------- + + // damage tracking + + void report_damage(std::string const &desc) { + damage_reasons_.push_back(desc); + dt_.bad_node(); + } + + void good_internal(block_address b) { + maybe_range64 mr = dt_.good_internal(b); + if (mr) + issue_damage(*mr); + } + + void good_leaf(block_address b, block_address e) { + maybe_range64 mr = dt_.good_leaf(b, e); + + if (mr) + issue_damage(*mr); + } + + void end_walk() { + maybe_range64 mr = dt_.end(); + if (mr) + issue_damage(*mr); + } + + void issue_damage(range64 const &r) { + // FIXME: we don't really know what level + // the damage is coming from + damage d(0, r, build_damage_desc()); + clear_damage_desc(); + damage_visitor_.visit(d); + } + + std::string build_damage_desc() const { + std::string r; + + std::list::const_iterator it, end = damage_reasons_.end(); + for (it = damage_reasons_.begin(); it != end; ++it) + r += *it; return r; } - return false; - } - - - template - bool already_visited(node const &n) { - block_address b = n.get_location(); - - counter_.inc(b); - - if (avoid_repeated_visits_) { - if (seen_.count(b) > 0) - return true; - - seen_.insert(b); + void clear_damage_desc() { + damage_reasons_.clear(); } - return false; - } + //-------------------------------- - template - bool check_block_nr(node const &n) { - if (n.get_location() != n.get_block_nr()) { - std::ostringstream out; - out << "block number mismatch: actually " - << n.get_location() - << ", claims " << n.get_block_nr(); + block_counter &counter_; + bool avoid_repeated_visits_; - report_damage(out.str()); - return false; - } + ValueVisitor &value_visitor_; + DamageVisitor &damage_visitor_; - return true; - } + std::set seen_; + boost::optional last_leaf_key_[Levels]; - template - bool check_max_entries(node const &n) { - size_t elt_size = sizeof(uint64_t) + n.get_value_size(); - if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { - std::ostringstream out; - out << "max entries too large: " << n.get_max_entries(); - report_damage(out.str()); - return false; - } + damage_tracker dt_; + std::list damage_reasons_; + }; + } - if (n.get_max_entries() % 3) { - std::ostringstream out; - out << "max entries is not divisible by 3: " << n.get_max_entries(); - report_damage(out.str()); - return false; - } - - return true; - } - - template - bool check_nr_entries(node const &n, bool is_root) { - if (n.get_nr_entries() > n.get_max_entries()) { - std::ostringstream out; - out << "bad nr_entries: " - << n.get_nr_entries() << " < " - << n.get_max_entries(); - report_damage(out.str()); - return false; - } - - block_address min = n.get_max_entries() / 3; - if (!is_root && (n.get_nr_entries() < min)) { - ostringstream out; - out << "too few entries in btree_node: " - << n.get_nr_entries() - << ", expected at least " - << min - << "(max_entries = " << n.get_max_entries() << ")"; - report_damage(out.str()); - return false; - } - - return true; - } - - template - bool check_ordered_keys(node const &n) { - unsigned nr_entries = n.get_nr_entries(); - - if (nr_entries == 0) - return true; // can only happen if a root node - - uint64_t last_key = n.key_at(0); - - for (unsigned i = 1; i < nr_entries; i++) { - uint64_t k = n.key_at(i); - if (k <= last_key) { - ostringstream out; - out << "keys are out of order, " << k << " <= " << last_key; - report_damage(out.str()); - return false; - } - last_key = k; - } - - return true; - } - - template - bool check_parent_key(boost::optional key, node const &n) { - if (!key) - return true; - - if (*key > n.key_at(0)) { - ostringstream out; - out << "parent key mismatch: parent was " << *key - << ", but lowest in node was " << n.key_at(0); - report_damage(out.str()); - return false; - } - - return true; - } - - template - bool check_leaf_key(unsigned level, node const &n) { - if (n.get_nr_entries() == 0) - return true; // can only happen if a root node - - if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) { - ostringstream out; - out << "the last key of the previous leaf was " << *last_leaf_key_[level] - << " and the first key of this leaf is " << n.key_at(0); - report_damage(out.str()); - return false; - } - - last_leaf_key_[level] = n.key_at(n.get_nr_entries() - 1); - return true; - } - - void new_root(unsigned level) { - // we're starting a new subtree, so should - // reset the last_leaf value. - last_leaf_key_[level] = boost::optional(); - } - - //-------------------------------- - - // damage tracking - - void report_damage(std::string const &desc) { - damage_reasons_.push_back(desc); - dt_.bad_node(); - } - - void good_internal(block_address b) { - maybe_range64 mr = dt_.good_internal(b); - if (mr) - issue_damage(*mr); - } - - void good_leaf(block_address b, block_address e) { - maybe_range64 mr = dt_.good_leaf(b, e); - - if (mr) - issue_damage(*mr); - } - - void end_walk() { - maybe_range64 mr = dt_.end(); - if (mr) - issue_damage(*mr); - } - - void issue_damage(range64 const &r) { - // FIXME: we don't really know what level - // the damage is coming from - damage d(0, r, build_damage_desc()); - clear_damage_desc(); - damage_visitor_.visit(d); - } - - std::string build_damage_desc() const { - std::string r; - - std::list::const_iterator it, end = damage_reasons_.end(); - for (it = damage_reasons_.begin(); it != end; ++it) - r += *it; - - return r; - } - - void clear_damage_desc() { - damage_reasons_.clear(); - } - - //-------------------------------- - - block_counter &counter_; - bool avoid_repeated_visits_; - - ValueVisitor &value_visitor_; - DamageVisitor &damage_visitor_; - - std::set seen_; - boost::optional last_leaf_key_[Levels]; - - damage_tracker dt_; - std::list damage_reasons_; - }; + template + void btree_visit_values(btree const &tree, + block_counter &counter, + ValueVisitor &value_visitor, + DamageVisitor &damage_visitor) { + btree_detail::btree_damage_visitor + v(counter, value_visitor, damage_visitor); + tree.visit_depth_first(v); + } } //---------------------------------------------------------------- diff --git a/unit-tests/btree_damage_visitor_t.cc b/unit-tests/btree_damage_visitor_t.cc index 45fb43f..8187e5f 100644 --- a/unit-tests/btree_damage_visitor_t.cc +++ b/unit-tests/btree_damage_visitor_t.cc @@ -344,9 +344,7 @@ namespace { virtual void run_() { block_counter counter; - btree_damage_visitor - visitor(counter, value_visitor_, damage_visitor_); - tree_->visit_depth_first(visitor); + btree_visit_values(*tree_, counter, value_visitor_, damage_visitor_); } }; @@ -399,9 +397,7 @@ namespace { virtual void run_() { block_counter counter; - btree_damage_visitor - visitor(counter, value_visitor_, damage_visitor_); - tree_->visit_depth_first(visitor); + btree_visit_values(*tree_, counter, value_visitor_, damage_visitor_); } }; }