emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/github-m-h-tsai/v0.6.2-repairtool' into 2016-02-29-mingus-merge

Browse Source 
Conflicts:
	Makefile.in
	VERSION
	thin-provisioning/commands.cc
	thin-provisioning/commands.h
	thin-provisioning/thin_delta.cc
This commit is contained in:
Joe Thornber
2016-02-29 10:29:28 +00:00
parent e78de5d3ad 08a7093cd5
commit 286f70ceac
32 changed files with 1592 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
persistent-data/block_counter.h
View File

@@ -20,6 +20,7 @@
#define BLOCK_COUNTER_H
#include "block.h"
#include "run_set.h"
//----------------------------------------------------------------
@@ -32,7 +33,9 @@ namespace persistent_data {
public:
typedef std::map<block_address, unsigned> count_map;
void inc(block_address b) {
virtual ~block_counter() {}
virtual void inc(block_address b) {
count_map::iterator it = counts_.find(b);
if (it == counts_.end())
counts_.insert(make_pair(b, 1));
@@ -40,7 +43,7 @@ namespace persistent_data {
it->second++;
}
unsigned get_count(block_address b) const {
virtual unsigned get_count(block_address b) const {
count_map::const_iterator it = counts_.find(b);
return (it == counts_.end()) ? 0 : it->second;
}
@@ -52,6 +55,29 @@ namespace persistent_data {
private:
count_map counts_;
};
//----------------------------------------------------------------
// Little helper class that keeps track of which blocks
// are referenced.
//----------------------------------------------------------------
class binary_block_counter : public block_counter {
public:
virtual ~binary_block_counter() {}
virtual void inc(block_address b) {
visited_.add(b);
}
virtual unsigned get_count(block_address b) const {
return visited_.member(b) ? 1 : 0;
}
base::run_set<block_address> const& get_visited() const {
return visited_;
}
private:
base::run_set<block_address> visited_;
};
}
//----------------------------------------------------------------

9
persistent-data/data-structures/array.h
View File

@@ -43,6 +43,15 @@ namespace persistent_data {
throw checksum_error("bad block nr in array block");
}
virtual bool check_raw(void const *raw) const {
array_block_disk const *data = reinterpret_cast<array_block_disk const *>(raw);
crc32c sum(ARRAY_CSUM_XOR);
sum.append(&data->max_entries, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(data->csum))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
array_block_disk *data = reinterpret_cast<array_block_disk *>(raw);
data->blocknr = to_disk<base::le64, uint64_t>(location);

23
persistent-data/data-structures/btree.tcc
View File

@@ -32,7 +32,6 @@ namespace {
using namespace persistent_data;
using namespace btree_detail;
using namespace std;
}
//----------------------------------------------------------------
@@ -90,14 +89,22 @@ namespace persistent_data {
{
uint32_t flags = to_cpu<uint32_t>(raw_->header.flags);
if (flags & INTERNAL_NODE) {
if (flags & LEAF_NODE)
throw runtime_error("btree node is both internal and leaf");
if (flags & LEAF_NODE) {
ostringstream out;
out << "btree node is both internal and leaf"
<< " (block " << location_ << ")";
throw runtime_error(out.str());
}
return INTERNAL;
} else if (flags & LEAF_NODE)
return LEAF;
else
throw runtime_error("unknown node type");
else {
ostringstream out;
out << "unknown node type"
<< " (block " << location_ << ")";
throw runtime_error(out.str());
}
}
template <typename ValueTraits>
@@ -352,7 +359,8 @@ namespace persistent_data {
std::ostringstream out;
out << "value size mismatch: expected " << sizeof(typename ValueTraits::disk_type)
<< ", but got " << get_value_size()
<< ". This is not the btree you are looking for." << std::endl;
<< ". This is not the btree you are looking for."
<< " (block " << location_ << ")" << std::endl;
return out.str();
}
@@ -371,7 +379,8 @@ namespace persistent_data {
if (max < get_nr_entries()) {
std::ostringstream out;
out << "Bad nr of elements: max per block = "
<< max << ", actual = " << get_nr_entries() << std::endl;
<< max << ", actual = " << get_nr_entries()
<< " (block " << location_ << ")" << std::endl;
throw std::runtime_error(out.str());
}

17
persistent-data/data-structures/btree_base_visitor.h Normal file
View File

@@ -0,0 +1,17 @@
#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_BASE_VISITOR_H
#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_BASE_VISITOR_H
#include "persistent-data/data-structures/btree.h"
namespace persistent_data {
namespace btree_detail {
template <typename ValueType>
class noop_value_visitor {
public:
virtual void visit(btree_path const &path, ValueType const &v) {
}
};
}
}
#endif

39
persistent-data/data-structures/btree_counter.h
View File

@@ -2,35 +2,44 @@
#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H
#include "persistent-data/data-structures/btree.h"
#include "persistent-data/data-structures/btree_base_visitor.h"
#include "persistent-data/data-structures/btree_damage_visitor.h"
#include "persistent-data/block_counter.h"
//----------------------------------------------------------------
namespace persistent_data {
namespace btree_count_detail {
template <unsigned Levels, typename ValueTraits, typename ValueCounter>
class counting_visitor : public btree<Levels, ValueTraits>::visitor {
template <typename ValueVisitor, typename DamageVisitor, unsigned Levels, typename ValueTraits, typename ValueCounter>
class counting_visitor : public btree_damage_visitor<ValueVisitor, DamageVisitor, Levels, ValueTraits> {
typedef btree_damage_visitor<ValueVisitor, DamageVisitor, Levels, ValueTraits> BtreeDamageVisitor;
public:
typedef btree<Levels, ValueTraits> tree;
counting_visitor(block_counter &bc, ValueCounter &vc)
: bc_(bc),
counting_visitor(ValueVisitor &value_visitor,
DamageVisitor &damage_visitor,
block_counter &bc,
ValueCounter &vc)
: BtreeDamageVisitor(value_visitor, damage_visitor, false),
bc_(bc),
vc_(vc) {
}
virtual bool visit_internal(node_location const &l,
typename tree::internal_node const &n) {
return visit_node(n);
return BtreeDamageVisitor::visit_internal(l, n) ?
visit_node(n) : false;
}
virtual bool visit_internal_leaf(node_location const &l,
typename tree::internal_node const &n) {
return visit_node(n);
return BtreeDamageVisitor::visit_internal_leaf(l, n) ?
visit_node(n) : false;
}
virtual bool visit_leaf(node_location const &l,
typename tree::leaf_node const &n) {
if (visit_node(n)) {
if (BtreeDamageVisitor::visit_leaf(l, n) && visit_node(n)) {
unsigned nr = n.get_nr_entries();
for (unsigned i = 0; i < nr; i++) {
@@ -85,7 +94,21 @@ namespace persistent_data {
// is not corrupt.
template <unsigned Levels, typename ValueTraits, typename ValueCounter>
void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc, ValueCounter &vc) {
btree_count_detail::counting_visitor<Levels, ValueTraits, ValueCounter> v(bc, vc);
typedef noop_value_visitor<typename ValueTraits::value_type> NoopValueVisitor;
NoopValueVisitor noop_vv;
noop_damage_visitor noop_dv;
btree_count_detail::counting_visitor<NoopValueVisitor, noop_damage_visitor, Levels, ValueTraits, ValueCounter> v(noop_vv, noop_dv, bc, vc);
tree.visit_depth_first(v);
}
template <unsigned Levels, typename ValueTraits>
void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc) {
typedef noop_value_visitor<typename ValueTraits::value_type> NoopValueVisitor;
NoopValueVisitor noop_vv;
noop_damage_visitor noop_dv;
typedef noop_value_counter<typename ValueTraits::value_type> NoopValueCounter;
NoopValueCounter vc;
btree_count_detail::counting_visitor<NoopValueVisitor, noop_damage_visitor, Levels, ValueTraits, NoopValueCounter> v(noop_vv, noop_dv, bc, vc);
tree.visit_depth_first(v);
}
}

32
persistent-data/data-structures/btree_damage_visitor.h
View File

@@ -27,6 +27,12 @@ namespace persistent_data {
return out;
}
class noop_damage_visitor {
public:
virtual void visit(btree_path const &path, damage const &d) {
}
};
// Tracks damage in a single level btree. Use multiple
// trackers if you have a multilayer tree.
class damage_tracker {
@@ -152,8 +158,9 @@ namespace persistent_data {
typedef boost::optional<run64> maybe_run64;
btree_damage_visitor(ValueVisitor &value_visitor,
DamageVisitor &damage_visitor)
: avoid_repeated_visits_(true),
DamageVisitor &damage_visitor,
bool avoid_repeated_visits = true)
: avoid_repeated_visits_(avoid_repeated_visits),
value_visitor_(value_visitor),
damage_visitor_(damage_visitor) {
}
@@ -300,14 +307,16 @@ namespace persistent_data {
size_t elt_size = sizeof(uint64_t) + n.get_value_size();
if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) {
std::ostringstream out;
out << "max entries too large: " << n.get_max_entries();
out << "max entries too large: " << n.get_max_entries()
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
if (n.get_max_entries() % 3) {
std::ostringstream out;
out << "max entries is not divisible by 3: " << n.get_max_entries();
out << "max entries is not divisible by 3: " << n.get_max_entries()
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
@@ -321,7 +330,8 @@ namespace persistent_data {
std::ostringstream out;
out << "bad nr_entries: "
<< n.get_nr_entries() << " < "
<< n.get_max_entries();
<< n.get_max_entries()
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
@@ -333,7 +343,8 @@ namespace persistent_data {
<< n.get_nr_entries()
<< ", expected at least "
<< min
<< "(max_entries = " << n.get_max_entries() << ")";
<< " (block " << n.get_location()
<< ", max_entries = " << n.get_max_entries() << ")";
report_damage(out.str());
return false;
}
@@ -354,7 +365,8 @@ namespace persistent_data {
uint64_t k = n.key_at(i);
if (k <= last_key) {
ostringstream out;
out << "keys are out of order, " << k << " <= " << last_key;
out << "keys are out of order, " << k << " <= " << last_key
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
@@ -372,7 +384,8 @@ namespace persistent_data {
if (*key > n.key_at(0)) {
ostringstream out;
out << "parent key mismatch: parent was " << *key
<< ", but lowest in node was " << n.key_at(0);
<< ", but lowest in node was " << n.key_at(0)
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}
@@ -388,7 +401,8 @@ namespace persistent_data {
if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) {
ostringstream out;
out << "the last key of the previous leaf was " << *last_leaf_key_[level]
<< " and the first key of this leaf is " << n.key_at(0);
<< " and the first key of this leaf is " << n.key_at(0)
<< " (block " << n.get_location() << ")";
report_damage(out.str());
return false;
}

39
persistent-data/data-structures/btree_key_value_extractor.h Normal file
View File

@@ -0,0 +1,39 @@
#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H
#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H
#include "persistent-data/data-structures/btree_damage_visitor.h"
#include <map>
namespace persistent_data {
namespace btree_detail {
template <typename ValueType>
class key_value_extractor {
typedef typename std::map<uint64_t, ValueType> MapType;
public:
key_value_extractor(MapType &map): map_(map) {
}
virtual ~key_value_extractor() {
}
virtual void visit(btree_path const &path, ValueType const &v) {
map_.insert(std::make_pair(path.back(), v));
}
private:
MapType &map_;
};
template <unsigned Levels, typename ValueTraits>
void btree_extract_key_values(btree<Levels, ValueTraits> const &tree,
std::map<uint64_t, typename ValueTraits::value_type> &map) {
typedef key_value_extractor<typename ValueTraits::value_type> KeyValueExtractor;
KeyValueExtractor kve(map);
noop_damage_visitor noop_dv;
btree_detail::btree_damage_visitor<KeyValueExtractor, noop_damage_visitor, Levels, ValueTraits>
v(kve, noop_dv);
tree.visit_depth_first(v);
}
}
}
#endif

33
persistent-data/space-maps/disk.cc
View File

@@ -50,6 +50,15 @@ namespace {
throw checksum_error("bad block nr in space map bitmap");
}
virtual bool check_raw(void const *raw) const {
bitmap_header const *data = reinterpret_cast<bitmap_header const *>(raw);
crc32c sum(BITMAP_CSUM_XOR);
sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(data->csum))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
bitmap_header *data = reinterpret_cast<bitmap_header *>(raw);
data->blocknr = to_disk<base::le64, uint64_t>(location);
@@ -75,6 +84,15 @@ namespace {
throw checksum_error("bad block nr in metadata index block");
}
virtual bool check_raw(void const *raw) const {
metadata_index const *mi = reinterpret_cast<metadata_index const *>(raw);
crc32c sum(INDEX_CSUM_XOR);
sum.append(&mi->padding_, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(mi->csum_))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
metadata_index *mi = reinterpret_cast<metadata_index *>(raw);
mi->blocknr_ = to_disk<base::le64, uint64_t>(location);
@@ -85,11 +103,6 @@ namespace {
}
};
bcache::validator::ptr
index_validator() {
return bcache::validator::ptr(new index_block_validator());
}
//--------------------------------
class bitmap {
@@ -771,4 +784,14 @@ persistent_data::open_metadata_sm(transaction_manager &tm, void *root)
checked_space_map::ptr(new sm_disk(store, tm, v))));
}
bcache::validator::ptr
persistent_data::bitmap_validator() {
return bcache::validator::ptr(new bitmap_block_validator());
}
bcache::validator::ptr
persistent_data::index_validator() {
return bcache::validator::ptr(new index_block_validator());
}
//----------------------------------------------------------------

6
persistent-data/space-maps/disk.h
View File

@@ -36,6 +36,12 @@ namespace persistent_data {
checked_space_map::ptr
open_metadata_sm(transaction_manager &tm, void *root);
bcache::validator::ptr
bitmap_validator();
bcache::validator::ptr
index_validator();
}
//----------------------------------------------------------------

10
persistent-data/validators.cc
View File

@@ -31,6 +31,16 @@ namespace {
}
}
virtual bool check_raw(void const *raw) const {
disk_node const *data = reinterpret_cast<disk_node const *>(raw);
node_header const *n = &data->header;
crc32c sum(BTREE_CSUM_XOR);
sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(n->csum))
return false;
return true;
}
virtual void prepare(void *raw, block_address location) const {
disk_node *data = reinterpret_cast<disk_node *>(raw);
node_header *n = &data->header;