emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

A stack of thin_check refactorings

Browse Source
This commit is contained in:
Joe Thornber
2013-04-25 16:27:07 +01:00
parent 20ff78c818
commit a55f6fcf78
11 changed files with 613 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile.in
View File

@@ -56,6 +56,7 @@ SOURCE=\
thin-provisioning/metadata_dumper.cc \
thin-provisioning/restore_emitter.cc \
thin-provisioning/superblock_validator.cc \
thin-provisioning/superblock_checker.cc \
thin-provisioning/thin_pool.cc \
thin-provisioning/xml_format.cc
PDATA_OBJECTS=$(subst .cc,.o,$(SOURCE))
@@ -136,6 +137,7 @@ THIN_CHECK_SOURCE=\
thin-provisioning/metadata.cc \
thin-provisioning/metadata_checker.cc \
thin-provisioning/metadata_disk_structures.cc \
thin-provisioning/superblock_checker.cc \
thin-provisioning/superblock_validator.cc
THIN_DEBUG_OBJECTS=$(subst .cc,.o,$(THIN_DEBUG_SOURCE))

1
features/thin_check.feature
View File

@@ -49,7 +49,6 @@ Feature: thin_check
When I run thin_check with --super-block-only
Then it should pass
@announce
Scenario: --super-block-only check fails with corrupt superblock
Given a corrupt superblock
When I run thin_check with --super-block-only

88
thin-provisioning/metadata.cc
View File

@@ -38,15 +38,18 @@ using namespace thin_provisioning;
namespace {
unsigned const METADATA_CACHE_SIZE = 1024;
transaction_manager::ptr
open_tm(string const &dev_path, bool writeable) {
block_manager<>::ptr open_bm(string const &dev_path, bool writeable) {
block_address nr_blocks = get_nr_blocks(dev_path);
typename block_io<>::mode m = writeable ?
block_io<>::READ_WRITE :
block_io<>::READ_ONLY;
block_manager<>::ptr bm(new block_manager<>(dev_path, nr_blocks, 1, m));
space_map::ptr sm(new core_map(nr_blocks));
return block_manager<>::ptr(new block_manager<>(dev_path, nr_blocks, 1, m));
}
transaction_manager::ptr
open_tm(block_manager<>::ptr bm) {
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
sm->inc(SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
@@ -69,6 +72,21 @@ namespace {
lhs->set_count(b, rhs->get_count(b));
}
}
void print_superblock(superblock const &sb) {
using namespace std;
cerr << "superblock " << sb.csum_ << endl
<< "flags " << sb.flags_ << endl
<< "blocknr " << sb.blocknr_ << endl
<< "transaction id " << sb.trans_id_ << endl
<< "data mapping root " << sb.data_mapping_root_ << endl
<< "details root " << sb.device_details_root_ << endl
<< "data block size " << sb.data_block_size_ << endl
<< "metadata block size " << sb.metadata_block_size_ << endl
<< "metadata nr blocks " << sb.metadata_nr_blocks_ << endl
;
}
}
//----------------------------------------------------------------
@@ -78,7 +96,7 @@ metadata::metadata(std::string const &dev_path, open_type ot,
{
switch (ot) {
case OPEN:
tm_ = open_tm(dev_path, false);
tm_ = open_tm(open_bm(dev_path, false));
sb_ = read_superblock(tm_->get_bm());
if (sb_.version_ != 1)
@@ -94,7 +112,7 @@ metadata::metadata(std::string const &dev_path, open_type ot,
break;
case CREATE:
tm_ = open_tm(dev_path, true);
tm_ = open_tm(open_bm(dev_path, true));
space_map::ptr core = tm_->get_sm();
metadata_sm_ = create_metadata_sm(tm_, tm_->get_bm()->get_nr_blocks());
copy_space_maps(metadata_sm_, core);
@@ -120,7 +138,7 @@ metadata::metadata(std::string const &dev_path, open_type ot,
metadata::metadata(std::string const &dev_path, block_address metadata_snap)
{
tm_ = open_tm(dev_path, false);
tm_ = open_tm(open_bm(dev_path, false));
sb_ = read_superblock(tm_->get_bm(), metadata_snap);
// We don't open the metadata sm for a held root
//metadata_sm_ = open_metadata_sm(tm_, &sb_.metadata_space_map_root_);
@@ -132,20 +150,50 @@ metadata::metadata(std::string const &dev_path, block_address metadata_snap)
mappings_ = mapping_tree::ptr(new mapping_tree(tm_, sb_.data_mapping_root_, block_time_ref_counter(data_sm_)));
}
namespace {
void print_superblock(superblock const &sb) {
using namespace std;
// FIXME: duplication
metadata::metadata(block_manager<>::ptr bm, open_type ot,
sector_t data_block_size,
block_address nr_data_blocks)
{
switch (ot) {
case OPEN:
tm_ = open_tm(bm);
sb_ = read_superblock(tm_->get_bm());
cerr << "superblock " << sb.csum_ << endl
<< "flags " << sb.flags_ << endl
<< "blocknr " << sb.blocknr_ << endl
<< "transaction id " << sb.trans_id_ << endl
<< "data mapping root " << sb.data_mapping_root_ << endl
<< "details root " << sb.device_details_root_ << endl
<< "data block size " << sb.data_block_size_ << endl
<< "metadata block size " << sb.metadata_block_size_ << endl
<< "metadata nr blocks " << sb.metadata_nr_blocks_ << endl
;
if (sb_.version_ != 1)
throw runtime_error("unknown metadata version");
metadata_sm_ = open_metadata_sm(tm_, &sb_.metadata_space_map_root_);
tm_->set_sm(metadata_sm_);
data_sm_ = open_disk_sm(tm_, static_cast<void *>(&sb_.data_space_map_root_));
details_ = detail_tree::ptr(new detail_tree(tm_, sb_.device_details_root_, device_details_traits::ref_counter()));
mappings_top_level_ = dev_tree::ptr(new dev_tree(tm_, sb_.data_mapping_root_, mtree_ref_counter(tm_)));
mappings_ = mapping_tree::ptr(new mapping_tree(tm_, sb_.data_mapping_root_, block_time_ref_counter(data_sm_)));
break;
case CREATE:
tm_ = open_tm(bm);
space_map::ptr core = tm_->get_sm();
metadata_sm_ = create_metadata_sm(tm_, tm_->get_bm()->get_nr_blocks());
copy_space_maps(metadata_sm_, core);
tm_->set_sm(metadata_sm_);
data_sm_ = create_disk_sm(tm_, nr_data_blocks);
details_ = detail_tree::ptr(new detail_tree(tm_, device_details_traits::ref_counter()));
mappings_ = mapping_tree::ptr(new mapping_tree(tm_, block_time_ref_counter(data_sm_)));
mappings_top_level_ = dev_tree::ptr(new dev_tree(tm_, mappings_->get_root(), mtree_ref_counter(tm_)));
::memset(&sb_, 0, sizeof(sb_));
sb_.magic_ = SUPERBLOCK_MAGIC;
sb_.version_ = 1;
sb_.data_mapping_root_ = mappings_->get_root();
sb_.device_details_root_ = details_->get_root();
sb_.data_block_size_ = data_block_size;
sb_.metadata_block_size_ = MD_BLOCK_SIZE;
sb_.metadata_nr_blocks_ = tm_->get_bm()->get_nr_blocks();
break;
}
}

18
thin-provisioning/metadata.h
View File

@@ -34,8 +34,6 @@ namespace thin_provisioning {
using namespace base;
using namespace persistent_data;
block_address const SUPERBLOCK_LOCATION = 0;
typedef uint64_t sector_t;
typedef uint32_t thin_dev_t;
@@ -148,17 +146,27 @@ namespace thin_provisioning {
OPEN
};
typedef block_manager<>::read_ref read_ref;
typedef block_manager<>::write_ref write_ref;
typedef boost::shared_ptr<metadata> ptr;
// Deprecated: it would be better if we passed in an already
// constructed block_manager.
metadata(std::string const &dev_path, open_type ot,
sector_t data_block_size = 128,
block_address nr_data_blocks = 0); // Only used if CREATE
metadata(std::string const &dev_path, block_address metadata_snap);
// ... use these instead ...
metadata(block_manager<>::ptr bm, open_type ot,
sector_t data_block_size = 128,
block_address nr_data_blocks = 0); // Only used if CREATE
metadata(block_manager<>::ptr bm, block_address metadata_snap);
void commit();
typedef block_manager<>::read_ref read_ref;
typedef block_manager<>::write_ref write_ref;
typedef boost::shared_ptr<metadata> ptr;
tm_ptr tm_;
superblock sb_;

179
thin-provisioning/metadata_checker.cc
View File

@@ -21,10 +21,155 @@
#include "thin-provisioning/metadata_checker.h"
#include "thin-provisioning/superblock_validator.h"
using namespace persistent_data;
using namespace thin_provisioning;
//----------------------------------------------------------------
void
metadata_damage::set_message(std::string const &message)
{
message_ = message;
}
std::string const &
metadata_damage::get_message() const
{
return message_;
}
//--------------------------------
void
super_block_corruption::visit(metadata_damage_visitor &visitor) const
{
visitor.visit(*this);
}
//--------------------------------
missing_device_details::missing_device_details(uint64_t missing_begin,
uint64_t missing_end)
: missing_begin_(missing_begin),
missing_end_(missing_end)
{
}
void
missing_device_details::visit(metadata_damage_visitor &visitor) const
{
visitor.visit(*this);
}
//--------------------------------
missing_devices::missing_devices(uint64_t missing_begin,
uint64_t missing_end)
: missing_begin_(missing_begin),
missing_end_(missing_end)
{
}
void
missing_devices::visit(metadata_damage_visitor &visitor) const
{
visitor.visit(*this);
}
//--------------------------------
missing_mappings::missing_mappings(uint64_t dev,
uint64_t missing_begin,
uint64_t missing_end)
: dev_(dev),
missing_begin_(missing_begin),
missing_end_(missing_end)
{
}
void
missing_mappings::visit(metadata_damage_visitor &visitor) const
{
visitor.visit(*this);
}
//--------------------------------
bad_metadata_ref_count::bad_metadata_ref_count(block_address b,
ref_t actual,
ref_t expected)
: b_(b),
actual_(actual),
expected_(expected)
{
}
void
bad_metadata_ref_count::visit(metadata_damage_visitor &visitor) const
{
visitor.visit(*this);
}
//--------------------------------
bad_data_ref_count::bad_data_ref_count(block_address b,
ref_t actual,
ref_t expected)
: b_(b),
actual_(actual),
expected_(expected)
{
}
void
bad_data_ref_count::visit(metadata_damage_visitor &visitor) const
{
visitor.visit(*this);
}
//--------------------------------
missing_metadata_ref_counts::missing_metadata_ref_counts(block_address missing_begin,
block_address missing_end)
: missing_begin_(missing_begin),
missing_end_(missing_end)
{
}
void
missing_metadata_ref_counts::visit(metadata_damage_visitor &visitor) const
{
visitor.visit(*this);
}
//--------------------------------
missing_data_ref_counts::missing_data_ref_counts(block_address missing_begin,
block_address missing_end)
: missing_begin_(missing_begin),
missing_end_(missing_end)
{
}
void
missing_data_ref_counts::visit(metadata_damage_visitor &visitor) const
{
visitor.visit(*this);
}
//--------------------------------
void
metadata_damage_visitor::visit(metadata_damage const &damage)
{
damage.visit(*this);
}
//----------------------------------------------------------------
#if 0
namespace {
// As well as the standard btree checks, we build up a set of what
// devices having mappings defined, which can later be cross
@@ -166,19 +311,24 @@ namespace {
class metadata_checker {
public:
metadata_checker(string const &dev_path)
: bm_(open_bm(dev_path)) {
: bm_(open_bm(dev_path)),
errors_(new error_set("Errors in metadata")) {
}
boost::optional<error_set::ptr> check() {
#if 1
// FIXME: finish
error_set::ptr errors(new error_set("Errors in metadata"));
superblock sb = read_superblock();
return errors;
// FIXME: check version?
check_mappings();
return (errors_->get_children().size() > 0) ?
optional<error_set::ptr>(errors_) :
optional<error_set::ptr>();
#else
error_set::ptr errors(new error_set("Errors in metadata"));
block_counter metadata_counter, data_counter;
if (md->sb_.metadata_snap_) {
block_manager<>::ptr bm = md->tm_->get_bm();
@@ -197,9 +347,6 @@ namespace {
metadata_counter.inc(sb.device_details_root_);
}
mapping_validator::ptr mv(new mapping_validator(metadata_counter,
data_counter));
md->mappings_->visit(mv);
set<uint64_t> const &mapped_devs = mv->get_devices();
details_validator::ptr dv(new details_validator(metadata_counter));
@@ -246,11 +393,26 @@ namespace {
return sb;
}
void check_mappings() {
mapping_validator::ptr mv(
new mapping_validator(metadata_counter_,
data_counter_));
md->mappings_->visit(mv);
}
typedef block_manager<>::read_ref read_ref;
typedef block_manager<>::write_ref write_ref;
typedef boost::shared_ptr<metadata> ptr;
block_manager<>::ptr bm_;
error_set::ptr errors_;
block_counter metadata_counter_, data_counter_;
#if 0
tm_ptr tm_;
superblock sb_;
@@ -275,3 +437,4 @@ thin_provisioning::metadata_check(std::string const &dev_path)
}
//----------------------------------------------------------------
#endif

132
thin-provisioning/metadata_checker.h
View File

@@ -19,14 +19,140 @@
#ifndef METADATA_CHECKER_H
#define METADATA_CHECKER_H
#include "persistent-data/block.h"
#include "persistent-data/error_set.h"
#include "persistent-data/space_map.h"
#include <deque>
//----------------------------------------------------------------
namespace thin_provisioning {
// FIXME: pass in flags like --super-block-only
boost::optional<persistent_data::error_set::ptr>
metadata_check(std::string const &dev_path);
class metadata_damage_visitor;
// Base class for all types of metadata damage. Used in reporting.
class metadata_damage {
public:
typedef boost::shared_ptr<metadata_damage> ptr;
virtual ~metadata_damage() {}
virtual void visit(metadata_damage_visitor &visitor) const = 0;
void set_message(std::string const &message);
std::string const &get_message() const;
private:
std::string message_;
};
class super_block_corruption : public metadata_damage {
void visit(metadata_damage_visitor &visitor) const;
};
struct missing_device_details : public metadata_damage {
missing_device_details(uint64_t missing_begin,
uint64_t missing_end);
virtual void visit(metadata_damage_visitor &visitor) const;
uint64_t missing_begin_;
uint64_t missing_end_;
};
struct missing_devices : public metadata_damage {
missing_devices(uint64_t missing_begin,
uint64_t missing_end);
virtual void visit(metadata_damage_visitor &visitor) const;
uint64_t missing_begin_;
uint64_t missing_end_;
};
struct missing_mappings : public metadata_damage {
missing_mappings(uint64_t dev,
uint64_t missing_begin,
uint64_t missing_end);
virtual void visit(metadata_damage_visitor &visitor) const;
uint64_t dev_;
uint64_t missing_begin_;
uint64_t missing_end_;
};
struct bad_metadata_ref_count : public metadata_damage {
bad_metadata_ref_count(block_address b,
ref_t actual,
ref_t expected);
virtual void visit(metadata_damage_visitor &visitor) const;
block_address b_;
ref_t actual_;
ref_t expected_;
};
struct bad_data_ref_count : public metadata_damage {
bad_data_ref_count(block_address b,
ref_t actual,
ref_t expected);
virtual void visit(metadata_damage_visitor &visitor) const;
block_address b_;
ref_t actual_;
ref_t expected_;
};
struct missing_metadata_ref_counts : public metadata_damage {
missing_metadata_ref_counts(block_address missing_begin,
block_address missing_end);
virtual void visit(metadata_damage_visitor &visitor) const;
block_address missing_begin_;
block_address missing_end_;
};
struct missing_data_ref_counts : public metadata_damage {
missing_data_ref_counts(block_address missing_begin,
block_address missing_end);
virtual void visit(metadata_damage_visitor &visitor) const;
block_address missing_begin_;
block_address missing_end_;
};
class metadata_damage_visitor {
public:
typedef boost::shared_ptr<metadata_damage_visitor> ptr;
virtual ~metadata_damage_visitor() {}
void visit(metadata_damage const &damage);
virtual void visit(super_block_corruption const &damage) = 0;
virtual void visit(missing_device_details const &damage) = 0;
virtual void visit(missing_devices const &damage) = 0;
virtual void visit(missing_mappings const &damage) = 0;
virtual void visit(bad_metadata_ref_count const &damage) = 0;
virtual void visit(bad_data_ref_count const &damage) = 0;
virtual void visit(missing_metadata_ref_counts const &damage) = 0;
virtual void visit(missing_data_ref_counts const &damage) = 0;
};
typedef std::deque<metadata_damage::ptr> damage_list;
typedef boost::shared_ptr<damage_list> damage_list_ptr;
//--------------------------------
class checker {
public:
typedef boost::shared_ptr<checker> ptr;
virtual ~checker() {};
virtual damage_list_ptr check() = 0;
};
}
//----------------------------------------------------------------

2
thin-provisioning/metadata_disk_structures.h
View File

@@ -128,6 +128,8 @@ namespace thin_provisioning {
static void unpack(superblock_disk const &disk, superblock &core);
static void pack(superblock const &core, superblock_disk &disk);
};
block_address const SUPERBLOCK_LOCATION = 0;
}
//----------------------------------------------------------------

45
thin-provisioning/superblock_checker.cc Normal file
View File

@@ -0,0 +1,45 @@
#include "thin-provisioning/superblock_checker.h"
#include "thin-provisioning/metadata_disk_structures.h"
#include "thin-provisioning/superblock_validator.h"
using namespace thin_provisioning;
//----------------------------------------------------------------
superblock_checker::superblock_checker(block_manager::ptr bm)
: bm_(bm),
damage(new damage_list)
{
}
// FIXME: Other things to check:
// - magic
// - version
// - 3 * flags (should be zero)
// - in bounds: metadata_snap, data_mapping_root
// - metadata_nr_blocks_ matches what we've been given.
damage_list_ptr
superblock_checker::check()
{
superblock sb;
damage_list_ptr damage(new damage_list);
try {
block_manager::read_ref r = bm_->read_lock(SUPERBLOCK_LOCATION, superblock_validator());
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(&r.data());
superblock_traits::unpack(*sbd, sb);
} catch (checksum_error const &e) {
metadata_damage::ptr err(new super_block_corruption);
err->set_message("checksum error");
damage->push_back(err);
return damage;
}
return damage;
}
//----------------------------------------------------------------

24
thin-provisioning/superblock_checker.h Normal file
View File

@@ -0,0 +1,24 @@
#ifndef THIN_SUPERBLOCK_CHECKER_H
#define THIN_SUPERBLOCK_CHECKER_H
#include "thin-provisioning/metadata_checker.h"
//----------------------------------------------------------------
namespace thin_provisioning {
class superblock_checker : public checker {
public:
typedef persistent_data::block_manager<> block_manager;
superblock_checker(block_manager::ptr bm);
boost::shared_ptr<damage_list> check();
private:
block_manager::ptr bm_;
damage_list_ptr damage;
};
}
//----------------------------------------------------------------
#endif

1
unit-tests/Makefile.in
View File

@@ -49,6 +49,7 @@ TEST_SOURCE=\
unit-tests/buffer_t.cc \
unit-tests/cache_t.cc \
unit-tests/endian_t.cc \
unit-tests/metadata_checker_t.cc \
unit-tests/space_map_t.cc \
unit-tests/span_iterator_t.cc \
unit-tests/thin_metadata_t.cc \

158
unit-tests/metadata_checker_t.cc Normal file
View File

@@ -0,0 +1,158 @@
#include "gmock/gmock.h"
#include "test_utils.h"
#include "persistent-data/block.h"
#include "thin-provisioning/restore_emitter.h"
#include "thin-provisioning/superblock_checker.h"
#include <unistd.h>
using namespace persistent_data;
using namespace std;
using namespace test;
using namespace testing;
using namespace thin_provisioning;
//----------------------------------------------------------------
namespace {
block_address const BLOCK_SIZE = 4096;
block_address const NR_BLOCKS = 10240;
// FIXME: move to utils
class with_directory {
public:
with_directory(std::string const &path)
: old_path_(pwd()) {
chdir(path);
}
~with_directory() {
chdir(old_path_);
}
private:
std::string pwd() const {
char buffer[PATH_MAX];
char *ptr = getcwd(buffer, sizeof(buffer));
if (!ptr) {
// FIXME: still need a standard syscall failed exception
throw std::runtime_error("getcwd failed");
}
return ptr;
}
void chdir(std::string const &path) {
int r = ::chdir(path.c_str());
if (r < 0)
throw std::runtime_error("chdir failed");
}
std::string old_path_;
std::string new_path_;
};
class with_temp_directory {
public:
with_temp_directory() {
std::string name("./tmp");
rm_rf(name);
mkdir(name);
dir_.reset(new with_directory(name));
}
private:
void rm_rf(std::string const &name) {
std::string cmd("rm -rf ");
cmd += name;
system(cmd);
}
void mkdir(std::string const &name) {
std::string cmd("mkdir ");
cmd += name;
system(cmd);
}
void system(std::string const &cmd) {
int r = ::system(cmd.c_str());
if (r < 0)
throw std::runtime_error("system failed");
}
std::auto_ptr<with_directory> dir_;
};
//--------------------------------
class metadata_builder {
public:
metadata_builder(block_manager<>::ptr bm)
: bm_(bm) {
}
void build() {
metadata::ptr md(new metadata(bm_, metadata::CREATE, 128, 10240));
emitter::ptr restorer = create_restore_emitter(md);
restorer->begin_superblock("test-generated", 0, 0, 128, 10240, boost::optional<uint64_t>());
restorer->end_superblock();
}
// FIXME: add methods to specify volumes with particular
// mapping patterns.
private:
block_manager<>::ptr bm_;
};
class SuperBlockCheckerTests : public Test {
public:
SuperBlockCheckerTests()
: bm_(create_bm<BLOCK_SIZE>()) {
metadata_builder builder(bm_);
builder.build();
}
void corrupt_superblock() {
block_manager<>::write_ref wr = bm_->write_lock(0);
wr.data()[57] = 0;
}
with_temp_directory dir_;
block_manager<>::ptr bm_;
};
}
//----------------------------------------------------------------
TEST_F(SuperBlockCheckerTests, creation_requires_a_block_manager)
{
superblock_checker sc(bm_);
}
TEST_F(SuperBlockCheckerTests, passes_with_good_superblock)
{
superblock_checker sc(bm_);
damage_list_ptr damage = sc.check();
ASSERT_THAT(damage->size(), Eq(0U));
}
TEST_F(SuperBlockCheckerTests, fails_with_bad_checksum)
{
corrupt_superblock();
superblock_checker sc(bm_);
damage_list_ptr damage = sc.check();
ASSERT_THAT(damage->size(), Eq(1u));
metadata_damage::ptr d = *damage->begin();
ASSERT_THAT(dynamic_cast<super_block_corruption *>(d.get()), NotNull());
}
//----------------------------------------------------------------