emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into 2020-10-09-thin-restore-rewrite

Browse Source
This commit is contained in:
Joe Thornber 2020-11-25 11:20:28 +00:00
commit 327fc80fb0
16 changed files with 492 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Cargo.lock generated
View File

@ -65,6 +65,18 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "bitvec"
version = "0.19.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7ba35e9565969edb811639dbebfe34edc0368e472c5018474c8eb2543397f81"
dependencies = [
"funty",
"radium",
"tap",
"wyz",
]
[[package]]
name = "byteorder"
version = "1.3.4"
@ -188,6 +200,12 @@ dependencies = [
"miniz_oxide",
]
[[package]]
name = "funty"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ba62103ce691c2fd80fbae2213dfdda9ce60804973ac6b6e97de818ea7f52c8"
[[package]]
name = "futures"
version = "0.3.5"
@ -396,8 +414,11 @@ dependencies = [
[[package]]
name = "nom"
version = "6.0.0-alpha1"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4489ccc7d668957ddf64af7cd027c081728903afa6479d35da7e99bf5728f75f"
dependencies = [
"bitvec",
"lexical-core",
"memchr",
"version_check",
@ -555,6 +576,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "radium"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
[[package]]
name = "rand"
version = "0.7.3"
@ -683,6 +710,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "tap"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36474e732d1affd3a6ed582781b3683df3d0563714c59c39591e8ff707cf078e"
[[package]]
name = "tempfile"
version = "3.1.0"
@ -895,3 +928,9 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "wyz"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"

2
Cargo.toml
View File

@ -21,7 +21,7 @@ io-uring = "0.3"
indicatif = "0.15"
libc = "0.2.71"
nix = "0.17"
nom = { path = "/home/ejt/builds/nom/" }
nom = "6.0.0"
num_cpus = "1.13"
num-derive = "0.3"
num-traits = "0.2"

1
Makefile.in
View File

@ -134,6 +134,7 @@ DEVTOOLS_SOURCE=\
thin-provisioning/thin_generate_metadata.cc \
thin-provisioning/thin_generate_mappings.cc \
thin-provisioning/variable_chunk_stream.cc \
thin-provisioning/thin_patch_superblock.cc \
thin-provisioning/thin_show_metadata.cc \
thin-provisioning/thin_scan.cc \
ui/ui.cc

22
persistent-data/data-structures/btree_counter.h
View File

@ -14,12 +14,14 @@ namespace persistent_data {
public:
typedef btree<Levels, ValueTraits> tree;
counting_visitor(block_counter &bc, ValueCounter &vc)
counting_visitor(block_counter &bc, ValueCounter &vc,
bool ignore_non_fatal = false)
: bc_(bc),
vc_(vc),
error_outcome_(bc.stop_on_error() ?
tree::visitor::RETHROW_EXCEPTION :
tree::visitor::EXCEPTION_HANDLED) {
tree::visitor::EXCEPTION_HANDLED),
ignore_non_fatal_(ignore_non_fatal) {
}
virtual bool visit_internal(node_location const &l,
@ -66,7 +68,7 @@ namespace persistent_data {
if (!checker_.check_block_nr(n) ||
!checker_.check_value_size(n) ||
!checker_.check_max_entries(n) ||
!checker_.check_nr_entries(n, l.is_sub_root()) ||
!check_nr_entries(l, n) ||
!checker_.check_ordered_keys(n) ||
!checker_.check_parent_key(n, l.is_sub_root() ? boost::optional<uint64_t>() : l.key))
return false;
@ -83,7 +85,7 @@ namespace persistent_data {
if (!checker_.check_block_nr(n) ||
!checker_.check_value_size(n) ||
!checker_.check_max_entries(n) ||
!checker_.check_nr_entries(n, l.is_sub_root()) ||
!check_nr_entries(l, n) ||
!checker_.check_ordered_keys(n) ||
!checker_.check_parent_key(n, l.is_sub_root() ? boost::optional<uint64_t>() : l.key) ||
!checker_.check_leaf_key(n, last_leaf_key_[l.level()]))
@ -109,11 +111,18 @@ namespace persistent_data {
return !seen;
}
template <typename ValueTraits2>
bool check_nr_entries(node_location const &loc,
btree_detail::node_ref<ValueTraits2> const &n) {
return ignore_non_fatal_ || checker_.check_nr_entries(n, loc.is_sub_root());
}
block_counter &bc_;
ValueCounter &vc_;
btree_node_checker checker_;
boost::optional<uint64_t> last_leaf_key_[Levels];
error_outcome error_outcome_;
bool ignore_non_fatal_;
};
}
@ -141,8 +150,9 @@ namespace persistent_data {
// walked. This walk should only be done once you're sure the tree
// is not corrupt.
template <unsigned Levels, typename ValueTraits, typename ValueCounter>
void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc, ValueCounter &vc) {
btree_count_detail::counting_visitor<Levels, ValueTraits, ValueCounter> v(bc, vc);
void count_btree_blocks(btree<Levels, ValueTraits> const &tree, block_counter &bc, ValueCounter &vc,
bool ignore_non_fatal = false) {
btree_count_detail::counting_visitor<Levels, ValueTraits, ValueCounter> v(bc, vc, ignore_non_fatal);
tree.visit_depth_first(v);
}
}

18
tests/thin_check.rs
View File

@ -166,11 +166,6 @@ fn clear_needs_check_incompatible_opts() -> Result<()> {
"--super-block-only",
&md
))?;
run_fail(thin_check!(
"--clear-needs-check-flag",
"--skip-mappings",
&md
))?;
run_fail(thin_check!(
"--clear-needs-check-flag",
"--ignore-non-fatal-errors",
@ -204,6 +199,19 @@ fn no_clear_needs_check_if_error() -> Result<()> {
Ok(())
}
#[test]
fn clear_needs_check_if_skip_mappings() -> Result<()> {
let mut td = TestDir::new()?;
let md = prep_metadata(&mut td)?;
set_needs_check(&md)?;
generate_metadata_leaks(&md, 1, 0, 1)?;
assert!(get_needs_check(&md)?);
thin_check!("--clear-needs-check-flag", "--skip-mappings", &md).run()?;
assert!(!get_needs_check(&md)?);
Ok(())
}
#[test]
fn metadata_leaks_are_non_fatal() -> Result<()> {
let mut td = TestDir::new()?;

1
thin-provisioning/commands.cc
View File

@ -25,6 +25,7 @@ thin_provisioning::register_thin_commands(base::application &app)
app.add_cmd(command::ptr(new thin_generate_damage_cmd()));
app.add_cmd(command::ptr(new thin_generate_metadata_cmd()));
app.add_cmd(command::ptr(new thin_generate_mappings_cmd()));
app.add_cmd(command::ptr(new thin_patch_superblock_cmd()));
app.add_cmd(command::ptr(new thin_show_duplicates_cmd()));
app.add_cmd(command::ptr(new thin_show_metadata_cmd()));
app.add_cmd(command::ptr(new thin_journal_cmd()));

7
thin-provisioning/commands.h
View File

@ -124,6 +124,13 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv);
};
class thin_patch_superblock_cmd : public base::command {
public:
thin_patch_superblock_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_show_metadata_cmd : public base::command {
public:
thin_show_metadata_cmd();

8
thin-provisioning/damage_generator.cc
View File

@ -63,6 +63,14 @@ void damage_generator::create_metadata_leaks(block_address nr_leaks,
std::set<block_address> leaks;
find_blocks(md_->metadata_sm_, nr_leaks, expected, leaks);
block_counter bc(true);
md_->metadata_sm_->count_metadata(bc);
block_address nr_blocks = md_->metadata_sm_->get_nr_blocks();
for (block_address b = 0; b < nr_blocks; b++) {
if (bc.get_count(b))
md_->tm_->mark_shadowed(b);
}
for (auto const &b : leaks)
md_->metadata_sm_->set_count(b, actual);
}

81
thin-provisioning/metadata_checker.cc
View File

@ -303,11 +303,12 @@ namespace {
error_state check_metadata_space_map_counts(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
block_counter &bc,
nested_output &out) {
nested_output &out,
bool ignore_non_fatal) {
out << "checking space map counts" << end_message();
nested_output::nest _ = out.push();
if (!count_metadata(tm, sb, bc))
if (!count_metadata(tm, sb, bc, false, ignore_non_fatal))
return FATAL;
// Finally we need to check the metadata space map agrees
@ -371,7 +372,8 @@ namespace {
out_(cerr, 2),
info_out_(cout, 0),
expected_rc_(true), // set stop on the first error
err_(NO_ERROR) {
err_(NO_ERROR),
metadata_checked_(false) {
if (output_opts == OUTPUT_QUIET) {
out_.disable();
@ -381,6 +383,22 @@ namespace {
sb_location_ = get_superblock_location();
}
void check_and_repair() {
check();
if (options_.fix_metadata_leaks_)
fix_metadata_leaks(options_.open_transaction_);
if (options_.clear_needs_check_)
clear_needs_check_flag();
}
bool get_status() const {
if (options_.ignore_non_fatal_)
return (err_ == FATAL) ? false : true;
return (err_ == NO_ERROR) ? true : false;
}
private:
void check() {
block_manager::ptr bm = open_bm(path_, block_manager::READ_ONLY,
!options_.use_metadata_snap_);
@ -411,7 +429,7 @@ namespace {
// if we're checking everything, and there were no errors,
// then we should check the space maps too.
err_ << examine_metadata_space_map(tm, sb, options_.sm_opts_, out_, expected_rc_);
err_ << examine_metadata_space_map(tm, sb, options_.sm_opts_, options_.ignore_non_fatal_, out_, expected_rc_);
// verify ref-counts of data blocks
if (err_ != FATAL && core_sm)
@ -419,10 +437,12 @@ namespace {
} else
err_ << examine_data_mappings(tm, sb, options_.data_mapping_opts_, out_,
optional<space_map::ptr>());
metadata_checked_ = true;
}
bool fix_metadata_leaks(bool open_transaction) {
if (!verify_preconditions_before_fixing()) {
if (!metadata_checked_) {
out_ << "metadata has not been fully examined" << end_message();
return false;
}
@ -458,7 +478,7 @@ namespace {
}
bool clear_needs_check_flag() {
if (!verify_preconditions_before_fixing()) {
if (!metadata_checked_) {
out_ << "metadata has not been fully examined" << end_message();
return false;
}
@ -480,14 +500,6 @@ namespace {
return true;
}
bool get_status() const {
if (options_.ignore_non_fatal_)
return (err_ == FATAL) ? false : true;
return (err_ == NO_ERROR) ? true : false;
}
private:
block_address
get_superblock_location() {
block_address sb_location = superblock_detail::SUPERBLOCK_LOCATION;
@ -530,13 +542,14 @@ namespace {
examine_metadata_space_map(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
check_options::space_map_options option,
bool ignore_non_fatal,
nested_output &out,
block_counter &bc) {
error_state err = NO_ERROR;
switch (option) {
case check_options::SPACE_MAP_FULL:
err << check_metadata_space_map_counts(tm, sb, bc, out);
err << check_metadata_space_map_counts(tm, sb, bc, out, ignore_non_fatal);
break;
default:
break; // do nothing
@ -545,19 +558,6 @@ namespace {
return err;
}
bool verify_preconditions_before_fixing() const {
if (options_.use_metadata_snap_ ||
!!options_.override_mapping_root_ ||
options_.sm_opts_ != check_options::SPACE_MAP_FULL ||
options_.data_mapping_opts_ != check_options::DATA_MAPPING_LEVEL2)
return false;
if (!expected_rc_.get_counts().size())
return false;
return true;
}
std::string const &path_;
check_options options_;
nested_output out_;
@ -565,6 +565,7 @@ namespace {
block_address sb_location_;
block_counter expected_rc_;
base::error_state err_; // metadata state
bool metadata_checked_;
};
}
@ -628,12 +629,22 @@ bool check_options::check_conformance() {
cerr << "cannot perform fix with an overridden mapping root" << endl;
return false;
}
}
if (data_mapping_opts_ != DATA_MAPPING_LEVEL2 ||
sm_opts_ != SPACE_MAP_FULL) {
cerr << "cannot perform fix without a full examination" << endl;
if (fix_metadata_leaks_ &&
(data_mapping_opts_ != DATA_MAPPING_LEVEL2 || sm_opts_ != SPACE_MAP_FULL)) {
cerr << "cannot perform fix without a full examination" << endl;
return false;
}
if (clear_needs_check_) {
if (data_mapping_opts_ == DATA_MAPPING_NONE) {
cerr << "cannot perform fix without partially examination" << endl;
return false;
}
if (data_mapping_opts_ != DATA_MAPPING_LEVEL2 || sm_opts_ != SPACE_MAP_FULL)
cerr << "clearing needs_check without a full examination is not suggested" << endl;
}
return true;
@ -647,13 +658,7 @@ thin_provisioning::check_metadata(std::string const &path,
output_options output_opts)
{
metadata_checker checker(path, check_opts, output_opts);
checker.check();
if (check_opts.fix_metadata_leaks_)
checker.fix_metadata_leaks(check_opts.open_transaction_);
if (check_opts.clear_needs_check_)
checker.clear_needs_check_flag();
checker.check_and_repair();
return checker.get_status();
}

3
thin-provisioning/metadata_checker.h
View File

@ -48,11 +48,14 @@ namespace thin_provisioning {
void set_auto_repair();
void set_clear_needs_check();
// flags for checking
bool use_metadata_snap_;
data_mapping_options data_mapping_opts_;
space_map_options sm_opts_;
boost::optional<bcache::block_address> override_mapping_root_;
bool ignore_non_fatal_;
// flags for repairing
bool fix_metadata_leaks_;
bool clear_needs_check_;
bool open_transaction_;

14
thin-provisioning/metadata_counter.cc
View File

@ -10,14 +10,15 @@ using namespace thin_provisioning;
namespace {
bool count_trees(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
block_counter &bc) {
block_counter &bc,
bool ignore_non_fatal) {
// Count the device tree
{
noop_value_counter<device_tree_detail::device_details> vc;
device_tree dtree(*tm, sb.device_details_root_,
device_tree_detail::device_details_traits::ref_counter());
count_btree_blocks(dtree, bc, vc);
count_btree_blocks(dtree, bc, vc, ignore_non_fatal);
}
// Count the mapping tree
@ -25,7 +26,7 @@ namespace {
noop_value_counter<mapping_tree_detail::block_time> vc;
mapping_tree mtree(*tm, sb.data_mapping_root_,
mapping_tree_detail::block_traits::ref_counter(space_map::ptr()));
count_btree_blocks(mtree, bc, vc);
count_btree_blocks(mtree, bc, vc, ignore_non_fatal);
}
return true;
@ -65,19 +66,20 @@ namespace {
bool thin_provisioning::count_metadata(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
block_counter &bc,
bool skip_metadata_snap) {
bool skip_metadata_snap,
bool ignore_non_fatal) {
bool ret = true;
// Count the superblock
bc.inc(superblock_detail::SUPERBLOCK_LOCATION);
ret &= count_trees(tm, sb, bc);
ret &= count_trees(tm, sb, bc, ignore_non_fatal);
// Count the metadata snap, if present
if (!skip_metadata_snap && sb.metadata_snap_ != superblock_detail::SUPERBLOCK_LOCATION) {
bc.inc(sb.metadata_snap_);
superblock_detail::superblock snap = read_superblock(tm->get_bm(), sb.metadata_snap_);
ret &= count_trees(tm, snap, bc);
ret &= count_trees(tm, snap, bc, ignore_non_fatal);
}
ret &= count_space_maps(tm, sb, bc);

3
thin-provisioning/metadata_counter.h
View File

@ -10,7 +10,8 @@ namespace thin_provisioning {
bool count_metadata(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
block_counter &bc,
bool skip_metadata_snap = false);
bool skip_metadata_snap = false,
bool ignore_non_fatal = false);
}
//----------------------------------------------------------------

164
thin-provisioning/thin_delta.cc
View File

@ -22,47 +22,7 @@ using namespace thin_provisioning;
//----------------------------------------------------------------
namespace local {
class application {
public:
application(string const &cmd)
: cmd_(cmd) {
}
void usage(ostream &out) {
out << "Usage: " << cmd_ << " [options] <device or file>\n"
<< "Options:\n"
<< " {--thin1, --snap1}\n"
<< " {--thin2, --snap2}\n"
<< " {-m, --metadata-snap} [block#]\n"
<< " {--verbose}\n"
<< " {-h|--help}\n"
<< " {-V|--version}" << endl;
}
void die(string const &msg) {
cerr << msg << endl;
usage(cerr);
exit(1);
}
uint64_t parse_int(string const &str, string const &desc) {
try {
return boost::lexical_cast<uint64_t>(str);
} catch (...) {
ostringstream out;
out << "Couldn't parse " << desc << ": '" << str << "'";
die(out.str());
}
return 0; // never get here
}
private:
string cmd_;
};
namespace {
struct flags {
flags()
: verbose(false),
@ -76,6 +36,8 @@ namespace local {
boost::optional<uint64_t> metadata_snap;
boost::optional<uint64_t> snap1;
boost::optional<uint64_t> snap2;
boost::optional<uint64_t> root1;
boost::optional<uint64_t> root2;
};
//--------------------------------
@ -96,13 +58,6 @@ namespace local {
uint64_t vbegin_, dbegin_, len_;
};
ostream &operator <<(ostream &out, mapping const &m) {
out << "mapping[vbegin = " << m.vbegin_
<< ", dbegin = " << m.dbegin_
<< ", len = " << m.len_ << "]";
return out;
}
//--------------------------------
template <typename Container>
@ -528,9 +483,26 @@ namespace local {
out << "</superblock>\n";
}
void begin_diff(indented_stream &out, uint64_t snap1, uint64_t snap2) {
// FIXME: always show the blocknr?
void begin_diff(indented_stream &out,
boost::optional<uint64_t> snap1,
boost::optional<uint64_t> root1,
boost::optional<uint64_t> snap2,
boost::optional<uint64_t> root2) {
out.indent();
out << "<diff left=\"" << snap1 << "\" right=\"" << snap2 << "\">\n";
out << "<diff";
if (snap1)
out << " left=\"" << *snap1 << "\"";
else if (root1)
out << " left_root=\"" << *root1 << "\"";
if (snap2)
out << " right=\"" << *snap2 << "\"";
else if (root2)
out << " right_root=\"" << *root2 << "\"";
out << ">\n";
out.inc();
}
@ -540,7 +512,7 @@ namespace local {
out << "</diff>\n";
}
void delta_(application &app, flags const &fs) {
void delta_(flags const &fs) {
mapping_recorder mr1;
mapping_recorder mr2;
damage_visitor damage_v;
@ -552,25 +524,33 @@ namespace local {
metadata::ptr md(fs.use_metadata_snap ? new metadata(bm, fs.metadata_snap) : new metadata(bm));
sb = md->sb_;
dev_tree::key k = {*fs.snap1};
boost::optional<uint64_t> snap1_root = md->mappings_top_level_->lookup(k);
boost::optional<uint64_t> snap1_root;
if (fs.snap1) {
dev_tree::key k = {*fs.snap1};
snap1_root = md->mappings_top_level_->lookup(k);
} else if (fs.root1)
snap1_root = *fs.root1;
if (!snap1_root) {
ostringstream out;
out << "Unable to find mapping tree for snap1 (" << *fs.snap1 << ")";
app.die(out.str());
throw std::runtime_error(out.str());
}
single_mapping_tree snap1(*md->tm_, *snap1_root,
mapping_tree_detail::block_traits::ref_counter(md->tm_->get_sm()));
k[0] = *fs.snap2;
boost::optional<uint64_t> snap2_root = md->mappings_top_level_->lookup(k);
boost::optional<uint64_t> snap2_root;
if (fs.snap2) {
dev_tree::key k = {*fs.snap2};
snap2_root = md->mappings_top_level_->lookup(k);
} else if (fs.root2)
snap2_root = *fs.root2;
if (!snap2_root) {
ostringstream out;
out << "Unable to find mapping tree for snap2 (" << *fs.snap2 << ")";
app.die(out.str());
throw std::runtime_error(out.str());
}
single_mapping_tree snap2(*md->tm_, *snap2_root,
@ -593,7 +573,7 @@ namespace local {
sb.metadata_snap_ ?
boost::optional<block_address>(sb.metadata_snap_) :
boost::optional<block_address>());
begin_diff(is, *fs.snap1, *fs.snap2);
begin_diff(is, fs.snap1, fs.root1, fs.snap2, fs.root2);
if (fs.verbose) {
verbose_emitter e(is);
@ -607,12 +587,12 @@ namespace local {
end_superblock(is);
}
int delta(application &app, flags const &fs) {
int delta(flags const &fs) {
try {
delta_(app, fs);
delta_(fs);
} catch (exception const &e) {
app.die(e.what());
return 1; // never get here
cerr << e.what() << endl;
return 1;
}
return 0;
@ -631,77 +611,95 @@ thin_delta_cmd::thin_delta_cmd()
void
thin_delta_cmd::usage(std::ostream &out) const
{
// FIXME: finish
out << "Usage: " << get_name() << " [options] <device or file>\n"
<< "Options:\n"
<< " {--thin1, --snap1, --root1}\n"
<< " {--thin2, --snap2, --root2}\n"
<< " {-m, --metadata-snap} [block#]\n"
<< " {--verbose}\n"
<< " {-h|--help}\n"
<< " {-V|--version}" << endl;
}
int
thin_delta_cmd::run(int argc, char **argv)
{
using namespace local;
int c;
flags fs;
local::application app(basename(argv[0]));
char const shortopts[] = "hVm::";
option const longopts[] = {
{ "help", no_argument, NULL, 'h' },
{ "metadata-snap", optional_argument, NULL, 'm' },
{ "version", no_argument, NULL, 'V' },
{ "thin1", required_argument, NULL, 1 },
{ "snap1", required_argument, NULL, 1 },
{ "thin2", required_argument, NULL, 2 },
{ "snap2", required_argument, NULL, 2 },
{ "metadata-snap", optional_argument, NULL, 'm' },
{ "verbose", no_argument, NULL, 4 },
{ "root1", required_argument, NULL, 5 },
{ "root2", required_argument, NULL, 6 },
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch (c) {
case 'h':
app.usage(cout);
usage(cout);
return 0;
case 'm':
fs.use_metadata_snap = true;
if (optarg)
fs.metadata_snap = parse_uint64(optarg, "metadata snapshot block");
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
case 1:
fs.snap1 = app.parse_int(optarg, "thin id 1");
fs.snap1 = parse_uint64(optarg, "thin id 1");
break;
case 2:
fs.snap2 = app.parse_int(optarg, "thin id 2");
break;
case 'm':
fs.use_metadata_snap = true;
if (optarg)
fs.metadata_snap = app.parse_int(optarg, "metadata snapshot block");
fs.snap2 = parse_uint64(optarg, "thin id 2");
break;
case 4:
fs.verbose = true;
break;
case 5:
fs.root1 = parse_uint64(optarg, "thin root 1");
break;
case 6:
fs.root2 = parse_uint64(optarg, "thin root 2");
break;
default:
app.usage(cerr);
usage(cerr);
return 1;
}
}
if (argc == optind)
app.die("No input device provided.");
die("No input device provided.");
else
fs.dev = argv[optind];
if (!fs.snap1)
app.die("--snap1 not specified.");
if (!fs.snap1 && !fs.root1)
die("--snap1 or --root1 not specified.");
if (!!fs.snap1 && !!fs.root1)
die("--snap1 and --root1 are not compatible.");
if (!fs.snap2)
app.die("--snap2 not specified.");
if (!fs.snap2 && !fs.root2)
die("--snap2 or --root2 not specified.");
if (!!fs.snap2 && !!fs.root2)
die("--snap2 and --root2 are not compatible.");
return delta(app, fs);
return delta(fs);
}
//----------------------------------------------------------------

193
thin-provisioning/thin_patch_superblock.cc Normal file
View File

@ -0,0 +1,193 @@
#include <getopt.h>
#include "persistent-data/file_utils.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "thin-provisioning/commands.h"
#include "thin-provisioning/metadata.h"
#include "thin-provisioning/superblock.h"
#include "version.h"
using namespace persistent_data;
using namespace sm_disk_detail;
using namespace thin_provisioning;
//----------------------------------------------------------------
namespace {
struct flags {
flags();
bool check_conformance();
string output;
boost::optional<uint64_t> transaction_id;
block_address data_mapping_root;
block_address device_details_root;
block_address metadata_bitmap_root;
block_address metadata_ref_count_root;
block_address data_bitmap_root;
block_address data_ref_count_root;
};
flags::flags()
: data_mapping_root(0), device_details_root(0),
metadata_bitmap_root(0), metadata_ref_count_root(0),
data_bitmap_root(0), data_ref_count_root(0) {
}
bool flags::check_conformance() {
if (!output.size())
return false;
return true;
}
void patch_space_map_root(void *root,
block_address bitmap_root,
block_address ref_count_root) {
sm_disk_detail::sm_root v;
sm_disk_detail::sm_root_disk d;
memcpy(&d, root, sizeof(d));
sm_root_traits::unpack(d, v);
if (bitmap_root)
v.bitmap_root_ = bitmap_root;
if (ref_count_root)
v.ref_count_root_ = ref_count_root;
sm_root_traits::pack(v, d);
memcpy(root, &d, sizeof(d));
}
int patch_superblock(flags const &fs) {
block_manager::ptr bm = open_bm(fs.output, block_manager::READ_WRITE);
superblock_detail::superblock sb = read_superblock(bm, superblock_detail::SUPERBLOCK_LOCATION);
if (fs.transaction_id)
sb.trans_id_ = *fs.transaction_id;
if (fs.data_mapping_root)
sb.data_mapping_root_ = fs.data_mapping_root;
if (fs.device_details_root)
sb.device_details_root_ = fs.device_details_root;
patch_space_map_root(sb.metadata_space_map_root_,
fs.metadata_bitmap_root,
fs.metadata_ref_count_root);
patch_space_map_root(sb.data_space_map_root_,
fs.data_bitmap_root,
fs.data_ref_count_root);
write_superblock(bm, sb);
return 0;
}
}
//----------------------------------------------------------------
thin_patch_superblock_cmd::thin_patch_superblock_cmd()
: command("thin_patch_superblock")
{
}
void
thin_patch_superblock_cmd::usage(std::ostream &out) const
{
out << "Usage: " << get_name() << " [options]\n"
<< "Options:\n"
<< " {-h|--help}\n"
<< " {-o|--output} <output device or file>\n"
<< " {--transaction-id} <tid>\n"
<< " {--data-mapping-root} <blocknr>\n"
<< " {--device-details-root} <blocknr>\n"
<< " {--metadata-bitmap-root} <blocknr>\n"
<< " {--metadata-ref-count-root} <blocknr>\n"
<< " {--data-bitmap-root} <blocknr>\n"
<< " {--data-ref-count-root} <blocknr>\n"
<< " {-V|--version}" << endl;
}
int
thin_patch_superblock_cmd::run(int argc, char **argv)
{
int c;
flags fs;
const char shortopts[] = "ho:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "output", required_argument, NULL, 'o'},
{ "transaction-id", required_argument, NULL, 1 },
{ "data-mapping-root", required_argument, NULL, 2 },
{ "device-details-root", required_argument, NULL, 3 },
{ "metadata-bitmap-root", required_argument, NULL, 4 },
{ "metadata-ref-count-root", required_argument, NULL, 5 },
{ "data-bitmap-root", required_argument, NULL, 6 },
{ "data-ref-count-root", required_argument, NULL, 7 },
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'o':
fs.output = optarg;
break;
case 1:
fs.transaction_id = parse_uint64(optarg, "transaction_id");
break;
case 2:
fs.data_mapping_root = parse_uint64(optarg, "data_mapping_root");
break;
case 3:
fs.device_details_root = parse_uint64(optarg, "device_details_root");
break;
case 4:
fs.metadata_bitmap_root = parse_uint64(optarg, "metadata_bitmap_root");
break;
case 5:
fs.metadata_ref_count_root = parse_uint64(optarg, "metadata_ref_count_root");
break;
case 6:
fs.data_bitmap_root = parse_uint64(optarg, "data_bitmap_root");
break;
case 7:
fs.data_ref_count_root = parse_uint64(optarg, "data_ref_count_root");
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr);
return 1;
}
}
if (!fs.check_conformance()) {
usage(cerr);
return 1;
}
return patch_superblock(fs);
}
//----------------------------------------------------------------

72
unit-tests/btree_counter_t.cc
View File

@ -28,11 +28,21 @@ namespace {
tm_(bm_, sm_) {
}
void check_nr_metadata_blocks_is_ge(unsigned n) {
block_counter bc;
size_t get_nr_metadata_blocks(bool ignore_non_fatal = false,
bool stop_on_error = false) {
block_counter bc(stop_on_error);
noop_value_counter<uint64_t> vc;
count_btree_blocks(*tree_, bc, vc);
ASSERT_THAT(bc.get_counts().size(), Ge(n));
count_btree_blocks(*tree_, bc, vc, ignore_non_fatal);
return bc.get_counts().size();
}
void damage_first_leaf_underfull() {
bcache::validator::ptr v = create_btree_node_validator();
block_address b = get_first_leaf();
block_manager::write_ref blk = bm_->write_lock(b, v);
btree<1, uint64_traits>::leaf_node n = to_node<uint64_traits>(blk);
n.set_nr_entries(1);
}
with_temp_directory dir_;
@ -53,6 +63,19 @@ namespace {
void commit() {
block_manager::write_ref superblock(bm_->superblock(SUPERBLOCK));
}
block_address get_first_leaf() {
bcache::validator::ptr v = create_btree_node_validator();
block_manager::read_ref root = bm_->read_lock(tree_->get_root(), v);
btree<1, uint64_traits>::internal_node n = to_node<block_traits>(root);
while (n.get_type() == INTERNAL) {
block_manager::read_ref internal = bm_->read_lock(n.value_at(0), v);
n = to_node<block_traits>(internal);
}
return n.get_block_nr();
}
};
}
@ -62,7 +85,7 @@ TEST_F(BTreeCounterTests, count_empty_tree)
{
tree_.reset(new btree<1, uint64_traits>(tm_, rc_));
tm_.get_bm()->flush();
check_nr_metadata_blocks_is_ge(1);
ASSERT_GE(get_nr_metadata_blocks(), 1u);
}
TEST_F(BTreeCounterTests, count_populated_tree)
@ -75,7 +98,44 @@ TEST_F(BTreeCounterTests, count_populated_tree)
}
tm_.get_bm()->flush();
check_nr_metadata_blocks_is_ge(40);
ASSERT_GE(get_nr_metadata_blocks(), 40u);
}
TEST_F(BTreeCounterTests, count_underfull_nodes)
{
tree_.reset(new btree<1, uint64_traits>(tm_, rc_));
for (unsigned i = 0; i < 10000; i++) {
uint64_t key[1] = {i};
tree_->insert(key, 0ull);
}
tm_.get_bm()->flush();
size_t nr_blocks = get_nr_metadata_blocks();
damage_first_leaf_underfull();
tm_.get_bm()->flush();
// underfull nodes are not counted
bool ignore_non_fatal = false;
bool stop_on_error = false;
ASSERT_EQ(get_nr_metadata_blocks(ignore_non_fatal, stop_on_error), nr_blocks - 1);
// underfull nodes are counted
ignore_non_fatal = true;
stop_on_error = false;
ASSERT_EQ(get_nr_metadata_blocks(ignore_non_fatal, stop_on_error), nr_blocks);
// logical errors like underfull nodes don't result in exceptions,
// therefore the stop_on_error flag has no effect.
// FIXME: is it necessary to stop the counting on logical errors?
ignore_non_fatal = false;
stop_on_error = true;
ASSERT_EQ(get_nr_metadata_blocks(ignore_non_fatal, stop_on_error), nr_blocks - 1);
ignore_non_fatal = true;
stop_on_error = true;
ASSERT_EQ(get_nr_metadata_blocks(ignore_non_fatal, stop_on_error), nr_blocks);
}
//----------------------------------------------------------------

10
unit-tests/space_map_t.cc
View File

@ -368,6 +368,7 @@ namespace {
sm_disk_detail::sm_root root;
get_root(root);
test::zero_block(bm_, root.bitmap_root_);
bm_->flush();
}
// TODO: trash the bitmap corresponding to a given key
@ -376,12 +377,14 @@ namespace {
load_ies(entries);
ASSERT_LT(index, entries.size());
test::zero_block(bm_, entries[index].blocknr_);
bm_->flush();
}
void trash_ref_count_root() {
sm_disk_detail::sm_root root;
get_root(root);
test::zero_block(bm_, root.ref_count_root_);
bm_->flush();
}
// TODO: trash the node corresponding to a given key
@ -394,6 +397,7 @@ namespace {
ASSERT_GT(ref_count_blocks_.size(), 0u);
test::zero_block(bm_, *ref_count_blocks_.begin());
bm_->flush();
}
std::set<block_address> ref_count_blocks_;
@ -450,17 +454,20 @@ namespace {
sm_disk_detail::sm_root root;
get_data_sm_root(root);
test::zero_block(bm_, root.bitmap_root_);
bm_->flush();
}
// TODO: trash the bitmap corresponding to a given key
void trash_bitmap_block() {
test::zero_block(bm_, *bitmap_blocks_.begin());
bm_->flush();
}
void trash_ref_count_root() {
sm_disk_detail::sm_root root;
get_data_sm_root(root);
test::zero_block(bm_, root.ref_count_root_);
bm_->flush();
}
// TODO: trash the node corresponding to a given key
@ -471,6 +478,7 @@ namespace {
ASSERT_GT(ref_count_blocks_.size(), 0u);
test::zero_block(bm_, *ref_count_blocks_.begin());
bm_->flush();
}
std::set<block_address> index_store_blocks_;
@ -749,7 +757,7 @@ TEST_F(MetaSMCountingTests, test_trashed_bitmap_root)
// explicitly test open_metadata_sm()
block_manager::ptr bm(new block_manager("./test.data", NR_BLOCKS, MAX_LOCKS, block_manager::READ_WRITE));
space_map::ptr core_sm{create_core_map(NR_BLOCKS)};
transaction_manager::ptr tm(new transaction_manager(bm_, core_sm));
transaction_manager::ptr tm(new transaction_manager(bm, core_sm));
ASSERT_THROW(persistent_data::open_metadata_sm(*tm, &d), runtime_error);
}