emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:jthornber/thin-provisioning-tools

Browse Source
This commit is contained in:
Joe Thornber 2020-07-22 16:09:33 +01:00
commit e9055f5d7d
15 changed files with 123 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION
View File

@ -1 +1 @@
0.9.0-rc1
0.9.0-rc2

4
base/io_generator.cc
View File

@ -48,8 +48,10 @@ namespace {
base::sector_t next_offset() {
sector_t r = current_;
current_ += block_size_;
if (current_ > end_)
if (current_ > end_) {
current_ = begin_;
return begin_;
}
return r;
}

4
functional-tests/scenario-string-constants.scm
View File

@ -76,7 +76,7 @@ Options:
{-V|--version}")
(define thin-metadata-pack-help
"thin_metadata_pack 0.9.0-rc1
"thin_metadata_pack 0.9.0-rc2
Produces a compressed file of thin metadata. Only packs metadata blocks that are actually used.
USAGE:
@ -91,7 +91,7 @@ OPTIONS:
-o <FILE> Specify packed output file")
(define thin-metadata-unpack-help
"thin_metadata_unpack 0.9.0-rc1
"thin_metadata_unpack 0.9.0-rc2
Unpack a compressed file of thin metadata.
USAGE:

4
functional-tests/thin-functional-tests.scm
View File

@ -511,7 +511,7 @@
(define-scenario (thin-metadata-pack version)
"accepts --version"
(run-ok-rcv (stdout _) (thin-metadata-pack "--version")
(assert-equal "thin_metadata_pack 0.9.0-rc1" stdout)))
(assert-equal "thin_metadata_pack 0.9.0-rc2" stdout)))
(define-scenario (thin-metadata-pack h)
"accepts -h"
@ -553,7 +553,7 @@
(define-scenario (thin-metadata-unpack version)
"accepts --version"
(run-ok-rcv (stdout _) (thin-metadata-unpack "--version")
(assert-equal "thin_metadata_unpack 0.9.0-rc1" stdout)))
(assert-equal "thin_metadata_unpack 0.9.0-rc2" stdout)))
(define-scenario (thin-metadata-unpack h)
"accepts -h"

13
persistent-data/data-structures/btree-remove.tcc
View File

@ -93,11 +93,12 @@ namespace persistent_data {
{
using namespace btree_detail;
unsigned i = 0;
bool r = false;
unsigned i = *index;
for (;;) {
r = spine.step(block);
bool inc = spine.step(block);
if (inc)
inc_children<ValueTraits>(spine, leaf_rc);
// patch up the parent to point to the new shadow
if (spine.has_parent()) {
@ -115,9 +116,9 @@ namespace persistent_data {
return true;
}
r = rebalance_children<ValueTraits>(spine, key);
bool r = rebalance_children<ValueTraits>(spine, key);
if (!r)
break;
return false;
n = spine.get_node<block_traits>();
if (n.get_type() == btree_detail::LEAF) {
@ -133,7 +134,7 @@ namespace persistent_data {
block = n.value_at(i);
}
return r;
return true;
}
template <unsigned Levels, typename _>

22
persistent-data/data-structures/btree_damage_visitor.h
View File

@ -106,10 +106,12 @@ namespace persistent_data {
typedef boost::optional<run64> maybe_run64;
btree_damage_visitor(ValueVisitor &value_visitor,
DamageVisitor &damage_visitor)
DamageVisitor &damage_visitor,
bool ignore_non_fatal)
: avoid_repeated_visits_(true),
value_visitor_(value_visitor),
damage_visitor_(damage_visitor) {
damage_visitor_(damage_visitor),
ignore_non_fatal_(ignore_non_fatal) {
}
bool visit_internal(node_location const &loc,
@ -178,7 +180,7 @@ namespace persistent_data {
else if (!checker_.check_block_nr(n) ||
!checker_.check_value_size(n) ||
!checker_.check_max_entries(n) ||
!checker_.check_nr_entries(n, loc.is_sub_root()) ||
!check_nr_entries(loc, n) ||
!checker_.check_ordered_keys(n) ||
!checker_.check_parent_key(n, loc.is_sub_root() ? boost::optional<uint64_t>() : loc.key)) {
report_damage(checker_.get_last_error_string());
@ -202,7 +204,7 @@ namespace persistent_data {
else if (!checker_.check_block_nr(n) ||
!checker_.check_value_size(n) ||
!checker_.check_max_entries(n) ||
!checker_.check_nr_entries(n, loc.is_sub_root()) ||
!check_nr_entries(loc, n) ||
!checker_.check_ordered_keys(n) ||
!checker_.check_parent_key(n, loc.is_sub_root() ? boost::optional<uint64_t>() : loc.key) ||
!checker_.check_leaf_key(n, last_leaf_key_[loc.level()])) {
@ -300,6 +302,12 @@ namespace persistent_data {
maybe_issue_damage(*old_path);
}
template <typename ValueTraits2>
bool check_nr_entries(node_location const &loc,
btree_detail::node_ref<ValueTraits2> const &n) {
return ignore_non_fatal_ || checker_.check_nr_entries(n, loc.is_sub_root());
}
//--------------------------------
bool avoid_repeated_visits_;
@ -314,15 +322,17 @@ namespace persistent_data {
path_tracker path_tracker_;
damage_tracker dt_;
std::list<std::string> damage_reasons_;
bool ignore_non_fatal_;
};
}
template <unsigned Levels, typename ValueTraits, typename ValueVisitor, typename DamageVisitor>
void btree_visit_values(btree<Levels, ValueTraits> const &tree,
ValueVisitor &value_visitor,
DamageVisitor &damage_visitor) {
DamageVisitor &damage_visitor,
bool ignore_non_fatal = false) {
btree_detail::btree_damage_visitor<ValueVisitor, DamageVisitor, Levels, ValueTraits>
v(value_visitor, damage_visitor);
v(value_visitor, damage_visitor, ignore_non_fatal);
tree.visit_depth_first(v);
}
}

21
src/pack/toplevel.rs
View File

@ -1,3 +1,4 @@
use anyhow::Result;
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use flate2::{read::ZlibDecoder, write::ZlibEncoder, Compression};
@ -35,11 +36,6 @@ fn shuffle<T>(v: &mut Vec<T>) {
v.shuffle(&mut rng);
}
// FIXME: move to a utils module
fn div_up(n: u64, d: u64) -> u64 {
(n + d - 1) / d
}
// Each thread processes multiple contiguous runs of blocks, called
// chunks. Chunks are shuffled so each thread gets chunks spread
// across the dev in case there are large regions that don't contain
@ -48,12 +44,17 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec<Vec<(u64, u64)>> {
use std::cmp::{max, min};
let chunk_size = min(4 * 1024u64, max(128u64, nr_blocks / (nr_jobs * 64)));
let nr_chunks = div_up(nr_blocks, chunk_size);
let nr_chunks = nr_blocks / chunk_size;
let mut chunks = Vec::with_capacity(nr_chunks as usize);
for i in 0..nr_chunks {
chunks.push((i * chunk_size, (i + 1) * chunk_size));
}
// there may be a smaller chunk at the back of the file.
if nr_chunks * chunk_size < nr_blocks {
chunks.push((nr_chunks * chunk_size, nr_blocks));
}
shuffle(&mut chunks);
let mut vs = Vec::with_capacity(nr_jobs as usize);
@ -105,11 +106,7 @@ pub fn pack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>>
Ok(())
}
fn crunch<R, W>(
input: Arc<Mutex<R>>,
output: Arc<Mutex<W>>,
ranges: Vec<(u64, u64)>,
) -> io::Result<()>
fn crunch<R, W>(input: Arc<Mutex<R>>, output: Arc<Mutex<W>>, ranges: Vec<(u64, u64)>) -> Result<()>
where
R: Read + Seek,
W: Write,
@ -257,7 +254,7 @@ fn pack_block<W: Write>(w: &mut W, kind: BT, buf: &[u8]) {
BT::NODE => check(&pack_btree_node(w, buf)),
BT::INDEX => check(&pack_index(w, buf)),
BT::BITMAP => check(&pack_bitmap(w, buf)),
BT::UNKNOWN => {panic!("asked to pack an unknown block type")}
BT::UNKNOWN => panic!("asked to pack an unknown block type"),
}
}

11
thin-provisioning/device_tree.cc
View File

@ -96,18 +96,21 @@ namespace thin_provisioning {
void
thin_provisioning::walk_device_tree(device_tree const &tree,
device_tree_detail::device_visitor &vv,
device_tree_detail::damage_visitor &dv)
device_tree_detail::damage_visitor &dv,
bool ignore_non_fatal)
{
visitor_adapter av(vv);
ll_damage_visitor ll_dv(dv);
btree_visit_values(tree, av, ll_dv);
btree_visit_values(tree, av, ll_dv, ignore_non_fatal);
}
void
thin_provisioning::check_device_tree(device_tree const &tree, damage_visitor &visitor)
thin_provisioning::check_device_tree(device_tree const &tree,
damage_visitor &visitor,
bool ignore_non_fatal)
{
noop_visitor vv;
walk_device_tree(tree, vv, visitor);
walk_device_tree(tree, vv, visitor, ignore_non_fatal);
}
//----------------------------------------------------------------

6
thin-provisioning/device_tree.h
View File

@ -75,9 +75,11 @@ namespace thin_provisioning {
void walk_device_tree(device_tree const &tree,
device_tree_detail::device_visitor &dev_v,
device_tree_detail::damage_visitor &dv);
device_tree_detail::damage_visitor &dv,
bool ignore_non_fatal = false);
void check_device_tree(device_tree const &tree,
device_tree_detail::damage_visitor &visitor);
device_tree_detail::damage_visitor &visitor,
bool ignore_non_fatal = false);
}
//----------------------------------------------------------------

30
thin-provisioning/mapping_tree.cc
View File

@ -228,54 +228,60 @@ namespace {
void
thin_provisioning::walk_mapping_tree(dev_tree const &tree,
mapping_tree_detail::device_visitor &dev_v,
mapping_tree_detail::damage_visitor &dv)
mapping_tree_detail::damage_visitor &dv,
bool ignore_non_fatal)
{
dev_tree_damage_visitor ll_dv(dv);
btree_visit_values(tree, dev_v, ll_dv);
btree_visit_values(tree, dev_v, ll_dv, ignore_non_fatal);
}
void
thin_provisioning::check_mapping_tree(dev_tree const &tree,
mapping_tree_detail::damage_visitor &visitor)
mapping_tree_detail::damage_visitor &visitor,
bool ignore_non_fatal)
{
noop_block_visitor dev_v;
walk_mapping_tree(tree, dev_v, visitor);
walk_mapping_tree(tree, dev_v, visitor, ignore_non_fatal);
}
void
thin_provisioning::walk_mapping_tree(mapping_tree const &tree,
mapping_tree_detail::mapping_visitor &mv,
mapping_tree_detail::damage_visitor &dv)
mapping_tree_detail::damage_visitor &dv,
bool ignore_non_fatal)
{
mapping_tree_damage_visitor ll_dv(dv);
btree_visit_values(tree, mv, ll_dv);
btree_visit_values(tree, mv, ll_dv, ignore_non_fatal);
}
void
thin_provisioning::check_mapping_tree(mapping_tree const &tree,
mapping_tree_detail::damage_visitor &visitor)
mapping_tree_detail::damage_visitor &visitor,
bool ignore_non_fatal)
{
noop_block_time_visitor mv;
walk_mapping_tree(tree, mv, visitor);
walk_mapping_tree(tree, mv, visitor, ignore_non_fatal);
}
void
thin_provisioning::walk_mapping_tree(single_mapping_tree const &tree,
uint64_t dev_id,
mapping_tree_detail::mapping_visitor &mv,
mapping_tree_detail::damage_visitor &dv)
mapping_tree_detail::damage_visitor &dv,
bool ignore_non_fatal)
{
single_mapping_tree_damage_visitor ll_dv(dv, dev_id);
btree_visit_values(tree, mv, ll_dv);
btree_visit_values(tree, mv, ll_dv, ignore_non_fatal);
}
void
thin_provisioning::check_mapping_tree(single_mapping_tree const &tree,
uint64_t dev_id,
mapping_tree_detail::damage_visitor &visitor)
mapping_tree_detail::damage_visitor &visitor,
bool ignore_non_fatal)
{
noop_block_time_visitor mv;
walk_mapping_tree(tree, dev_id, mv, visitor);
walk_mapping_tree(tree, dev_id, mv, visitor, ignore_non_fatal);
}
//----------------------------------------------------------------

25
thin-provisioning/mapping_tree.h
View File

@ -129,23 +129,32 @@ namespace thin_provisioning {
void walk_mapping_tree(dev_tree const &tree,
mapping_tree_detail::device_visitor &dev_v,
mapping_tree_detail::damage_visitor &dv);
void check_mapping_tree(dev_tree const &tree,
mapping_tree_detail::damage_visitor &visitor);
mapping_tree_detail::damage_visitor &dv,
bool ignore_non_fatal = false);
void walk_mapping_tree(mapping_tree const &tree,
mapping_tree_detail::mapping_visitor &mv,
mapping_tree_detail::damage_visitor &dv);
void check_mapping_tree(mapping_tree const &tree,
mapping_tree_detail::damage_visitor &visitor);
mapping_tree_detail::damage_visitor &dv,
bool ignore_non_fatal = false);
void walk_mapping_tree(single_mapping_tree const &tree,
uint64_t dev_id,
mapping_tree_detail::mapping_visitor &mv,
mapping_tree_detail::damage_visitor &dv);
mapping_tree_detail::damage_visitor &dv,
bool ignore_non_fatal = false);
void check_mapping_tree(single_mapping_tree const &tree,
uint64_t dev_id,
mapping_tree_detail::damage_visitor &visitor);
mapping_tree_detail::damage_visitor &visitor,
bool ignore_non_fatal = false);
void check_mapping_tree(dev_tree const &tree,
mapping_tree_detail::damage_visitor &visitor,
bool ignore_non_fatal = false);
void check_mapping_tree(mapping_tree const &tree,
mapping_tree_detail::damage_visitor &visitor,
bool ignore_non_fatal = false);
}
//----------------------------------------------------------------

41
thin-provisioning/metadata_checker.cc
View File

@ -150,28 +150,30 @@ namespace {
error_state examine_devices_tree_(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
nested_output &out) {
nested_output &out,
bool ignore_non_fatal) {
out << "examining devices tree" << end_message();
nested_output::nest _ = out.push();
devices_reporter dev_rep(out);
device_tree dtree(*tm, sb.device_details_root_,
device_tree_detail::device_details_traits::ref_counter());
check_device_tree(dtree, dev_rep);
check_device_tree(dtree, dev_rep, ignore_non_fatal);
return dev_rep.get_error();
}
error_state examine_top_level_mapping_tree_(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
nested_output &out) {
nested_output &out,
bool ignore_non_fatal) {
out << "examining top level of mapping tree" << end_message();
nested_output::nest _ = out.push();
mapping_reporter mapping_rep(out);
dev_tree dtree(*tm, sb.data_mapping_root_,
mapping_tree_detail::mtree_traits::ref_counter(*tm));
check_mapping_tree(dtree, mapping_rep);
check_mapping_tree(dtree, mapping_rep, ignore_non_fatal);
return mapping_rep.get_error();
}
@ -179,7 +181,8 @@ namespace {
error_state examine_mapping_tree_(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
nested_output &out,
optional<space_map::ptr> data_sm) {
optional<space_map::ptr> data_sm,
bool ignore_non_fatal) {
out << "examining mapping tree" << end_message();
nested_output::nest _ = out.push();
@ -189,18 +192,19 @@ namespace {
if (data_sm) {
data_ref_counter dcounter(*data_sm);
walk_mapping_tree(mtree, dcounter, mapping_rep);
walk_mapping_tree(mtree, dcounter, mapping_rep, ignore_non_fatal);
} else
check_mapping_tree(mtree, mapping_rep);
check_mapping_tree(mtree, mapping_rep, ignore_non_fatal);
return mapping_rep.get_error();
}
error_state examine_top_level_mapping_tree(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
nested_output &out) {
error_state err = examine_devices_tree_(tm, sb, out);
err << examine_top_level_mapping_tree_(tm, sb, out);
nested_output &out,
bool ignore_non_fatal) {
error_state err = examine_devices_tree_(tm, sb, out, ignore_non_fatal);
err << examine_top_level_mapping_tree_(tm, sb, out, ignore_non_fatal);
return err;
}
@ -208,9 +212,10 @@ namespace {
error_state examine_mapping_tree(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
nested_output &out,
optional<space_map::ptr> data_sm) {
error_state err = examine_devices_tree_(tm, sb, out);
err << examine_mapping_tree_(tm, sb, out, data_sm);
optional<space_map::ptr> data_sm,
bool ignore_non_fatal) {
error_state err = examine_devices_tree_(tm, sb, out, ignore_non_fatal);
err << examine_mapping_tree_(tm, sb, out, data_sm, ignore_non_fatal);
return err;
}
@ -351,7 +356,7 @@ namespace {
}
private:
static error_state
error_state
examine_data_mappings(transaction_manager::ptr tm,
superblock_detail::superblock const &sb,
check_options::data_mapping_options option,
@ -361,10 +366,10 @@ namespace {
switch (option) {
case check_options::DATA_MAPPING_LEVEL1:
err << examine_top_level_mapping_tree(tm, sb, out);
err << examine_top_level_mapping_tree(tm, sb, out, options_.ignore_non_fatal_);
break;
case check_options::DATA_MAPPING_LEVEL2:
err << examine_mapping_tree(tm, sb, out, data_sm);
err << examine_mapping_tree(tm, sb, out, data_sm, options_.ignore_non_fatal_);
break;
default:
break; // do nothing
@ -424,6 +429,10 @@ void check_options::set_metadata_snap() {
use_metadata_snap_ = true;
sm_opts_ = SPACE_MAP_NONE;
}
void check_options::set_ignore_non_fatal() {
ignore_non_fatal_ = true;
}
base::error_state
thin_provisioning::check_metadata(block_manager::ptr bm,

2
thin-provisioning/metadata_checker.h
View File

@ -44,11 +44,13 @@ namespace thin_provisioning {
void set_skip_mappings();
void set_override_mapping_root(bcache::block_address b);
void set_metadata_snap();
void set_ignore_non_fatal();
bool use_metadata_snap_;
data_mapping_options check_data_mappings_;
space_map_options sm_opts_;
boost::optional<bcache::block_address> override_mapping_root_;
bool ignore_non_fatal_;
};
enum output_options {

18
thin-provisioning/metadata_dumper.cc
View File

@ -132,7 +132,7 @@ namespace {
auto tree = device_tree(tm, root, device_tree_detail::device_details_traits::ref_counter());
try {
walk_device_tree(tree, de, dv);
walk_device_tree(tree, de, dv, true);
} catch (...) {
return optional<set<uint32_t>>();
}
@ -157,7 +157,7 @@ namespace {
auto tree = dev_tree(tm, root, mapping_tree_detail::mtree_traits::ref_counter(tm));
try {
walk_mapping_tree(tree, me, mv);
walk_mapping_tree(tree, me, mv, true);
} catch (...) {
return optional<set<uint32_t>>();
}
@ -614,8 +614,8 @@ namespace {
return it->second;
} else {
node_info info = get_info_(b);
examined_[b] = true;
node_info info = get_info_(b);
if (!failed(info))
infos_.insert(make_pair(b, info));
@ -754,7 +754,7 @@ namespace {
// Since we're not mutating the btrees we don't need a real space map
noop_map::ptr sm(new noop_map);
single_mapping_tree tree(tm_, subtree_root, mapping_tree_detail::block_time_ref_counter(sm));
walk_mapping_tree(tree, dev_id, static_cast<mapping_tree_detail::mapping_visitor &>(me), *damage_policy_);
walk_mapping_tree(tree, dev_id, static_cast<mapping_tree_detail::mapping_visitor &>(me), *damage_policy_, true);
}
dump_options const &opts_;
@ -785,7 +785,7 @@ namespace {
dump_options opts;
details_extractor de(opts);
device_tree_detail::damage_visitor::ptr dd_policy(details_damage_policy(true));
walk_device_tree(*md.details_, de, *dd_policy);
walk_device_tree(*md.details_, de, *dd_policy, true);
e->begin_superblock("", sb.time_,
sb.trans_id_,
@ -798,7 +798,7 @@ namespace {
{
mapping_tree_detail::damage_visitor::ptr md_policy(mapping_damage_policy(true));
mapping_tree_emit_visitor mte(opts, *md.tm_, e, de.get_details(), mapping_damage_policy(true));
walk_mapping_tree(*md.mappings_top_level_, mte, *md_policy);
walk_mapping_tree(*md.mappings_top_level_, mte, *md_policy, true);
}
e->end_superblock();
@ -882,7 +882,7 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, dump_options
{
details_extractor de(opts);
device_tree_detail::damage_visitor::ptr dd_policy(details_damage_policy(false));
walk_device_tree(*md->details_, de, *dd_policy);
walk_device_tree(*md->details_, de, *dd_policy, true);
e->begin_superblock("", md->sb_.time_,
md->sb_.trans_id_,
@ -895,7 +895,7 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, dump_options
{
mapping_tree_detail::damage_visitor::ptr md_policy(mapping_damage_policy(false));
mapping_tree_emit_visitor mte(opts, *md->tm_, e, de.get_details(), mapping_damage_policy(false));
walk_mapping_tree(*md->mappings_top_level_, mte, *md_policy);
walk_mapping_tree(*md->mappings_top_level_, mte, *md_policy, true);
}
e->end_superblock();
@ -924,7 +924,7 @@ thin_provisioning::metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool
mapping_tree_detail::block_time_ref_counter(md->data_sm_));
// FIXME: pass the current device id instead of zero
walk_mapping_tree(tree, 0, static_cast<mapping_tree_detail::mapping_visitor &>(me),
*mapping_damage_policy(repair));
*mapping_damage_policy(repair), true);
}
//----------------------------------------------------------------

1
thin-provisioning/thin_check.cc
View File

@ -174,6 +174,7 @@ thin_check_cmd::run(int argc, char **argv)
case 3:
// ignore-non-fatal-errors
fs.ignore_non_fatal_errors = true;
fs.check_opts.set_ignore_non_fatal();
break;
case 4: