From 09ce099c2f184924e1b9be0617d4a9ce6de4481c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 17 Feb 2016 10:42:42 +0000 Subject: [PATCH 01/25] [thin_delta] set nr_data_blocks to zero if there are no space maps. ie. a metadata_snap is being used. --- thin-provisioning/thin_delta.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 3ec71c8..3b33c73 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -527,7 +527,7 @@ namespace local { mapping_recorder mr2; damage_visitor damage_v; superblock_detail::superblock sb; - checked_space_map::ptr data_sm; + block_address nr_data_blocks = 0ull; { block_manager<>::ptr bm = open_bm(*fs.dev, block_manager<>::READ_ONLY, !fs.use_metadata_snap); @@ -562,13 +562,16 @@ namespace local { btree_visit_values(snap2, mr2, damage_v); mr2.complete(); + + if (md->data_sm_) + nr_data_blocks = md->data_sm_->get_nr_blocks(); } indented_stream is(cout); begin_superblock(is, "", sb.time_, sb.trans_id_, sb.data_block_size_, - data_sm->get_nr_blocks(), + nr_data_blocks, sb.metadata_snap_ ? boost::optional(sb.metadata_snap_) : boost::optional()); From 768bbd2715fccbfeef214b555abfe3a1dfe20b86 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 17 Feb 2016 11:44:44 +0000 Subject: [PATCH 02/25] [thin_delta] Fix bug when comparing the mappings --- thin-provisioning/thin_delta.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 3b33c73..61f910f 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -432,7 +432,8 @@ namespace local { mapping left_mapping; mapping right_mapping; - while (left_it != left.end() && right_it != right.end()) { + while ((left_mapping.len_ || left_it != left.end()) && + (right_mapping.len_ || right_it != right.end())) { if (!left_mapping.len_ && left_it != left.end()) left_mapping = *left_it++; From e4c5e5afb5a09b43937ac860b86c227f380095aa Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 17 Feb 2016 11:50:59 +0000 Subject: [PATCH 03/25] v0.6.2-rc4 --- CHANGES | 5 +++-- VERSION | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 567f5b4..9a99f62 100644 --- a/CHANGES +++ b/CHANGES @@ -1,8 +1,9 @@ v0.6.2 ====== -Fix recent regression in thin_repair. -Force g++-98 dialect +- Fix bug in thin_delta +- Fix recent regression in thin_repair. +- Force g++-98 dialect v0.6.1 ====== diff --git a/VERSION b/VERSION index e6d527c..5fb98bf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.2-rc3 +0.6.2-rc4 From 2461cdce43dc1d620ac910a39525948bbd25c002 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 17 Feb 2016 15:17:02 +0000 Subject: [PATCH 04/25] [thin_delta] tidy up the comparison function --- thin-provisioning/thin_delta.cc | 123 +++++++++++++++++++------------- 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 61f910f..9d6337a 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -103,13 +103,6 @@ namespace local { len_(len) { } - void consume(uint64_t delta) { - delta = min(delta, len_); - vbegin_ += delta; - dbegin_ += delta; - len_ -= delta; - } - uint64_t vbegin_, dbegin_, len_; }; @@ -120,6 +113,51 @@ namespace local { return out; } + //-------------------------------- + + template + class mapping_stream { + public: + mapping_stream(Container const &c) + : it_(c.begin()), + end_(c.end()) { + m_ = *it_; + } + + mapping const &get_mapping() const { + return m_; + } + + bool more_mappings() const { + return it_ != end_; + } + + void consume(uint64_t delta) { + if (it_ == end_) + throw runtime_error("end of stream already reached"); + + if (delta > m_.len_) + throw runtime_error("delta too long"); + + if (delta == m_.len_) { + ++it_; + m_ = *it_; + + } else { + m_.vbegin_ += delta; + m_.dbegin_ += delta; + m_.len_ -= delta; + } + } + + private: + typename Container::const_iterator it_; + typename Container::const_iterator end_; + mapping m_; + }; + + //-------------------------------- + typedef std::deque mapping_deque; // Builds up an in core rep of the mappings for a device. @@ -426,58 +464,45 @@ namespace local { // We iterate through both sets of mappings in parallel // noting any differences. - mapping_deque::const_iterator left_it = left.begin(); - mapping_deque::const_iterator right_it = right.begin(); + mapping_stream ls{left}; + mapping_stream rs{right}; - mapping left_mapping; - mapping right_mapping; + while (ls.more_mappings() && rs.more_mappings()) { + auto &lm = ls.get_mapping(); + auto &rm = rs.get_mapping(); - while ((left_mapping.len_ || left_it != left.end()) && - (right_mapping.len_ || right_it != right.end())) { - if (!left_mapping.len_ && left_it != left.end()) - left_mapping = *left_it++; + if (lm.vbegin_ < rm.vbegin_) { + auto delta = min(lm.len_, rm.vbegin_ - lm.vbegin_); + e.left_only(lm.vbegin_, lm.dbegin_, delta); + ls.consume(delta); - if (!right_mapping.len_ && right_it != right.end()) - right_mapping = *right_it++; + } else if (lm.vbegin_ > rm.vbegin_) { + auto delta = min(rm.len_, lm.vbegin_ - rm.vbegin_); + e.right_only(rm.vbegin_, rm.dbegin_, delta); + rs.consume(delta); - while (left_mapping.len_ && right_mapping.len_) { - if (left_mapping.vbegin_ < right_mapping.vbegin_) { - uint64_t delta = min(left_mapping.len_, right_mapping.vbegin_ - left_mapping.vbegin_); - e.left_only(left_mapping.vbegin_, left_mapping.dbegin_, delta); - left_mapping.consume(delta); + } else if (lm.dbegin_ != rm.dbegin_) { + auto delta = min(lm.len_, rm.len_); + e.blocks_differ(lm.vbegin_, lm.dbegin_, rm.dbegin_, delta); + ls.consume(delta); + rs.consume(delta); - } else if (left_mapping.vbegin_ > right_mapping.vbegin_) { - uint64_t delta = min(right_mapping.len_, left_mapping.vbegin_ - right_mapping.vbegin_); - e.right_only(right_mapping.vbegin_, right_mapping.dbegin_, delta); - right_mapping.consume(delta); - - } else if (left_mapping.dbegin_ != right_mapping.dbegin_) { - uint64_t delta = min(left_mapping.len_, right_mapping.len_); - e.blocks_differ(left_mapping.vbegin_, left_mapping.dbegin_, right_mapping.dbegin_, delta); - left_mapping.consume(delta); - right_mapping.consume(delta); - - } else { - uint64_t delta = min(left_mapping.len_, right_mapping.len_); - e.blocks_same(left_mapping.vbegin_, left_mapping.dbegin_, delta); - left_mapping.consume(delta); - right_mapping.consume(delta); - } + } else { + auto delta = min(lm.len_, rm.len_); + e.blocks_same(lm.vbegin_, lm.dbegin_, delta); + ls.consume(delta); + rs.consume(delta); } } - while (left_it != left.end()) { - left_mapping = *left_it++; - - if (left_mapping.len_) - e.left_only(left_mapping.vbegin_, left_mapping.dbegin_, left_mapping.len_); + while (ls.more_mappings()) { + auto &lm = ls.get_mapping(); + e.left_only(lm.vbegin_, lm.dbegin_, lm.len_); } - while (right_it != right.end()) { - right_mapping = *right_it++; - - if (right_mapping.len_) - e.right_only(right_mapping.vbegin_, right_mapping.dbegin_, right_mapping.len_); + while (rs.more_mappings()) { + auto &rm = rs.get_mapping(); + e.right_only(rm.vbegin_, rm.dbegin_, rm.len_); } e.complete(); From a21cee55c74b1ef7a0a3b10e1bf7777ac2cda310 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 18 Feb 2016 11:31:43 +0000 Subject: [PATCH 05/25] [thin-delta] fix bug in earlier refactor --- thin-provisioning/thin_delta.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 9d6337a..346d62c 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -498,11 +498,13 @@ namespace local { while (ls.more_mappings()) { auto &lm = ls.get_mapping(); e.left_only(lm.vbegin_, lm.dbegin_, lm.len_); + ls.consume(lm.len_); } while (rs.more_mappings()) { auto &rm = rs.get_mapping(); e.right_only(rm.vbegin_, rm.dbegin_, rm.len_); + rs.consume(rm.len_); } e.complete(); From 0ec27c7835efa8c5b16ce9ecca5438f87dc3260b Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 18 Feb 2016 11:37:51 +0000 Subject: [PATCH 06/25] [thin_delta] backport cherry-picked changes to c++98 --- thin-provisioning/thin_delta.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/thin-provisioning/thin_delta.cc b/thin-provisioning/thin_delta.cc index 346d62c..bc40f76 100644 --- a/thin-provisioning/thin_delta.cc +++ b/thin-provisioning/thin_delta.cc @@ -464,31 +464,31 @@ namespace local { // We iterate through both sets of mappings in parallel // noting any differences. - mapping_stream ls{left}; - mapping_stream rs{right}; + mapping_stream ls(left); + mapping_stream rs(right); while (ls.more_mappings() && rs.more_mappings()) { - auto &lm = ls.get_mapping(); - auto &rm = rs.get_mapping(); + mapping const &lm = ls.get_mapping(); + mapping const &rm = rs.get_mapping(); if (lm.vbegin_ < rm.vbegin_) { - auto delta = min(lm.len_, rm.vbegin_ - lm.vbegin_); + uint64_t delta = min(lm.len_, rm.vbegin_ - lm.vbegin_); e.left_only(lm.vbegin_, lm.dbegin_, delta); ls.consume(delta); } else if (lm.vbegin_ > rm.vbegin_) { - auto delta = min(rm.len_, lm.vbegin_ - rm.vbegin_); + uint64_t delta = min(rm.len_, lm.vbegin_ - rm.vbegin_); e.right_only(rm.vbegin_, rm.dbegin_, delta); rs.consume(delta); } else if (lm.dbegin_ != rm.dbegin_) { - auto delta = min(lm.len_, rm.len_); + uint64_t delta = min(lm.len_, rm.len_); e.blocks_differ(lm.vbegin_, lm.dbegin_, rm.dbegin_, delta); ls.consume(delta); rs.consume(delta); } else { - auto delta = min(lm.len_, rm.len_); + uint64_t delta = min(lm.len_, rm.len_); e.blocks_same(lm.vbegin_, lm.dbegin_, delta); ls.consume(delta); rs.consume(delta); @@ -496,13 +496,13 @@ namespace local { } while (ls.more_mappings()) { - auto &lm = ls.get_mapping(); + mapping const &lm = ls.get_mapping(); e.left_only(lm.vbegin_, lm.dbegin_, lm.len_); ls.consume(lm.len_); } while (rs.more_mappings()) { - auto &rm = rs.get_mapping(); + mapping const &rm = rs.get_mapping(); e.right_only(rm.vbegin_, rm.dbegin_, rm.len_); rs.consume(rm.len_); } From c8276a41f7a35eb6859782dad297a60e138ee522 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 22 Feb 2016 12:05:17 +0000 Subject: [PATCH 07/25] v0.6.2-rc5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5fb98bf..5070360 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.2-rc4 +0.6.2-rc5 From 11cd796652523f4c6a4d23fd121ee0dac34a9f25 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 01:27:27 +0800 Subject: [PATCH 08/25] Show the block address if block_cache::get() failed --- block-cache/block_cache.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/block-cache/block_cache.cc b/block-cache/block_cache.cc index 6ecce1f..e7e6f14 100644 --- a/block-cache/block_cache.cc +++ b/block-cache/block_cache.cc @@ -620,7 +620,9 @@ block_cache::get(block_address index, unsigned flags, validator::ptr v) return *b; } - throw std::runtime_error("couldn't get block"); + std::ostringstream out; + out << "couldn't get block " << index; + throw std::runtime_error(out.str()); } void From b1d4b9f7c8e9953b5222278a137fc6b115444364 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 01:28:41 +0800 Subject: [PATCH 09/25] Show the block address of error nodes --- .../data-structures/btree_damage_visitor.h | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index fa11378..f0368bb 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -300,14 +300,16 @@ namespace persistent_data { size_t elt_size = sizeof(uint64_t) + n.get_value_size(); if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { std::ostringstream out; - out << "max entries too large: " << n.get_max_entries(); + out << "max entries too large: " << n.get_max_entries() + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } if (n.get_max_entries() % 3) { std::ostringstream out; - out << "max entries is not divisible by 3: " << n.get_max_entries(); + out << "max entries is not divisible by 3: " << n.get_max_entries() + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } @@ -321,7 +323,8 @@ namespace persistent_data { std::ostringstream out; out << "bad nr_entries: " << n.get_nr_entries() << " < " - << n.get_max_entries(); + << n.get_max_entries() + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } @@ -333,7 +336,8 @@ namespace persistent_data { << n.get_nr_entries() << ", expected at least " << min - << "(max_entries = " << n.get_max_entries() << ")"; + << " (block " << n.get_location() + << ", max_entries = " << n.get_max_entries() << ")"; report_damage(out.str()); return false; } @@ -354,7 +358,8 @@ namespace persistent_data { uint64_t k = n.key_at(i); if (k <= last_key) { ostringstream out; - out << "keys are out of order, " << k << " <= " << last_key; + out << "keys are out of order, " << k << " <= " << last_key + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } @@ -372,7 +377,8 @@ namespace persistent_data { if (*key > n.key_at(0)) { ostringstream out; out << "parent key mismatch: parent was " << *key - << ", but lowest in node was " << n.key_at(0); + << ", but lowest in node was " << n.key_at(0) + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } @@ -388,7 +394,8 @@ namespace persistent_data { if (last_leaf_key_[level] && *last_leaf_key_[level] >= n.key_at(0)) { ostringstream out; out << "the last key of the previous leaf was " << *last_leaf_key_[level] - << " and the first key of this leaf is " << n.key_at(0); + << " and the first key of this leaf is " << n.key_at(0) + << " (block " << n.get_location() << ")"; report_damage(out.str()); return false; } From d2260dee34f80844f6a2468c753d0d2a239f47c1 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 01:30:45 +0800 Subject: [PATCH 10/25] Show the block address in exception string --- persistent-data/data-structures/btree.tcc | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/persistent-data/data-structures/btree.tcc b/persistent-data/data-structures/btree.tcc index 80f2b94..06d0deb 100644 --- a/persistent-data/data-structures/btree.tcc +++ b/persistent-data/data-structures/btree.tcc @@ -90,14 +90,22 @@ namespace persistent_data { { uint32_t flags = to_cpu(raw_->header.flags); if (flags & INTERNAL_NODE) { - if (flags & LEAF_NODE) - throw runtime_error("btree node is both internal and leaf"); + if (flags & LEAF_NODE) { + ostringstream out; + out << "btree node is both internal and leaf" + << " (block " << location_ << ")"; + throw runtime_error(out.str()); + } return INTERNAL; } else if (flags & LEAF_NODE) return LEAF; - else - throw runtime_error("unknown node type"); + else { + ostringstream out; + out << "unknown node type" + << " (block " << location_ << ")"; + throw runtime_error(out.str()); + } } template @@ -352,7 +360,8 @@ namespace persistent_data { std::ostringstream out; out << "value size mismatch: expected " << sizeof(typename ValueTraits::disk_type) << ", but got " << get_value_size() - << ". This is not the btree you are looking for." << std::endl; + << ". This is not the btree you are looking for." + << " (block " << location_ << ")" << std::endl; return out.str(); } @@ -371,7 +380,8 @@ namespace persistent_data { if (max < get_nr_entries()) { std::ostringstream out; out << "Bad nr of elements: max per block = " - << max << ", actual = " << get_nr_entries() << std::endl; + << max << ", actual = " << get_nr_entries() + << " (block " << location_ << ")" << std::endl; throw std::runtime_error(out.str()); } From b47c02ed8bbd1623a5db9936b33f6d80306d53b9 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 01:31:24 +0800 Subject: [PATCH 11/25] Show the wanted checksum in bad-superblock-checksum exception --- thin-provisioning/superblock.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/thin-provisioning/superblock.cc b/thin-provisioning/superblock.cc index 9258b53..fb0d17e 100644 --- a/thin-provisioning/superblock.cc +++ b/thin-provisioning/superblock.cc @@ -91,8 +91,11 @@ namespace { superblock_disk const *sbd = reinterpret_cast(raw); crc32c sum(SUPERBLOCK_CSUM_SEED); sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t)); - if (sum.get_sum() != to_cpu(sbd->csum_)) - throw checksum_error("bad checksum in superblock"); + if (sum.get_sum() != to_cpu(sbd->csum_)) { + ostringstream out; + out << "bad checksum in superblock, wanted " << sum.get_sum(); + throw checksum_error(out.str()); + } } virtual void prepare(void *raw, block_address location) const { From 7ec47158b56eacf2d9642997adb13adca6638a84 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:20:02 +0800 Subject: [PATCH 12/25] [emitter] Preserve the flags and version entries in superblock --- thin-provisioning/emitter.h | 2 ++ thin-provisioning/human_readable_format.cc | 4 ++++ thin-provisioning/metadata_dumper.cc | 2 ++ thin-provisioning/restore_emitter.cc | 4 ++++ thin-provisioning/xml_format.cc | 6 ++++++ 5 files changed, 18 insertions(+) diff --git a/thin-provisioning/emitter.h b/thin-provisioning/emitter.h index 58658a9..bb944da 100644 --- a/thin-provisioning/emitter.h +++ b/thin-provisioning/emitter.h @@ -49,6 +49,8 @@ namespace thin_provisioning { virtual void begin_superblock(std::string const &uuid, uint64_t time, uint64_t trans_id, + boost::optional flags, + boost::optional version, uint32_t data_block_size, uint64_t nr_data_blocks, boost::optional metadata_snap) = 0; diff --git a/thin-provisioning/human_readable_format.cc b/thin-provisioning/human_readable_format.cc index 3cfc188..6c53b3c 100644 --- a/thin-provisioning/human_readable_format.cc +++ b/thin-provisioning/human_readable_format.cc @@ -43,12 +43,16 @@ namespace { void begin_superblock(string const &uuid, uint64_t time, uint64_t trans_id, + boost::optional flags, + boost::optional version, uint32_t data_block_size, uint64_t nr_data_blocks, boost::optional metadata_snap) { out_ << "begin superblock: \"" << uuid << "\"" << ", " << time << ", " << trans_id + << ", " << (flags ? *flags : 0) + << ", " << (version ? *version : 1) << ", " << data_block_size << ", " << nr_data_blocks; if (metadata_snap) diff --git a/thin-provisioning/metadata_dumper.cc b/thin-provisioning/metadata_dumper.cc index 0c0fbe4..5006fc0 100644 --- a/thin-provisioning/metadata_dumper.cc +++ b/thin-provisioning/metadata_dumper.cc @@ -228,6 +228,8 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair) e->begin_superblock("", md->sb_.time_, md->sb_.trans_id_, + md->sb_.flags_, + md->sb_.version_, md->sb_.data_block_size_, nr_data_blocks, boost::optional()); diff --git a/thin-provisioning/restore_emitter.cc b/thin-provisioning/restore_emitter.cc index fd83f56..2fab704 100644 --- a/thin-provisioning/restore_emitter.cc +++ b/thin-provisioning/restore_emitter.cc @@ -44,6 +44,8 @@ namespace { virtual void begin_superblock(std::string const &uuid, uint64_t time, uint64_t trans_id, + boost::optional flags, + boost::optional version, uint32_t data_block_size, uint64_t nr_data_blocks, boost::optional metadata_snap) { @@ -54,6 +56,8 @@ namespace { memcpy(&sb.uuid_, uuid.c_str(), std::min(sizeof(sb.uuid_), uuid.length())); sb.time_ = time; sb.trans_id_ = trans_id; + sb.flags_ = flags ? *flags : 0; + sb.version_ = version ? *version : 1; sb.data_block_size_ = data_block_size; sb.metadata_snap_ = metadata_snap ? *metadata_snap : 0; md_->data_sm_->extend(nr_data_blocks); diff --git a/thin-provisioning/xml_format.cc b/thin-provisioning/xml_format.cc index 333204f..efbfd3f 100644 --- a/thin-provisioning/xml_format.cc +++ b/thin-provisioning/xml_format.cc @@ -50,6 +50,8 @@ namespace { void begin_superblock(string const &uuid, uint64_t time, uint64_t trans_id, + boost::optional flags, + boost::optional version, uint32_t data_block_size, uint64_t nr_data_blocks, boost::optional metadata_snap) { @@ -57,6 +59,8 @@ namespace { out_ << "begin_superblock(get_attr(attr, "uuid"), get_attr(attr, "time"), get_attr(attr, "transaction"), + get_opt_attr(attr, "flags"), + get_opt_attr(attr, "version"), get_attr(attr, "data_block_size"), get_attr(attr, "nr_data_blocks"), get_opt_attr(attr, "metadata_snap")); From ced9929ca6d85b32b83d81fbc3148b94a5a10e87 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:20:31 +0800 Subject: [PATCH 13/25] [metadata_dumper] Support dump a bottom-level data mapping tree --- thin-provisioning/metadata_dumper.cc | 11 +++++++++++ thin-provisioning/metadata_dumper.h | 1 + 2 files changed, 12 insertions(+) diff --git a/thin-provisioning/metadata_dumper.cc b/thin-provisioning/metadata_dumper.cc index 5006fc0..9ba0614 100644 --- a/thin-provisioning/metadata_dumper.cc +++ b/thin-provisioning/metadata_dumper.cc @@ -244,3 +244,14 @@ thin_provisioning::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair) } //---------------------------------------------------------------- + +void +thin_provisioning::metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool repair, uint64_t subtree_root) { + mapping_emitter me(e); + single_mapping_tree tree(*md->tm_, subtree_root, + mapping_tree_detail::block_time_ref_counter(md->data_sm_)); + walk_mapping_tree(tree, static_cast(me), + *mapping_damage_policy(repair)); +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/metadata_dumper.h b/thin-provisioning/metadata_dumper.h index c96d22e..007256a 100644 --- a/thin-provisioning/metadata_dumper.h +++ b/thin-provisioning/metadata_dumper.h @@ -29,6 +29,7 @@ namespace thin_provisioning { // the dumper to do it's best to recover info. If not set, any // corruption encountered will cause an exception to be thrown. void metadata_dump(metadata::ptr md, emitter::ptr e, bool repair); + void metadata_dump_subtree(metadata::ptr md, emitter::ptr e, bool repair, uint64_t subtree_root); } //---------------------------------------------------------------- From d28e64aff027606254b8d546218a17cbc3315fa1 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:20:45 +0800 Subject: [PATCH 14/25] [xml_parser] Allow element handlers to stop parsing --- base/xml_utils.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/base/xml_utils.cc b/base/xml_utils.cc index fb34153..edbd16a 100644 --- a/base/xml_utils.cc +++ b/base/xml_utils.cc @@ -19,13 +19,16 @@ xml_parser::parse(std::string const &backup_file, bool quiet) size_t total = 0; size_t input_length = get_file_length(backup_file); - while (!in.eof()) { + XML_Error error_code = XML_ERROR_NONE; + while (!in.eof() && error_code == XML_ERROR_NONE) { char buffer[4096]; in.read(buffer, sizeof(buffer)); size_t len = in.gcount(); int done = in.eof(); - if (!XML_Parse(parser_, buffer, len, done)) { + // Do not throw while normally aborted by element handlers + if (!XML_Parse(parser_, buffer, len, done) && + (error_code = XML_GetErrorCode(parser_)) != XML_ERROR_ABORTED) { ostringstream out; out << "Parse error at line " << XML_GetCurrentLineNumber(parser_) From 992ad02ce93594b5bfcaf18579f4dcf88b54ce2a Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:20:56 +0800 Subject: [PATCH 15/25] [restore_emitter] Recount device_details::mapped_blocks_ --- thin-provisioning/restore_emitter.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/thin-provisioning/restore_emitter.cc b/thin-provisioning/restore_emitter.cc index 2fab704..f6ca294 100644 --- a/thin-provisioning/restore_emitter.cc +++ b/thin-provisioning/restore_emitter.cc @@ -82,10 +82,11 @@ namespace { if (device_exists(dev)) throw std::runtime_error("Device already exists"); - // Add entry to the details tree - uint64_t key[1] = {dev}; - device_tree_detail::device_details details = {mapped_blocks, trans_id, (uint32_t)creation_time, (uint32_t)snap_time}; - md_->details_->insert(key, details); + // Store the entry of the details tree + current_device_details_.mapped_blocks_ = 0; + current_device_details_.transaction_id_ = trans_id; + current_device_details_.creation_time_ = (uint32_t)creation_time; + current_device_details_.snapshotted_time_ = (uint32_t)snap_time; current_mapping_ = empty_mapping_->clone(); current_device_ = boost::optional(dev); @@ -94,6 +95,9 @@ namespace { virtual void end_device() { uint64_t key[1] = {*current_device_}; + // Add entry to the details tree + md_->details_->insert(key, current_device_details_); + md_->mappings_top_level_->insert(key, current_mapping_->get_root()); md_->mappings_->set_root(md_->mappings_top_level_->get_root()); // FIXME: ugly @@ -134,6 +138,8 @@ namespace { bt.time_ = time; current_mapping_->insert(key, bt); md_->data_sm_->inc(data_block); + + current_device_details_.mapped_blocks_ += 1; } private: @@ -153,6 +159,7 @@ namespace { bool in_superblock_; block_address nr_data_blocks_; boost::optional current_device_; + device_tree_detail::device_details current_device_details_; single_mapping_tree::ptr current_mapping_; single_mapping_tree::ptr empty_mapping_; }; From 45e9916428f3f13c7ad704d3ea8a46154600f851 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:05 +0800 Subject: [PATCH 16/25] Expose validator creation interface --- persistent-data/space-maps/disk.cc | 15 ++++++++++----- persistent-data/space-maps/disk.h | 6 ++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/persistent-data/space-maps/disk.cc b/persistent-data/space-maps/disk.cc index e915eb4..cab9583 100644 --- a/persistent-data/space-maps/disk.cc +++ b/persistent-data/space-maps/disk.cc @@ -87,11 +87,6 @@ namespace { } }; - bcache::validator::ptr - index_validator() { - return bcache::validator::ptr(new index_block_validator()); - } - //-------------------------------- class bitmap { @@ -773,4 +768,14 @@ persistent_data::open_metadata_sm(transaction_manager &tm, void *root) checked_space_map::ptr(new sm_disk(store, tm, v)))); } +bcache::validator::ptr +persistent_data::bitmap_validator() { + return bcache::validator::ptr(new bitmap_block_validator()); +} + +bcache::validator::ptr +persistent_data::index_validator() { + return bcache::validator::ptr(new index_block_validator()); +} + //---------------------------------------------------------------- diff --git a/persistent-data/space-maps/disk.h b/persistent-data/space-maps/disk.h index 0a69f04..775ef7b 100644 --- a/persistent-data/space-maps/disk.h +++ b/persistent-data/space-maps/disk.h @@ -36,6 +36,12 @@ namespace persistent_data { checked_space_map::ptr open_metadata_sm(transaction_manager &tm, void *root); + + bcache::validator::ptr + bitmap_validator(); + + bcache::validator::ptr + index_validator(); } //---------------------------------------------------------------- From 778c153c1e71cd779b856e705492ae59fb9955e2 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:13 +0800 Subject: [PATCH 17/25] [block-cache] Add check_raw() to bcache::validator --- block-cache/block_cache.h | 2 ++ caching/superblock.cc | 9 +++++++++ era/superblock.cc | 9 +++++++++ persistent-data/data-structures/array.h | 9 +++++++++ persistent-data/data-structures/btree.tcc | 1 - persistent-data/space-maps/disk.cc | 18 ++++++++++++++++++ persistent-data/validators.cc | 10 ++++++++++ thin-provisioning/superblock.cc | 9 +++++++++ unit-tests/block_t.cc | 8 ++++++++ 9 files changed, 74 insertions(+), 1 deletion(-) diff --git a/block-cache/block_cache.h b/block-cache/block_cache.h index 4bc6667..5de2c44 100644 --- a/block-cache/block_cache.h +++ b/block-cache/block_cache.h @@ -26,12 +26,14 @@ namespace bcache { virtual ~validator() {} virtual void check(void const *data, block_address location) const = 0; + virtual bool check_raw(void const *data) const = 0; virtual void prepare(void *data, block_address location) const = 0; }; class noop_validator : public validator { public: void check(void const *data, block_address location) const {} + bool check_raw(void const *data) const {return true;} void prepare(void *data, block_address location) const {} }; diff --git a/caching/superblock.cc b/caching/superblock.cc index 2edd6c1..3270b3f 100644 --- a/caching/superblock.cc +++ b/caching/superblock.cc @@ -292,6 +292,15 @@ namespace validator { throw checksum_error("bad checksum in superblock"); } + virtual bool check_raw(void const *raw) const { + superblock_disk const *sbd = reinterpret_cast(raw); + crc32c sum(SUPERBLOCK_CSUM_SEED); + sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(sbd->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { superblock_disk *sbd = reinterpret_cast(raw); crc32c sum(SUPERBLOCK_CSUM_SEED); diff --git a/era/superblock.cc b/era/superblock.cc index e013064..f61a542 100644 --- a/era/superblock.cc +++ b/era/superblock.cc @@ -219,6 +219,15 @@ namespace era_validator { throw checksum_error("bad checksum in superblock"); } + virtual bool check_raw(void const *raw) const { + superblock_disk const *sbd = reinterpret_cast(raw); + crc32c sum(SUPERBLOCK_CSUM_SEED); + sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(sbd->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { superblock_disk *sbd = reinterpret_cast(raw); crc32c sum(SUPERBLOCK_CSUM_SEED); diff --git a/persistent-data/data-structures/array.h b/persistent-data/data-structures/array.h index 1b87160..d7b1f70 100644 --- a/persistent-data/data-structures/array.h +++ b/persistent-data/data-structures/array.h @@ -43,6 +43,15 @@ namespace persistent_data { throw checksum_error("bad block nr in array block"); } + virtual bool check_raw(void const *raw) const { + array_block_disk const *data = reinterpret_cast(raw); + crc32c sum(ARRAY_CSUM_XOR); + sum.append(&data->max_entries, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(data->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { array_block_disk *data = reinterpret_cast(raw); data->blocknr = to_disk(location); diff --git a/persistent-data/data-structures/btree.tcc b/persistent-data/data-structures/btree.tcc index 06d0deb..3fc96e4 100644 --- a/persistent-data/data-structures/btree.tcc +++ b/persistent-data/data-structures/btree.tcc @@ -32,7 +32,6 @@ namespace { using namespace persistent_data; using namespace btree_detail; using namespace std; - } //---------------------------------------------------------------- diff --git a/persistent-data/space-maps/disk.cc b/persistent-data/space-maps/disk.cc index cab9583..ed7bae8 100644 --- a/persistent-data/space-maps/disk.cc +++ b/persistent-data/space-maps/disk.cc @@ -50,6 +50,15 @@ namespace { throw checksum_error("bad block nr in space map bitmap"); } + virtual bool check_raw(void const *raw) const { + bitmap_header const *data = reinterpret_cast(raw); + crc32c sum(BITMAP_CSUM_XOR); + sum.append(&data->not_used, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(data->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { bitmap_header *data = reinterpret_cast(raw); data->blocknr = to_disk(location); @@ -77,6 +86,15 @@ namespace { throw checksum_error("bad block nr in metadata index block"); } + virtual bool check_raw(void const *raw) const { + metadata_index const *mi = reinterpret_cast(raw); + crc32c sum(INDEX_CSUM_XOR); + sum.append(&mi->padding_, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(mi->csum_)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { metadata_index *mi = reinterpret_cast(raw); mi->blocknr_ = to_disk(location); diff --git a/persistent-data/validators.cc b/persistent-data/validators.cc index b9c163c..a50947d 100644 --- a/persistent-data/validators.cc +++ b/persistent-data/validators.cc @@ -31,6 +31,16 @@ namespace { } } + virtual bool check_raw(void const *raw) const { + disk_node const *data = reinterpret_cast(raw); + node_header const *n = &data->header; + crc32c sum(BTREE_CSUM_XOR); + sum.append(&n->flags, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(n->csum)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { disk_node *data = reinterpret_cast(raw); node_header *n = &data->header; diff --git a/thin-provisioning/superblock.cc b/thin-provisioning/superblock.cc index fb0d17e..f63ed52 100644 --- a/thin-provisioning/superblock.cc +++ b/thin-provisioning/superblock.cc @@ -98,6 +98,15 @@ namespace { } } + virtual bool check_raw(void const *raw) const { + superblock_disk const *sbd = reinterpret_cast(raw); + crc32c sum(SUPERBLOCK_CSUM_SEED); + sum.append(&sbd->flags_, MD_BLOCK_SIZE - sizeof(uint32_t)); + if (sum.get_sum() != to_cpu(sbd->csum_)) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { superblock_disk *sbd = reinterpret_cast(raw); crc32c sum(SUPERBLOCK_CSUM_SEED); diff --git a/unit-tests/block_t.cc b/unit-tests/block_t.cc index c2a6d58..1a61d2b 100644 --- a/unit-tests/block_t.cc +++ b/unit-tests/block_t.cc @@ -44,6 +44,14 @@ namespace { throw runtime_error("validator check zero"); } + virtual bool check_raw(void const *raw) const { + unsigned char const *data = reinterpret_cast(raw); + for (unsigned b = 0; b < BlockSize; b++) + if (data[b] != 0) + return false; + return true; + } + virtual void prepare(void *raw, block_address location) const { unsigned char *data = reinterpret_cast(raw); for (unsigned b = 0; b < BlockSize; b++) From a48227188e950c2f32ff5a5b64fa245844a2b969 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:23 +0800 Subject: [PATCH 18/25] Add utility class btree_detail::noop_damage_visitor --- persistent-data/data-structures/btree_damage_visitor.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index f0368bb..5a96d5d 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -27,6 +27,12 @@ namespace persistent_data { return out; } + class noop_damage_visitor { + public: + virtual void visit(btree_path const &path, damage const &d) { + } + }; + // Tracks damage in a single level btree. Use multiple // trackers if you have a multilayer tree. class damage_tracker { From c571a08f6ce90e14da82d155a85a240293f18f97 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:32 +0800 Subject: [PATCH 19/25] Add utility class btree_detail::noop_value_visitor --- .../data-structures/btree_base_visitor.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 persistent-data/data-structures/btree_base_visitor.h diff --git a/persistent-data/data-structures/btree_base_visitor.h b/persistent-data/data-structures/btree_base_visitor.h new file mode 100644 index 0000000..17519e1 --- /dev/null +++ b/persistent-data/data-structures/btree_base_visitor.h @@ -0,0 +1,17 @@ +#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_BASE_VISITOR_H +#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_BASE_VISITOR_H + +#include "persistent-data/data-structures/btree.h" + +namespace persistent_data { + namespace btree_detail { + template + class noop_value_visitor { + public: + virtual void visit(btree_path const &path, ValueType const &v) { + } + }; + } +} + +#endif From ad03114bf7703bb47bac508e4cab5a62792bb77a Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:21:44 +0800 Subject: [PATCH 20/25] Add utility class btree_detail::key_value_extractor --- .../btree_key_value_extractor.h | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 persistent-data/data-structures/btree_key_value_extractor.h diff --git a/persistent-data/data-structures/btree_key_value_extractor.h b/persistent-data/data-structures/btree_key_value_extractor.h new file mode 100644 index 0000000..40b7f7b --- /dev/null +++ b/persistent-data/data-structures/btree_key_value_extractor.h @@ -0,0 +1,39 @@ +#ifndef PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H +#define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_KEY_VALUE_EXTRACTOR_H + +#include "persistent-data/data-structures/btree_damage_visitor.h" +#include + +namespace persistent_data { + namespace btree_detail { + template + class key_value_extractor { + typedef typename std::map MapType; + public: + key_value_extractor(MapType &map): map_(map) { + } + + virtual ~key_value_extractor() { + } + + virtual void visit(btree_path const &path, ValueType const &v) { + map_.insert(std::make_pair(path.back(), v)); + } + private: + MapType &map_; + }; + + template + void btree_extract_key_values(btree const &tree, + std::map &map) { + typedef key_value_extractor KeyValueExtractor; + KeyValueExtractor kve(map); + noop_damage_visitor noop_dv; + btree_detail::btree_damage_visitor + v(kve, noop_dv); + tree.visit_depth_first(v); + } + } +} + +#endif From b22495997a8bf427741225ef86edf267ee746d7c Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:22:00 +0800 Subject: [PATCH 21/25] Allow counting_visitor to work with damaged btrees --- .../data-structures/btree_counter.h | 28 +++++++++++++------ .../data-structures/btree_damage_visitor.h | 5 ++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/persistent-data/data-structures/btree_counter.h b/persistent-data/data-structures/btree_counter.h index ed7b845..c215626 100644 --- a/persistent-data/data-structures/btree_counter.h +++ b/persistent-data/data-structures/btree_counter.h @@ -2,35 +2,44 @@ #define PERSISTENT_DATA_DATA_STRUCTURES_BTREE_COUNTER_H #include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/btree_base_visitor.h" +#include "persistent-data/data-structures/btree_damage_visitor.h" #include "persistent-data/block_counter.h" //---------------------------------------------------------------- namespace persistent_data { namespace btree_count_detail { - template - class counting_visitor : public btree::visitor { + template + class counting_visitor : public btree_damage_visitor { + typedef btree_damage_visitor BtreeDamageVisitor; public: typedef btree tree; - counting_visitor(block_counter &bc, ValueCounter &vc) - : bc_(bc), + counting_visitor(ValueVisitor &value_visitor, + DamageVisitor &damage_visitor, + block_counter &bc, + ValueCounter &vc) + : BtreeDamageVisitor(value_visitor, damage_visitor, false), + bc_(bc), vc_(vc) { } virtual bool visit_internal(node_location const &l, typename tree::internal_node const &n) { - return visit_node(n); + return BtreeDamageVisitor::visit_internal(l, n) ? + visit_node(n) : false; } virtual bool visit_internal_leaf(node_location const &l, typename tree::internal_node const &n) { - return visit_node(n); + return BtreeDamageVisitor::visit_internal_leaf(l, n) ? + visit_node(n) : false; } virtual bool visit_leaf(node_location const &l, typename tree::leaf_node const &n) { - if (visit_node(n)) { + if (BtreeDamageVisitor::visit_leaf(l, n) && visit_node(n)) { unsigned nr = n.get_nr_entries(); for (unsigned i = 0; i < nr; i++) { @@ -85,7 +94,10 @@ namespace persistent_data { // is not corrupt. template void count_btree_blocks(btree const &tree, block_counter &bc, ValueCounter &vc) { - btree_count_detail::counting_visitor v(bc, vc); + typedef noop_value_visitor NoopValueVisitor; + NoopValueVisitor noop_vv; + noop_damage_visitor noop_dv; + btree_count_detail::counting_visitor v(noop_vv, noop_dv, bc, vc); tree.visit_depth_first(v); } } diff --git a/persistent-data/data-structures/btree_damage_visitor.h b/persistent-data/data-structures/btree_damage_visitor.h index 5a96d5d..8be623b 100644 --- a/persistent-data/data-structures/btree_damage_visitor.h +++ b/persistent-data/data-structures/btree_damage_visitor.h @@ -158,8 +158,9 @@ namespace persistent_data { typedef boost::optional maybe_run64; btree_damage_visitor(ValueVisitor &value_visitor, - DamageVisitor &damage_visitor) - : avoid_repeated_visits_(true), + DamageVisitor &damage_visitor, + bool avoid_repeated_visits = true) + : avoid_repeated_visits_(avoid_repeated_visits), value_visitor_(value_visitor), damage_visitor_(damage_visitor) { } From d068ec8082928001e4abfbbe5490236df9485a87 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:22:09 +0800 Subject: [PATCH 22/25] Add utility class binary_block_counter --- persistent-data/block_counter.h | 30 +++++++++++++++++-- .../data-structures/btree_counter.h | 11 +++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/persistent-data/block_counter.h b/persistent-data/block_counter.h index e4232b4..ea70f93 100644 --- a/persistent-data/block_counter.h +++ b/persistent-data/block_counter.h @@ -20,6 +20,7 @@ #define BLOCK_COUNTER_H #include "block.h" +#include "run_set.h" //---------------------------------------------------------------- @@ -32,7 +33,9 @@ namespace persistent_data { public: typedef std::map count_map; - void inc(block_address b) { + virtual ~block_counter() {} + + virtual void inc(block_address b) { count_map::iterator it = counts_.find(b); if (it == counts_.end()) counts_.insert(make_pair(b, 1)); @@ -40,7 +43,7 @@ namespace persistent_data { it->second++; } - unsigned get_count(block_address b) const { + virtual unsigned get_count(block_address b) const { count_map::const_iterator it = counts_.find(b); return (it == counts_.end()) ? 0 : it->second; } @@ -52,6 +55,29 @@ namespace persistent_data { private: count_map counts_; }; + + //---------------------------------------------------------------- + // Little helper class that keeps track of which blocks + // are referenced. + //---------------------------------------------------------------- + class binary_block_counter : public block_counter { + public: + virtual ~binary_block_counter() {} + + virtual void inc(block_address b) { + visited_.add(b); + } + + virtual unsigned get_count(block_address b) const { + return visited_.member(b) ? 1 : 0; + } + + base::run_set const& get_visited() const { + return visited_; + } + private: + base::run_set visited_; + }; } //---------------------------------------------------------------- diff --git a/persistent-data/data-structures/btree_counter.h b/persistent-data/data-structures/btree_counter.h index c215626..6e52c29 100644 --- a/persistent-data/data-structures/btree_counter.h +++ b/persistent-data/data-structures/btree_counter.h @@ -100,6 +100,17 @@ namespace persistent_data { btree_count_detail::counting_visitor v(noop_vv, noop_dv, bc, vc); tree.visit_depth_first(v); } + + template + void count_btree_blocks(btree const &tree, block_counter &bc) { + typedef noop_value_visitor NoopValueVisitor; + NoopValueVisitor noop_vv; + noop_damage_visitor noop_dv; + typedef noop_value_counter NoopValueCounter; + NoopValueCounter vc; + btree_count_detail::counting_visitor v(noop_vv, noop_dv, bc, vc); + tree.visit_depth_first(v); + } } //---------------------------------------------------------------- From 80783e77292c4540cb005b855a735fe05c82bbbe Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:23:06 +0800 Subject: [PATCH 23/25] Add metadata_counter.{h,cc} --- Makefile.in | 1 + thin-provisioning/metadata_counter.cc | 73 +++++++++++++++++++++++++++ thin-provisioning/metadata_counter.h | 24 +++++++++ 3 files changed, 98 insertions(+) create mode 100644 thin-provisioning/metadata_counter.cc create mode 100644 thin-provisioning/metadata_counter.h diff --git a/Makefile.in b/Makefile.in index 30b0c1a..b515463 100644 --- a/Makefile.in +++ b/Makefile.in @@ -81,6 +81,7 @@ SOURCE=\ thin-provisioning/mapping_tree.cc \ thin-provisioning/metadata.cc \ thin-provisioning/metadata_checker.cc \ + thin-provisioning/metadata_counter.cc \ thin-provisioning/metadata_dumper.cc \ thin-provisioning/restore_emitter.cc \ thin-provisioning/rmap_visitor.cc \ diff --git a/thin-provisioning/metadata_counter.cc b/thin-provisioning/metadata_counter.cc new file mode 100644 index 0000000..bf3f809 --- /dev/null +++ b/thin-provisioning/metadata_counter.cc @@ -0,0 +1,73 @@ +#include "thin-provisioning/metadata_counter.h" +#include "persistent-data/space-maps/core.h" +#include "persistent-data/space-maps/disk_structures.h" + +using namespace persistent_data; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +void thin_provisioning::count_trees(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc) { + + // Count the device tree + { + noop_value_counter vc; + device_tree dtree(*tm, sb.device_details_root_, + device_tree_detail::device_details_traits::ref_counter()); + count_btree_blocks(dtree, bc, vc); + } + + // Count the mapping tree + { + noop_value_counter vc; + mapping_tree mtree(*tm, sb.data_mapping_root_, + mapping_tree_detail::block_traits::ref_counter(space_map::ptr())); + count_btree_blocks(mtree, bc, vc); + } +} + +void thin_provisioning::count_space_maps(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc) { + // Count the metadata space map (no-throw) + try { + persistent_space_map::ptr metadata_sm = + open_metadata_sm(*tm, static_cast(&sb.metadata_space_map_root_)); + metadata_sm->count_metadata(bc); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + + // Count the data space map (no-throw) + try { + persistent_space_map::ptr data_sm = + open_disk_sm(*tm, static_cast(&sb.data_space_map_root_)); + data_sm->count_metadata(bc); + } catch (std::exception &e) { + cerr << e.what() << endl; + } +} + +void thin_provisioning::count_metadata(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc, + bool skip_metadata_snap) { + // Count the superblock + bc.inc(superblock_detail::SUPERBLOCK_LOCATION); + count_trees(tm, sb, bc); + + // Count the metadata snap, if present + if (!skip_metadata_snap && sb.metadata_snap_ != superblock_detail::SUPERBLOCK_LOCATION) { + bc.inc(sb.metadata_snap_); + + superblock_detail::superblock snap = read_superblock(tm->get_bm(), sb.metadata_snap_); + count_trees(tm, snap, bc); + } + + count_trees(tm, sb, bc); + count_space_maps(tm, sb, bc); +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/metadata_counter.h b/thin-provisioning/metadata_counter.h new file mode 100644 index 0000000..861d72f --- /dev/null +++ b/thin-provisioning/metadata_counter.h @@ -0,0 +1,24 @@ +#ifndef METADATA_COUNTER_H +#define METADATA_COUNTER_H + +#include "thin-provisioning/metadata.h" +#include "persistent-data/data-structures/btree_counter.h" + +//---------------------------------------------------------------- + +namespace thin_provisioning { + void count_trees(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc); + void count_space_maps(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc); + void count_metadata(transaction_manager::ptr tm, + superblock_detail::superblock &sb, + block_counter &bc, + bool skip_metadata_snap = false); +} + +//---------------------------------------------------------------- + +#endif From b05b9aa2275a894c5e716a22413f705f6943b0bf Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:24:14 +0800 Subject: [PATCH 24/25] [thin_scan] first draft --- Makefile.in | 1 + thin-provisioning/commands.cc | 1 + thin-provisioning/commands.h | 7 + thin-provisioning/thin_scan.cc | 417 +++++++++++++++++++++++++++++++++ 4 files changed, 426 insertions(+) create mode 100644 thin-provisioning/thin_scan.cc diff --git a/Makefile.in b/Makefile.in index b515463..931cb38 100644 --- a/Makefile.in +++ b/Makefile.in @@ -95,6 +95,7 @@ SOURCE=\ thin-provisioning/thin_repair.cc \ thin-provisioning/thin_restore.cc \ thin-provisioning/thin_rmap.cc \ + thin-provisioning/thin_scan.cc \ thin-provisioning/thin_trim.cc \ thin-provisioning/xml_format.cc diff --git a/thin-provisioning/commands.cc b/thin-provisioning/commands.cc index c25abce..6d2f7e4 100644 --- a/thin-provisioning/commands.cc +++ b/thin-provisioning/commands.cc @@ -16,6 +16,7 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_restore_cmd())); app.add_cmd(command::ptr(new thin_repair_cmd())); app.add_cmd(command::ptr(new thin_rmap_cmd())); + app.add_cmd(command::ptr(new thin_scan_cmd())); app.add_cmd(command::ptr(new thin_trim_cmd())); } diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index ec1f1ec..a8fa4a4 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -63,6 +63,13 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_scan_cmd : public base::command { + public: + thin_scan_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + class thin_trim_cmd : public base::command { public: thin_trim_cmd(); diff --git a/thin-provisioning/thin_scan.cc b/thin-provisioning/thin_scan.cc new file mode 100644 index 0000000..c5afbd9 --- /dev/null +++ b/thin-provisioning/thin_scan.cc @@ -0,0 +1,417 @@ +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include +#include +#include +#include +#include + +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/simple_traits.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/space-maps/core.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "thin-provisioning/metadata.h" +#include "thin-provisioning/superblock.h" +#include "thin-provisioning/commands.h" +#include "version.h" + +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + // extracted from btree_damage_visitor.h + template + bool check_block_nr(node const &n) { + if (n.get_location() != n.get_block_nr()) { + return false; + } + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_max_entries(node const &n) { + size_t elt_size = sizeof(uint64_t) + n.get_value_size(); + if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { + return false; + } + + if (n.get_max_entries() % 3) { + return false; + } + + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_nr_entries(node const &n, bool is_root) { + if (n.get_nr_entries() > n.get_max_entries()) { + return false; + } + + block_address min = n.get_max_entries() / 3; + if (!is_root && (n.get_nr_entries() < min)) { + return false; + } + + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_ordered_keys(node const &n) { + unsigned nr_entries = n.get_nr_entries(); + + if (nr_entries == 0) + return true; // can only happen if a root node + + uint64_t last_key = n.key_at(0); + + for (unsigned i = 1; i < nr_entries; i++) { + uint64_t k = n.key_at(i); + if (k <= last_key) { + return false; + } + last_key = k; + } + + return true; + } + + transaction_manager::ptr + open_tm(block_manager<>::ptr bm) { + space_map::ptr sm(new core_map(bm->get_nr_blocks())); + sm->inc(superblock_detail::SUPERBLOCK_LOCATION); + transaction_manager::ptr tm(new transaction_manager(bm, sm)); + return tm; + } +} + +namespace { + // FIXME: deprecated conversion from string constant to ‘char*’ + char const* metadata_block_type_name[] = { + "unknown", + "zero", + "superblock", + "btree_internal", + "btree_leaf", + "btree_unknown", + "index_block", + "bitmap_block" + }; + + enum metadata_block_type { + UNKNOWN = 0, + ZERO, + SUPERBLOCK, + BTREE_INTERNAL, + BTREE_LEAF, + BTREE_UNKNOWN, + INDEX_BLOCK, + BITMAP_BLOCK + }; + + struct block_range { + uint64_t begin_; + uint64_t end_; // one-pass-the-end + boost::optional blocknr_begin_; + metadata_block_type type_; + int64_t ref_count_; // ref_count in metadata space map + size_t value_size_; // btree node only + bool is_valid_; // btree node only + + block_range() + : begin_(0), end_(0), + type_(UNKNOWN), ref_count_(-1), + value_size_(0), is_valid_(false) + { + } + + block_range(block_range const &rhs) + : begin_(rhs.begin_), end_(rhs.end_), + blocknr_begin_(rhs.blocknr_begin_), + type_(rhs.type_), ref_count_(rhs.ref_count_), + value_size_(rhs.value_size_), is_valid_(rhs.is_valid_) + { + } + }; + + void output_block_range(block_range const &r, std::ostream &out) { + if (r.end_ <= r.begin_) + return; + + if (r.end_ - r.begin_ > 1) { + out << "" << endl; + } else + out << "\"/>" << endl; + } + + //------------------------------------------------------------------- + + struct flags { + flags() { + } + + boost::optional scan_begin_; + boost::optional scan_end_; + }; + + int scan_metadata_(string const &input, + std::ostream &out, + flags const &f) { + using namespace persistent_data; + using namespace thin_provisioning; + using namespace sm_disk_detail; + + block_manager<>::ptr bm; + bm = open_bm(input, block_manager<>::READ_ONLY); + + block_address scan_begin = f.scan_begin_ ? *f.scan_begin_ : 0; + block_address scan_end = f.scan_end_ ? *f.scan_end_ : bm->get_nr_blocks(); + + const std::vector zeros(MD_BLOCK_SIZE, 0); + + // try to open metadata space-map (it's okay to fail) + // note: transaction_manager and space_map must be in the same scope + transaction_manager::ptr tm; + checked_space_map::ptr metadata_sm; + try { + superblock_detail::superblock sb = read_superblock(bm); + tm = open_tm(bm); + metadata_sm = open_metadata_sm(*tm, &sb.metadata_space_map_root_); + tm->set_sm(metadata_sm); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + + block_range curr_range; + block_range run_range; + + bcache::validator::ptr sv = superblock_validator(); + bcache::validator::ptr nv = create_btree_node_validator(); + bcache::validator::ptr iv = index_validator(); + bcache::validator::ptr bv = bitmap_validator(); + + for (block_address b = scan_begin; b < scan_end; ++b) { + block_manager<>::read_ref rr = bm->read_lock(b); + + curr_range.begin_ = b; + curr_range.end_ = b + 1; + curr_range.blocknr_begin_ = boost::none; + curr_range.type_ = UNKNOWN; + curr_range.is_valid_ = false; + + if (!memcmp(rr.data(), zeros.data(), MD_BLOCK_SIZE)) + curr_range.type_ = ZERO; + + if (curr_range.type_ == UNKNOWN && sv->check_raw(rr.data())) { + curr_range.type_ = SUPERBLOCK; + curr_range.is_valid_ = true; + } + + if (curr_range.type_ == UNKNOWN && nv->check_raw(rr.data())) { + // note: check_raw() doesn't check node_header::blocknr_ + node_ref n = btree_detail::to_node(rr); + uint32_t flags = to_cpu(n.raw()->header.flags); + if ((flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) + curr_range.type_ = BTREE_INTERNAL; + else if (flags & LEAF_NODE) + curr_range.type_ = BTREE_LEAF; + else + curr_range.type_ = BTREE_UNKNOWN; + + if (curr_range.type_ != BTREE_UNKNOWN && + check_block_nr(n) && + check_max_entries(n) && + check_nr_entries(n, true) && + check_ordered_keys(n)) + curr_range.is_valid_ = true; + else + curr_range.is_valid_ = false; + + curr_range.blocknr_begin_ = n.get_block_nr(); + curr_range.value_size_ = n.get_value_size(); + } + + if (curr_range.type_ == UNKNOWN && bv->check_raw(rr.data())) { + curr_range.type_ = BITMAP_BLOCK; + bitmap_header const *data = reinterpret_cast(rr.data()); + curr_range.blocknr_begin_ = to_cpu(data->blocknr); + curr_range.is_valid_ = (to_cpu(data->blocknr) == b) ? true : false; + } + + if (curr_range.type_ == UNKNOWN && iv->check_raw(rr.data())) { + curr_range.type_ = INDEX_BLOCK; + metadata_index const *mi = reinterpret_cast(rr.data()); + curr_range.blocknr_begin_ = to_cpu(mi->blocknr_); + curr_range.is_valid_ = (to_cpu(mi->blocknr_) == b) ? true : false; + } + + try { + curr_range.ref_count_ = metadata_sm ? + static_cast(metadata_sm->get_count(b)) : -1; + } catch (std::exception &e) { + curr_range.ref_count_ = -1; + } + + // output the current block + if (run_range.end_ == 0) + run_range = curr_range; + else if (((!curr_range.blocknr_begin_ && !run_range.blocknr_begin_) || + (curr_range.blocknr_begin_ && run_range.blocknr_begin_ && + *curr_range.blocknr_begin_ == *run_range.blocknr_begin_ + (run_range.end_ - run_range.begin_))) && + curr_range.type_ == run_range.type_ && + curr_range.ref_count_ == run_range.ref_count_ && + curr_range.value_size_ == run_range.value_size_ && + curr_range.is_valid_ == run_range.is_valid_) { + ++run_range.end_; + } else { + output_block_range(run_range, out); + run_range = curr_range; + } + } + + // output the last run + output_block_range(run_range, out); + + return 0; + } + + int scan_metadata(string const &input, + boost::optional output, + flags const &f) { + try { + if (output) { + std::ofstream out(output->c_str()); + scan_metadata_(input, out, f); + } else + scan_metadata_(input, cout, f); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + return 0; + } +} + +//--------------------------------------------------------------------------- + +thin_scan_cmd::thin_scan_cmd() + : command("thin_scan") +{ +} + +void +thin_scan_cmd::usage(std::ostream &out) const { + out << "Usage: " << get_name() << " [options] {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-o|--output} " << endl + << " {--begin} " << endl + << " {--end} " << endl + << " {-V|--version}" << endl; +} + +int +thin_scan_cmd::run(int argc, char **argv) +{ + const char shortopts[] = "ho:V"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "output", required_argument, NULL, 'o'}, + { "version", no_argument, NULL, 'V'}, + { "begin", required_argument, NULL, 1}, + { "end", required_argument, NULL, 2}, + { NULL, no_argument, NULL, 0 } + }; + boost::optional output; + flags f; + + char c; + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'o': + output = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + case 1: + try { + f.scan_begin_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + case 2: + try { + f.scan_end_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + default: + usage(cerr); + return 1; + } + } + + if (argc == optind) { + cerr << "No input file provided." << endl; + usage(cerr); + return 1; + } + + if (f.scan_begin_ && f.scan_end_ && (*f.scan_end_ <= *f.scan_begin_)) { + cerr << "badly formed region (end <= begin)" << endl; + return 1; + } + + return scan_metadata(argv[optind], output, f); +} + +//--------------------------------------------------------------------------- From 08a7093cd51b76e792b23b07d69f2cedb92d214b Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sat, 27 Feb 2016 15:24:28 +0800 Subject: [PATCH 25/25] [thin_ll_dump][thin_ll_restore] first draft --- Makefile.in | 2 + thin-provisioning/commands.cc | 2 + thin-provisioning/commands.h | 14 + thin-provisioning/thin_ll_dump.cc | 473 +++++++++++++++++++++++++++ thin-provisioning/thin_ll_restore.cc | 276 ++++++++++++++++ 5 files changed, 767 insertions(+) create mode 100644 thin-provisioning/thin_ll_dump.cc create mode 100644 thin-provisioning/thin_ll_restore.cc diff --git a/Makefile.in b/Makefile.in index 931cb38..542154d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -89,6 +89,8 @@ SOURCE=\ thin-provisioning/thin_check.cc \ thin-provisioning/thin_delta.cc \ thin-provisioning/thin_dump.cc \ + thin-provisioning/thin_ll_dump.cc \ + thin-provisioning/thin_ll_restore.cc \ thin-provisioning/thin_ls.cc \ thin-provisioning/thin_metadata_size.cc \ thin-provisioning/thin_pool.cc \ diff --git a/thin-provisioning/commands.cc b/thin-provisioning/commands.cc index 6d2f7e4..ba240c9 100644 --- a/thin-provisioning/commands.cc +++ b/thin-provisioning/commands.cc @@ -11,6 +11,8 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_check_cmd())); app.add_cmd(command::ptr(new thin_delta_cmd())); app.add_cmd(command::ptr(new thin_dump_cmd())); + app.add_cmd(command::ptr(new thin_ll_dump_cmd())); + app.add_cmd(command::ptr(new thin_ll_restore_cmd())); app.add_cmd(command::ptr(new thin_ls_cmd())); app.add_cmd(command::ptr(new thin_metadata_size_cmd())); app.add_cmd(command::ptr(new thin_restore_cmd())); diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index a8fa4a4..d6ed78f 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -28,6 +28,20 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_ll_dump_cmd : public base::command { + public: + thin_ll_dump_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + class thin_ll_restore_cmd : public base::command { + public: + thin_ll_restore_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + class thin_ls_cmd : public base::command { public: thin_ls_cmd(); diff --git a/thin-provisioning/thin_ll_dump.cc b/thin-provisioning/thin_ll_dump.cc new file mode 100644 index 0000000..7f7b14c --- /dev/null +++ b/thin-provisioning/thin_ll_dump.cc @@ -0,0 +1,473 @@ +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include +#include +#include +#include +#include + +#include "base/indented_stream.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/btree_counter.h" +#include "persistent-data/data-structures/simple_traits.h" +#include "persistent-data/space-maps/core.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "thin-provisioning/metadata.h" +#include "thin-provisioning/metadata_counter.h" +#include "thin-provisioning/commands.h" +#include "version.h" + +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +namespace { + // extracted from btree_damage_visitor.h + template + bool check_block_nr(node const &n) { + if (n.get_location() != n.get_block_nr()) { + return false; + } + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_max_entries(node const &n) { + size_t elt_size = sizeof(uint64_t) + n.get_value_size(); + if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) { + return false; + } + + if (n.get_max_entries() % 3) { + return false; + } + + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_nr_entries(node const &n, bool is_root) { + if (n.get_nr_entries() > n.get_max_entries()) { + return false; + } + + block_address min = n.get_max_entries() / 3; + if (!is_root && (n.get_nr_entries() < min)) { + return false; + } + + return true; + } + + // extracted from btree_damage_visitor.h + template + bool check_ordered_keys(node const &n) { + unsigned nr_entries = n.get_nr_entries(); + + if (nr_entries == 0) + return true; // can only happen if a root node + + uint64_t last_key = n.key_at(0); + + for (unsigned i = 1; i < nr_entries; i++) { + uint64_t k = n.key_at(i); + if (k <= last_key) { + return false; + } + last_key = k; + } + + return true; + } + + transaction_manager::ptr + open_tm(block_manager<>::ptr bm) { + space_map::ptr sm(new core_map(bm->get_nr_blocks())); + sm->inc(superblock_detail::SUPERBLOCK_LOCATION); + transaction_manager::ptr tm(new transaction_manager(bm, sm)); + return tm; + } +} + +//--------------------------------------------------------------------------- + +namespace { + struct node_info { + uint64_t blocknr_; + uint32_t flags_; + uint64_t key_begin_; + uint64_t key_end_; + uint64_t nr_entries_; + uint32_t value_size_; + }; + + //------------------------------------------------------------------- + + struct btree_node_checker { + typedef boost::shared_ptr ptr; + virtual ~btree_node_checker() {} + virtual bool check(node_ref &n) = 0; + }; + + struct unvisited_btree_node_filter: public btree_node_checker { + unvisited_btree_node_filter(block_counter const &bc) + : nv_(create_btree_node_validator()), bc_(bc) { + } + + virtual bool check(node_ref &n) { + uint32_t flags = to_cpu(n.raw()->header.flags); + if ((n.get_value_size() == sizeof(mapping_tree_detail::block_traits::disk_type) || + n.get_value_size() == sizeof(device_tree_detail::device_details_traits::disk_type)) && + !bc_.get_count(n.get_location()) && + check_block_nr(n) && + (((flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) || + (flags & LEAF_NODE)) && + nv_->check_raw(n.raw()) && + check_max_entries(n) && + check_nr_entries(n, true) && + check_ordered_keys(n)) + return true; + return false; + } + + bcache::validator::ptr nv_; + block_counter const &bc_; + }; + + //------------------------------------------------------------------- + + void find_btree_nodes(block_manager<>::ptr bm, + block_address begin, + block_address end, + btree_node_checker::ptr checker, + base::run_set &found) { + using namespace persistent_data; + + for (block_address b = begin; b < end; ++b) { + block_manager<>::read_ref rr = bm->read_lock(b); + node_ref n = btree_detail::to_node(rr); + + if (checker->check(n)) + found.add(b); + } + } + + //------------------------------------------------------------------- + + bool first_key_cmp(node_info const &lhs, node_info const &rhs) { + return lhs.key_begin_ < rhs.key_begin_; + } + + template + void convert_to_node_info(node_ref const &n, node_info &ni) { + ni.blocknr_ = n.get_location(); + ni.flags_ = to_cpu(n.raw()->header.flags); + if ((ni.nr_entries_ = n.get_nr_entries()) > 0) { + ni.key_begin_ = n.key_at(0); + ni.key_end_ = n.key_at(n.get_nr_entries() - 1); + } + ni.value_size_ = n.get_value_size(); + } + + void output_node_info(indented_stream &out, node_info const &ni) { + out.indent(); + out << "" << endl; + } + + //------------------------------------------------------------------- + + class ll_mapping_tree_emitter : public mapping_tree_detail::device_visitor { + public: + ll_mapping_tree_emitter(block_manager<>::ptr bm, + indented_stream &out) + : bm_(bm), out_(out) { + } + + void visit(btree_path const &path, block_address tree_root) { + out_.indent(); + out_ << "" << endl; + out_.inc(); + + // Do not throw exception. Process the next entry inside the current node. + try { + block_manager<>::read_ref rr = bm_->read_lock(tree_root); + node_ref n = btree_detail::to_node(rr); + node_info ni; + convert_to_node_info(n, ni); + output_node_info(out_, ni); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + + out_.dec(); + out_.indent(); + out_ << "" << endl; + } + private: + block_manager<>::ptr bm_; + indented_stream& out_; + }; + + //------------------------------------------------------------------- + + struct flags { + flags() : use_metadata_snap_(false) { + } + + bool use_metadata_snap_; + boost::optional metadata_snap_; + boost::optional data_mapping_root_; + boost::optional device_details_root_; + boost::optional scan_begin_; + boost::optional scan_end_; + }; + + int low_level_dump_(string const &input, + std::ostream &output, + flags const &f) { + block_manager<>::ptr bm = open_bm(input, block_manager<>::READ_ONLY); + + block_address scan_begin = f.scan_begin_ ? *f.scan_begin_ : 0; + block_address scan_end = f.scan_end_ ? *f.scan_end_ : bm->get_nr_blocks(); + + // Allow to read superblock at arbitrary location for low-level dump, + // without checking equality between the given metadata_snap and sb.metadata_snap_ + superblock_detail::superblock sb = read_superblock(bm, superblock_detail::SUPERBLOCK_LOCATION); + if (f.use_metadata_snap_) { + sb = f.metadata_snap_ ? + read_superblock(bm, *f.metadata_snap_) : + read_superblock(bm, sb.metadata_snap_); + } + // override sb.data_mapping_root_ + if (f.data_mapping_root_) + sb.data_mapping_root_ = *f.data_mapping_root_; + // override sb.device_details_root_ + if (f.device_details_root_) + sb.device_details_root_ = *f.device_details_root_; + + transaction_manager::ptr tm = open_tm(bm); + + indented_stream out(output); + + out.indent(); + out << "" << endl; + out.inc(); + + // output the top-level data mapping tree + ll_mapping_tree_emitter ll_mte(tm->get_bm(), out); + dev_tree dtree(*tm, sb.data_mapping_root_, + mapping_tree_detail::mtree_traits::ref_counter(tm)); + noop_damage_visitor noop_dv; + btree_visit_values(dtree, ll_mte, noop_dv); + + out.dec(); + out.indent(); + out << "" << endl; + + // find orphans + binary_block_counter bc; + bc.inc(superblock_detail::SUPERBLOCK_LOCATION); + count_metadata(tm, sb, bc, true); + btree_node_checker::ptr filter = btree_node_checker::ptr( + new unvisited_btree_node_filter(bc)); + base::run_set orphans; + find_btree_nodes(bm, scan_begin, scan_end, filter, orphans); + + // sort orphans + std::vector nodes; + for (base::run_set::const_iterator it = orphans.begin(); + it != orphans.end(); + ++it) { + if (it->begin_ && it->end_) { + for (block_address b = *it->begin_; b < *it->end_; ++b) { + block_manager<>::read_ref rr = bm->read_lock(b); + node_ref n = btree_detail::to_node(rr); + nodes.push_back(node_info()); + convert_to_node_info(n, nodes.back()); + } + } + } + std::sort(nodes.begin(), nodes.end(), first_key_cmp); + + // output orphans + out.indent(); + out << "" << std::endl; + out.inc(); + + for (size_t i = 0; i < nodes.size(); ++i) + output_node_info(out, nodes[i]); + + out.dec(); + out.indent(); + out << "" << std::endl; + + return 0; + } + + int low_level_dump(string const &input, + boost::optional output, + flags const &f) { + try { + if (output) { + ofstream out(output->c_str()); + low_level_dump_(input, out, f); + } else + low_level_dump_(input, cout, f); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + return 0; + } +} + +//--------------------------------------------------------------------------- + +thin_ll_dump_cmd::thin_ll_dump_cmd() + : command("thin_ll_dump") +{ +} + +void +thin_ll_dump_cmd::usage(ostream &out) const { + out << "Usage: " << get_name() << " [options] {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-m|--metadata-snap}[block#]" << endl + << " {-o|--output} " << endl + << " {--begin} " << endl + << " {--end} " << endl + << " {--data-mapping-root} " << endl + << " {--device-details-root} " << endl + << " {-V|--version}" << endl; +} + +int +thin_ll_dump_cmd::run(int argc, char **argv) +{ + const char shortopts[] = "hm:o:V"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "metadata-snap", optional_argument, NULL, 'm'}, + { "output", required_argument, NULL, 'o'}, + { "version", no_argument, NULL, 'V'}, + { "begin", required_argument, NULL, 1}, + { "end", required_argument, NULL, 2}, + { "data-mapping-root", required_argument, NULL, 3}, + { "device-details-root", required_argument, NULL, 4}, + { NULL, no_argument, NULL, 0 } + }; + boost::optional output; + flags f; + + char c; + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'm': + f.use_metadata_snap_ = true; + if (optarg) { + try { + f.metadata_snap_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + } + break; + + case 'o': + output = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + case 1: + try { + f.scan_begin_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + case 2: + try { + f.scan_end_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + case 3: + try { + f.data_mapping_root_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + case 4: + try { + f.device_details_root_ = boost::lexical_cast(optarg); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + break; + + default: + usage(cerr); + return 1; + } + } + + if (argc == optind) { + cerr << "No input file provided." << endl; + usage(cerr); + return 1; + } + + if (f.scan_begin_ && f.scan_end_ && (*f.scan_end_ <= *f.scan_begin_)) { + cerr << "badly formed region (end <= begin)" << endl; + usage(cerr); + return 1; + } + + return low_level_dump(argv[optind], output, f); +} + +//--------------------------------------------------------------------------- diff --git a/thin-provisioning/thin_ll_restore.cc b/thin-provisioning/thin_ll_restore.cc new file mode 100644 index 0000000..1168579 --- /dev/null +++ b/thin-provisioning/thin_ll_restore.cc @@ -0,0 +1,276 @@ +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include "base/xml_utils.h" +#include "metadata_dumper.h" +#include "metadata.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "restore_emitter.h" +#include "xml_format.h" +#include "thin-provisioning/commands.h" +#include "version.h" + +#include +#include +#include + +using namespace persistent_data; +using namespace std; +using namespace thin_provisioning; +using namespace xml_utils; + +//---------------------------------------------------------------- + +namespace { + struct user_data { + block_manager<>::ptr input_bm_; + block_manager<>::ptr output_bm_; + + metadata::ptr md_; + XML_Parser parser_; + emitter::ptr emitter_; + }; + + void open_resources(user_data &ud, attributes const &attr) { + boost::optional val; + + // open the input metadata + // Allow to read superblock at arbitrary location for low-level restore + block_address sb_location = (val = get_opt_attr(attr, "blocknr")) ? + *val : superblock_detail::SUPERBLOCK_LOCATION; + ud.md_ = metadata::ptr(new metadata(ud.input_bm_, sb_location)); + + // override superblock::device_details_root_ + if ((val = get_opt_attr(attr, "device_details_root"))) { + ud.md_->sb_.device_details_root_ = *val; + ud.md_->details_ = device_tree::ptr(new device_tree(*ud.md_->tm_, *val, + device_tree_detail::device_details_traits::ref_counter())); + } + + // open the output metadata + metadata::ptr new_md(new metadata(ud.output_bm_, metadata::CREATE, 128, 0)); + + ud.emitter_ = create_restore_emitter(new_md); + } + + void parse_superblock(metadata::ptr md, emitter::ptr e, attributes const &attr) { + sm_disk_detail::sm_root_disk const *d = + reinterpret_cast(md->sb_.data_space_map_root_); + sm_disk_detail::sm_root v; + sm_disk_detail::sm_root_traits::unpack(*d, v); + + e->begin_superblock("", md->sb_.time_, + md->sb_.trans_id_, + md->sb_.flags_, + md->sb_.version_, + md->sb_.data_block_size_, + v.nr_blocks_, + boost::optional()); + } + + void parse_device(metadata::ptr md, emitter::ptr e, attributes const &attr) { + uint32_t dev_id = get_attr(attr, "dev_id"); + device_tree_detail::device_details details; + details.transaction_id_ = 0; + details.creation_time_ = 0; + details.snapshotted_time_ = 0; + + device_tree::ptr details_tree; + boost::optional details_root = get_opt_attr(attr, "blocknr"); + if (details_root) + details_tree = device_tree::ptr(new device_tree(*md->tm_, *details_root, + device_tree_detail::device_details_traits::ref_counter())); + else + details_tree = md->details_; + + uint64_t key[1] = {dev_id}; + device_tree::maybe_value v; + try { + v = details_tree->lookup(key); + } catch (std::exception &e) { + cerr << "missing device " << dev_id << ": " << e.what() << endl; + } + if (v) + details = *v; + + e->begin_device(dev_id, + 0, + details.transaction_id_, + details.creation_time_, + details.snapshotted_time_); + } + + void parse_node(metadata::ptr md, emitter::ptr e, attributes const &attr) { + metadata_dump_subtree(md, e, true, get_attr(attr, "blocknr")); + } + + void start_tag(void *data, char const *el, char const **attr) { + user_data *ud = static_cast(data); + attributes a; + + build_attributes(a, attr); + + if (!strcmp(el, "superblock")) { + open_resources(*ud, a); + parse_superblock(ud->md_, ud->emitter_, a); + + } else if (!strcmp(el, "device")) + parse_device(ud->md_, ud->emitter_, a); + + else if (!strcmp(el, "node")) + parse_node(ud->md_, ud->emitter_, a); + + else + throw runtime_error("unknown tag type"); + } + + void end_tag(void *data, const char *el) { + user_data *ud = static_cast(data); + + if (!strcmp(el, "superblock")) { + ud->emitter_->end_superblock(); + XML_StopParser(ud->parser_, XML_FALSE); // skip the rest elements + } + + else if (!strcmp(el, "device")) + ud->emitter_->end_device(); + + else if (!strcmp(el, "node")) + ; + + else + throw runtime_error("unknown tag type"); + } +} + +//--------------------------------------------------------------------------- + +namespace { + struct flags { + flags() { + } + }; + + int low_level_restore_(string const &src_metadata, string const &input, + string const &output, flags const &f) { + user_data ud; + ud.input_bm_ = open_bm(src_metadata, block_manager<>::READ_ONLY); + ud.output_bm_ = open_bm(output, block_manager<>::READ_WRITE); + + xml_parser p; + ud.parser_ = p.get_parser(); + + XML_SetUserData(p.get_parser(), &ud); + XML_SetElementHandler(p.get_parser(), start_tag, end_tag); + + bool quiet = true; + p.parse(input, quiet); + + return 0; + } + + int low_level_restore(string const &src_metadata, string const &input, + string const &output, flags const &f) { + try { + low_level_restore_(src_metadata, input, output, f); + } catch (std::exception &e) { + cerr << e.what() << endl; + return 1; + } + return 0; + } +} + +//--------------------------------------------------------------------------- + +thin_ll_restore_cmd::thin_ll_restore_cmd() + : command("thin_ll_restore") +{ +} + +void +thin_ll_restore_cmd::usage(ostream &out) const { + out << "Usage: " << get_name() << " [options]" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-E|--source-metadata} " << endl + << " {-i|--input} " << endl + << " {-o|--output} " << endl + << " {-V|--version}" << endl; +} + +int +thin_ll_restore_cmd::run(int argc, char **argv) { + string input; + string output; + string input_metadata; + flags f; + char c; + + const char shortopts[] = "hi:o:E:V"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "input", required_argument, NULL, 'i'}, + { "output", required_argument, NULL, 'o'}, + { "source-metadata", required_argument, NULL, 'E'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'i': + input = optarg; + break; + + case 'o': + output = optarg; + break; + + case 'E': + input_metadata = optarg; + break; + + case 'V': + cout << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + + default: + usage(cerr); + return 1; + } + } + + if (argc != optind) { + usage(cerr); + return 1; + } + + if (!input_metadata.length() || !input.length() || !output.length()) { + cerr << "No input/output file provided." << endl; + usage(cerr); + return 1; + } + + return low_level_restore(input_metadata, input, output, f); +} + +//---------------------------------------------------------------------------