[thin_ll_dump][thin_ll_restore] first draft

This commit is contained in:
Ming-Hung Tsai 2016-02-27 15:24:28 +08:00
parent b05b9aa227
commit 08a7093cd5
5 changed files with 767 additions and 0 deletions

View File

@ -89,6 +89,8 @@ SOURCE=\
thin-provisioning/thin_check.cc \
thin-provisioning/thin_delta.cc \
thin-provisioning/thin_dump.cc \
thin-provisioning/thin_ll_dump.cc \
thin-provisioning/thin_ll_restore.cc \
thin-provisioning/thin_ls.cc \
thin-provisioning/thin_metadata_size.cc \
thin-provisioning/thin_pool.cc \

View File

@ -11,6 +11,8 @@ thin_provisioning::register_thin_commands(base::application &app)
app.add_cmd(command::ptr(new thin_check_cmd()));
app.add_cmd(command::ptr(new thin_delta_cmd()));
app.add_cmd(command::ptr(new thin_dump_cmd()));
app.add_cmd(command::ptr(new thin_ll_dump_cmd()));
app.add_cmd(command::ptr(new thin_ll_restore_cmd()));
app.add_cmd(command::ptr(new thin_ls_cmd()));
app.add_cmd(command::ptr(new thin_metadata_size_cmd()));
app.add_cmd(command::ptr(new thin_restore_cmd()));

View File

@ -28,6 +28,20 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv);
};
class thin_ll_dump_cmd : public base::command {
public:
thin_ll_dump_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_ll_restore_cmd : public base::command {
public:
thin_ll_restore_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_ls_cmd : public base::command {
public:
thin_ls_cmd();

View File

@ -0,0 +1,473 @@
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include <boost/lexical_cast.hpp>
#include <boost/optional.hpp>
#include <getopt.h>
#include <vector>
#include <fstream>
#include "base/indented_stream.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/data-structures/btree.h"
#include "persistent-data/data-structures/btree_counter.h"
#include "persistent-data/data-structures/simple_traits.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "thin-provisioning/metadata.h"
#include "thin-provisioning/metadata_counter.h"
#include "thin-provisioning/commands.h"
#include "version.h"
using namespace thin_provisioning;
//----------------------------------------------------------------
namespace {
// extracted from btree_damage_visitor.h
template <typename node>
bool check_block_nr(node const &n) {
if (n.get_location() != n.get_block_nr()) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_max_entries(node const &n) {
size_t elt_size = sizeof(uint64_t) + n.get_value_size();
if (elt_size * n.get_max_entries() + sizeof(node_header) > MD_BLOCK_SIZE) {
return false;
}
if (n.get_max_entries() % 3) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_nr_entries(node const &n, bool is_root) {
if (n.get_nr_entries() > n.get_max_entries()) {
return false;
}
block_address min = n.get_max_entries() / 3;
if (!is_root && (n.get_nr_entries() < min)) {
return false;
}
return true;
}
// extracted from btree_damage_visitor.h
template <typename node>
bool check_ordered_keys(node const &n) {
unsigned nr_entries = n.get_nr_entries();
if (nr_entries == 0)
return true; // can only happen if a root node
uint64_t last_key = n.key_at(0);
for (unsigned i = 1; i < nr_entries; i++) {
uint64_t k = n.key_at(i);
if (k <= last_key) {
return false;
}
last_key = k;
}
return true;
}
transaction_manager::ptr
open_tm(block_manager<>::ptr bm) {
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
sm->inc(superblock_detail::SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
}
}
//---------------------------------------------------------------------------
namespace {
struct node_info {
uint64_t blocknr_;
uint32_t flags_;
uint64_t key_begin_;
uint64_t key_end_;
uint64_t nr_entries_;
uint32_t value_size_;
};
//-------------------------------------------------------------------
struct btree_node_checker {
typedef boost::shared_ptr<btree_node_checker> ptr;
virtual ~btree_node_checker() {}
virtual bool check(node_ref<uint64_traits> &n) = 0;
};
struct unvisited_btree_node_filter: public btree_node_checker {
unvisited_btree_node_filter(block_counter const &bc)
: nv_(create_btree_node_validator()), bc_(bc) {
}
virtual bool check(node_ref<uint64_traits> &n) {
uint32_t flags = to_cpu<uint32_t>(n.raw()->header.flags);
if ((n.get_value_size() == sizeof(mapping_tree_detail::block_traits::disk_type) ||
n.get_value_size() == sizeof(device_tree_detail::device_details_traits::disk_type)) &&
!bc_.get_count(n.get_location()) &&
check_block_nr(n) &&
(((flags & INTERNAL_NODE) && !(flags & LEAF_NODE)) ||
(flags & LEAF_NODE)) &&
nv_->check_raw(n.raw()) &&
check_max_entries(n) &&
check_nr_entries(n, true) &&
check_ordered_keys(n))
return true;
return false;
}
bcache::validator::ptr nv_;
block_counter const &bc_;
};
//-------------------------------------------------------------------
void find_btree_nodes(block_manager<>::ptr bm,
block_address begin,
block_address end,
btree_node_checker::ptr checker,
base::run_set<block_address> &found) {
using namespace persistent_data;
for (block_address b = begin; b < end; ++b) {
block_manager<>::read_ref rr = bm->read_lock(b);
node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
if (checker->check(n))
found.add(b);
}
}
//-------------------------------------------------------------------
bool first_key_cmp(node_info const &lhs, node_info const &rhs) {
return lhs.key_begin_ < rhs.key_begin_;
}
template <typename ValueTraits>
void convert_to_node_info(node_ref<ValueTraits> const &n, node_info &ni) {
ni.blocknr_ = n.get_location();
ni.flags_ = to_cpu<uint32_t>(n.raw()->header.flags);
if ((ni.nr_entries_ = n.get_nr_entries()) > 0) {
ni.key_begin_ = n.key_at(0);
ni.key_end_ = n.key_at(n.get_nr_entries() - 1);
}
ni.value_size_ = n.get_value_size();
}
void output_node_info(indented_stream &out, node_info const &ni) {
out.indent();
out << "<node blocknr=\"" << ni.blocknr_
<< "\" flags=\"" << ni.flags_
<< "\" key_begin=\"" << ni.key_begin_
<< "\" key_end=\"" << ni.key_end_
<< "\" nr_entries=\"" << ni.nr_entries_
<< "\" value_size=\"" << ni.value_size_
<< "\"/>" << endl;
}
//-------------------------------------------------------------------
class ll_mapping_tree_emitter : public mapping_tree_detail::device_visitor {
public:
ll_mapping_tree_emitter(block_manager<>::ptr bm,
indented_stream &out)
: bm_(bm), out_(out) {
}
void visit(btree_path const &path, block_address tree_root) {
out_.indent();
out_ << "<device dev_id=\"" << path[0] <<"\">" << endl;
out_.inc();
// Do not throw exception. Process the next entry inside the current node.
try {
block_manager<>::read_ref rr = bm_->read_lock(tree_root);
node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
node_info ni;
convert_to_node_info(n, ni);
output_node_info(out_, ni);
} catch (std::exception &e) {
cerr << e.what() << endl;
}
out_.dec();
out_.indent();
out_ << "</device>" << endl;
}
private:
block_manager<>::ptr bm_;
indented_stream& out_;
};
//-------------------------------------------------------------------
struct flags {
flags() : use_metadata_snap_(false) {
}
bool use_metadata_snap_;
boost::optional<block_address> metadata_snap_;
boost::optional<block_address> data_mapping_root_;
boost::optional<block_address> device_details_root_;
boost::optional<block_address> scan_begin_;
boost::optional<block_address> scan_end_;
};
int low_level_dump_(string const &input,
std::ostream &output,
flags const &f) {
block_manager<>::ptr bm = open_bm(input, block_manager<>::READ_ONLY);
block_address scan_begin = f.scan_begin_ ? *f.scan_begin_ : 0;
block_address scan_end = f.scan_end_ ? *f.scan_end_ : bm->get_nr_blocks();
// Allow to read superblock at arbitrary location for low-level dump,
// without checking equality between the given metadata_snap and sb.metadata_snap_
superblock_detail::superblock sb = read_superblock(bm, superblock_detail::SUPERBLOCK_LOCATION);
if (f.use_metadata_snap_) {
sb = f.metadata_snap_ ?
read_superblock(bm, *f.metadata_snap_) :
read_superblock(bm, sb.metadata_snap_);
}
// override sb.data_mapping_root_
if (f.data_mapping_root_)
sb.data_mapping_root_ = *f.data_mapping_root_;
// override sb.device_details_root_
if (f.device_details_root_)
sb.device_details_root_ = *f.device_details_root_;
transaction_manager::ptr tm = open_tm(bm);
indented_stream out(output);
out.indent();
out << "<superblock blocknr=\"" << sb.blocknr_
<< "\" data_mapping_root=\"" << sb.data_mapping_root_
<< "\" device_details_root=\"" << sb.device_details_root_
<< "\">" << endl;
out.inc();
// output the top-level data mapping tree
ll_mapping_tree_emitter ll_mte(tm->get_bm(), out);
dev_tree dtree(*tm, sb.data_mapping_root_,
mapping_tree_detail::mtree_traits::ref_counter(tm));
noop_damage_visitor noop_dv;
btree_visit_values(dtree, ll_mte, noop_dv);
out.dec();
out.indent();
out << "</superblock>" << endl;
// find orphans
binary_block_counter bc;
bc.inc(superblock_detail::SUPERBLOCK_LOCATION);
count_metadata(tm, sb, bc, true);
btree_node_checker::ptr filter = btree_node_checker::ptr(
new unvisited_btree_node_filter(bc));
base::run_set<block_address> orphans;
find_btree_nodes(bm, scan_begin, scan_end, filter, orphans);
// sort orphans
std::vector<node_info> nodes;
for (base::run_set<block_address>::const_iterator it = orphans.begin();
it != orphans.end();
++it) {
if (it->begin_ && it->end_) {
for (block_address b = *it->begin_; b < *it->end_; ++b) {
block_manager<>::read_ref rr = bm->read_lock(b);
node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
nodes.push_back(node_info());
convert_to_node_info(n, nodes.back());
}
}
}
std::sort(nodes.begin(), nodes.end(), first_key_cmp);
// output orphans
out.indent();
out << "<orphans>" << std::endl;
out.inc();
for (size_t i = 0; i < nodes.size(); ++i)
output_node_info(out, nodes[i]);
out.dec();
out.indent();
out << "</orphans>" << std::endl;
return 0;
}
int low_level_dump(string const &input,
boost::optional<string> output,
flags const &f) {
try {
if (output) {
ofstream out(output->c_str());
low_level_dump_(input, out, f);
} else
low_level_dump_(input, cout, f);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
}
//---------------------------------------------------------------------------
thin_ll_dump_cmd::thin_ll_dump_cmd()
: command("thin_ll_dump")
{
}
void
thin_ll_dump_cmd::usage(ostream &out) const {
out << "Usage: " << get_name() << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-m|--metadata-snap}[block#]" << endl
<< " {-o|--output} <xml file>" << endl
<< " {--begin} <block#>" << endl
<< " {--end} <block#>" << endl
<< " {--data-mapping-root} <block#>" << endl
<< " {--device-details-root} <block#>" << endl
<< " {-V|--version}" << endl;
}
int
thin_ll_dump_cmd::run(int argc, char **argv)
{
const char shortopts[] = "hm:o:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "metadata-snap", optional_argument, NULL, 'm'},
{ "output", required_argument, NULL, 'o'},
{ "version", no_argument, NULL, 'V'},
{ "begin", required_argument, NULL, 1},
{ "end", required_argument, NULL, 2},
{ "data-mapping-root", required_argument, NULL, 3},
{ "device-details-root", required_argument, NULL, 4},
{ NULL, no_argument, NULL, 0 }
};
boost::optional<string> output;
flags f;
char c;
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'm':
f.use_metadata_snap_ = true;
if (optarg) {
try {
f.metadata_snap_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
}
break;
case 'o':
output = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
case 1:
try {
f.scan_begin_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
case 2:
try {
f.scan_end_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
case 3:
try {
f.data_mapping_root_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
case 4:
try {
f.device_details_root_ = boost::lexical_cast<uint64_t>(optarg);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
break;
default:
usage(cerr);
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr);
return 1;
}
if (f.scan_begin_ && f.scan_end_ && (*f.scan_end_ <= *f.scan_begin_)) {
cerr << "badly formed region (end <= begin)" << endl;
usage(cerr);
return 1;
}
return low_level_dump(argv[optind], output, f);
}
//---------------------------------------------------------------------------

View File

@ -0,0 +1,276 @@
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include "base/xml_utils.h"
#include "metadata_dumper.h"
#include "metadata.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "restore_emitter.h"
#include "xml_format.h"
#include "thin-provisioning/commands.h"
#include "version.h"
#include <fstream>
#include <getopt.h>
#include <iostream>
using namespace persistent_data;
using namespace std;
using namespace thin_provisioning;
using namespace xml_utils;
//----------------------------------------------------------------
namespace {
struct user_data {
block_manager<>::ptr input_bm_;
block_manager<>::ptr output_bm_;
metadata::ptr md_;
XML_Parser parser_;
emitter::ptr emitter_;
};
void open_resources(user_data &ud, attributes const &attr) {
boost::optional<uint64_t> val;
// open the input metadata
// Allow to read superblock at arbitrary location for low-level restore
block_address sb_location = (val = get_opt_attr<uint64_t>(attr, "blocknr")) ?
*val : superblock_detail::SUPERBLOCK_LOCATION;
ud.md_ = metadata::ptr(new metadata(ud.input_bm_, sb_location));
// override superblock::device_details_root_
if ((val = get_opt_attr<uint64_t>(attr, "device_details_root"))) {
ud.md_->sb_.device_details_root_ = *val;
ud.md_->details_ = device_tree::ptr(new device_tree(*ud.md_->tm_, *val,
device_tree_detail::device_details_traits::ref_counter()));
}
// open the output metadata
metadata::ptr new_md(new metadata(ud.output_bm_, metadata::CREATE, 128, 0));
ud.emitter_ = create_restore_emitter(new_md);
}
void parse_superblock(metadata::ptr md, emitter::ptr e, attributes const &attr) {
sm_disk_detail::sm_root_disk const *d =
reinterpret_cast<sm_disk_detail::sm_root_disk const *>(md->sb_.data_space_map_root_);
sm_disk_detail::sm_root v;
sm_disk_detail::sm_root_traits::unpack(*d, v);
e->begin_superblock("", md->sb_.time_,
md->sb_.trans_id_,
md->sb_.flags_,
md->sb_.version_,
md->sb_.data_block_size_,
v.nr_blocks_,
boost::optional<block_address>());
}
void parse_device(metadata::ptr md, emitter::ptr e, attributes const &attr) {
uint32_t dev_id = get_attr<uint32_t>(attr, "dev_id");
device_tree_detail::device_details details;
details.transaction_id_ = 0;
details.creation_time_ = 0;
details.snapshotted_time_ = 0;
device_tree::ptr details_tree;
boost::optional<uint32_t> details_root = get_opt_attr<uint32_t>(attr, "blocknr");
if (details_root)
details_tree = device_tree::ptr(new device_tree(*md->tm_, *details_root,
device_tree_detail::device_details_traits::ref_counter()));
else
details_tree = md->details_;
uint64_t key[1] = {dev_id};
device_tree::maybe_value v;
try {
v = details_tree->lookup(key);
} catch (std::exception &e) {
cerr << "missing device " << dev_id << ": " << e.what() << endl;
}
if (v)
details = *v;
e->begin_device(dev_id,
0,
details.transaction_id_,
details.creation_time_,
details.snapshotted_time_);
}
void parse_node(metadata::ptr md, emitter::ptr e, attributes const &attr) {
metadata_dump_subtree(md, e, true, get_attr<uint32_t>(attr, "blocknr"));
}
void start_tag(void *data, char const *el, char const **attr) {
user_data *ud = static_cast<user_data *>(data);
attributes a;
build_attributes(a, attr);
if (!strcmp(el, "superblock")) {
open_resources(*ud, a);
parse_superblock(ud->md_, ud->emitter_, a);
} else if (!strcmp(el, "device"))
parse_device(ud->md_, ud->emitter_, a);
else if (!strcmp(el, "node"))
parse_node(ud->md_, ud->emitter_, a);
else
throw runtime_error("unknown tag type");
}
void end_tag(void *data, const char *el) {
user_data *ud = static_cast<user_data *>(data);
if (!strcmp(el, "superblock")) {
ud->emitter_->end_superblock();
XML_StopParser(ud->parser_, XML_FALSE); // skip the rest elements
}
else if (!strcmp(el, "device"))
ud->emitter_->end_device();
else if (!strcmp(el, "node"))
;
else
throw runtime_error("unknown tag type");
}
}
//---------------------------------------------------------------------------
namespace {
struct flags {
flags() {
}
};
int low_level_restore_(string const &src_metadata, string const &input,
string const &output, flags const &f) {
user_data ud;
ud.input_bm_ = open_bm(src_metadata, block_manager<>::READ_ONLY);
ud.output_bm_ = open_bm(output, block_manager<>::READ_WRITE);
xml_parser p;
ud.parser_ = p.get_parser();
XML_SetUserData(p.get_parser(), &ud);
XML_SetElementHandler(p.get_parser(), start_tag, end_tag);
bool quiet = true;
p.parse(input, quiet);
return 0;
}
int low_level_restore(string const &src_metadata, string const &input,
string const &output, flags const &f) {
try {
low_level_restore_(src_metadata, input, output, f);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
}
//---------------------------------------------------------------------------
thin_ll_restore_cmd::thin_ll_restore_cmd()
: command("thin_ll_restore")
{
}
void
thin_ll_restore_cmd::usage(ostream &out) const {
out << "Usage: " << get_name() << " [options]" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-E|--source-metadata} <input device or file>" << endl
<< " {-i|--input} <input xml file>" << endl
<< " {-o|--output} <output device or file>" << endl
<< " {-V|--version}" << endl;
}
int
thin_ll_restore_cmd::run(int argc, char **argv) {
string input;
string output;
string input_metadata;
flags f;
char c;
const char shortopts[] = "hi:o:E:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "input", required_argument, NULL, 'i'},
{ "output", required_argument, NULL, 'o'},
{ "source-metadata", required_argument, NULL, 'E'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'i':
input = optarg;
break;
case 'o':
output = optarg;
break;
case 'E':
input_metadata = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr);
return 1;
}
}
if (argc != optind) {
usage(cerr);
return 1;
}
if (!input_metadata.length() || !input.length() || !output.length()) {
cerr << "No input/output file provided." << endl;
usage(cerr);
return 1;
}
return low_level_restore(input_metadata, input, output, f);
}
//---------------------------------------------------------------------------