emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into 2020-05-27-check-data-space-map

Browse Source
This commit is contained in:
Joe Thornber
2020-06-09 13:39:46 +01:00
parent 16a10d2554 409a660082
commit 01a7462ece
27 changed files with 2038 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
thin-provisioning/commands.cc
View File

@@ -12,8 +12,6 @@ thin_provisioning::register_thin_commands(base::application &app)
app.add_cmd(command::ptr(new thin_delta_cmd()));
app.add_cmd(command::ptr(new thin_dump_cmd()));
app.add_cmd(command::ptr(new thin_ls_cmd()));
app.add_cmd(command::ptr(new thin_metadata_pack_cmd()));
app.add_cmd(command::ptr(new thin_metadata_unpack_cmd()));
app.add_cmd(command::ptr(new thin_metadata_size_cmd()));
app.add_cmd(command::ptr(new thin_restore_cmd()));
app.add_cmd(command::ptr(new thin_repair_cmd()));

16
thin-provisioning/commands.h
View File

@@ -71,22 +71,6 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv);
};
class thin_metadata_pack_cmd : public base::command {
public:
thin_metadata_pack_cmd();
virtual void usage(std::ostream &out) const override;
virtual int run(int argc, char **argv) override;
};
class thin_metadata_unpack_cmd : public base::command {
public:
thin_metadata_unpack_cmd();
virtual void usage(std::ostream &out) const override;
virtual int run(int argc, char **argv) override;
};
#ifdef DEV_TOOLS
class thin_ll_dump_cmd : public base::command {
public:

213
thin-provisioning/thin_generate_metadata.cc
View File

@@ -19,22 +19,11 @@
#include "base/output_file_requirements.h"
#include "persistent-data/file_utils.h"
#include "thin-provisioning/commands.h"
#include "thin-provisioning/emitter.h"
#include "thin-provisioning/human_readable_format.h"
#include "thin-provisioning/metadata.h"
#include "thin-provisioning/restore_emitter.h"
#include "thin-provisioning/xml_format.h"
#include "version.h"
#include <boost/optional.hpp>
#include <fstream>
#include <getopt.h>
#include <iostream>
#include <libgen.h>
#include <linux/fs.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
using namespace boost;
@@ -46,76 +35,128 @@ using namespace thin_provisioning;
namespace {
struct flags {
enum metadata_operations {
METADATA_OP_NONE,
METADATA_OP_FORMAT,
METADATA_OP_OPEN,
METADATA_OP_CREATE_THIN,
METADATA_OP_LAST
};
flags()
: data_block_size(128),
nr_data_blocks(10240),
nr_thins(1),
blocks_per_thin(1024),
run_lengths(1024) {
: op(METADATA_OP_NONE),
data_block_size(128),
nr_data_blocks(10240)
{
}
block_address data_block_size;
bool check_conformance();
metadata_operations op;
sector_t data_block_size;
block_address nr_data_blocks;
unsigned nr_thins;
block_address blocks_per_thin;
block_address run_lengths;
optional<uint64_t> dev_id;
optional<string> output;
};
// This is how we stir in some entropy. It mixes up the data
// device.
class shuffler {
public:
shuffler(block_address nr_blocks, unsigned run_lengths)
: nr_blocks_(nr_blocks / run_lengths),
run_lengths_(run_lengths) {
// FIXME: modulize the conditions
bool flags::check_conformance() {
if (op == METADATA_OP_NONE || op >= METADATA_OP_LAST) {
cerr << "Invalid operation." << endl;
return false;
}
block_address map(block_address b) const {
return reverse(b / run_lengths_) + (b % run_lengths_);
if (!output) {
cerr << "No output file provided." << endl;
return false;
} else
check_output_file_requirements(*output);
if (op == METADATA_OP_CREATE_THIN && !dev_id) {
cerr << "no device id provided." << endl;
return false;
}
private:
block_address reverse(block_address b) const {
return nr_blocks_ - b - 1ull;
}
block_address nr_blocks_;
block_address run_lengths_;
};
void generate_device(emitter::ptr e, shuffler const &s, uint32_t dev_id,
block_address nr_blocks, block_address base) {
e->begin_device(dev_id, nr_blocks, 0, 0, 0);
for (unsigned b = 0; b < nr_blocks; b++)
e->single_map(b, s.map(base + b), 0);
e->end_device();
return true;
}
void generate_metadata(flags const &fs, emitter::ptr e) {
e->begin_superblock("fake metadata", 0, 0, optional<uint32_t>(), optional<uint32_t>(),
fs.data_block_size, fs.nr_data_blocks, optional<uint64_t>());
//--------------------------------
shuffler s(fs.nr_data_blocks, fs.run_lengths);
for (unsigned i = 0; i < fs.nr_thins; i++)
generate_device(e, s, i, fs.blocks_per_thin, i * fs.blocks_per_thin);
e->end_superblock();
single_mapping_tree::ptr new_mapping_tree(metadata::ptr md) {
return single_mapping_tree::ptr(
new single_mapping_tree(*md->tm_,
mapping_tree_detail::block_time_ref_counter(md->data_sm_)));
}
int create_metadata(flags const &fs) {
try {
// The block size gets updated by the restorer.
block_manager::ptr bm(open_bm(*fs.output, block_manager::READ_WRITE));
metadata::ptr md(new metadata(bm, metadata::CREATE, 128, 0));
emitter::ptr restorer = create_restore_emitter(md);
bool is_device_exists(metadata::ptr md, uint64_t dev_id) {
uint64_t key[1] = {dev_id};
generate_metadata(fs, restorer);
device_tree::maybe_value v1 = md->details_->lookup(key);
if (v1)
return true;
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
dev_tree::maybe_value v2 = md->mappings_top_level_->lookup(key);
if (v2)
return true;
return false;
}
//--------------------------------
metadata::ptr format_metadata(block_manager::ptr bm,
sector_t data_block_size,
block_address nr_data_blocks) {
metadata::ptr md(new metadata(bm,
metadata::CREATE,
data_block_size,
nr_data_blocks));
md->commit();
return md;
}
metadata::ptr open_metadata(block_manager::ptr bm) {
metadata::ptr md(new metadata(bm, true));
return md;
}
void create_thin(metadata::ptr md, uint64_t dev_id) {
uint64_t key[1] = {dev_id};
if (is_device_exists(md, dev_id))
throw runtime_error("device already exists");
device_tree_detail::device_details details;
details.transaction_id_ = md->sb_.trans_id_;
details.creation_time_ = md->sb_.time_;
details.snapshotted_time_ = details.creation_time_;
md->details_->insert(key, details);
single_mapping_tree::ptr subtree = new_mapping_tree(md);
md->mappings_top_level_->insert(key, subtree->get_root());
md->mappings_->set_root(md->mappings_top_level_->get_root()); // FIXME: ugly
md->commit();
}
metadata::ptr open_or_format_metadata(block_manager::ptr bm, flags const &fs) {
if (fs.op == flags::METADATA_OP_FORMAT)
return format_metadata(bm, fs.data_block_size, fs.nr_data_blocks);
else
return open_metadata(bm);
}
int generate_metadata(flags const &fs) {
block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE);
metadata::ptr md = open_or_format_metadata(bm, fs);
switch (fs.op) {
case flags::METADATA_OP_CREATE_THIN:
create_thin(md, *fs.dev_id);
break;
default:
break;
}
return 0;
@@ -137,9 +178,6 @@ thin_generate_metadata_cmd::usage(std::ostream &out) const
<< " {-h|--help}\n"
<< " --data-block-size <block size>\n"
<< " --nr-data-blocks <nr>\n"
<< " --nr-thins <count>\n"
<< " --blocks-per-thin <count>\n"
<< " --run-lengths <count>\n"
<< " {-o|--output} <output device or file>\n"
<< " {-V|--version}" << endl;
}
@@ -151,14 +189,15 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
struct flags fs;
const char *shortopts = "hi:o:qV";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "output", required_argument, NULL, 'o'},
{ "data-block-size", required_argument, NULL, 1},
{ "nr-data-blocks", required_argument, NULL, 2},
{ "nr-thins", required_argument, NULL, 3},
{ "blocks-per-thin", required_argument, NULL, 4},
{ "run-lengths", required_argument, NULL, 5},
{ "version", no_argument, NULL, 'V'},
{ "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' },
{ "format", no_argument, NULL, 1 },
{ "open", no_argument, NULL, 2 },
{ "create-thin", no_argument, NULL, 3 },
{ "data-block-size", required_argument, NULL, 101 },
{ "nr-data-blocks", required_argument, NULL, 102 },
{ "dev-id", required_argument, NULL, 301 },
{ "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 }
};
@@ -173,23 +212,27 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
break;
case 1:
fs.data_block_size = parse_uint64(optarg, "data block size");
fs.op = flags::METADATA_OP_FORMAT;
break;
case 2:
fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
fs.op = flags::METADATA_OP_OPEN;
break;
case 3:
fs.nr_thins = parse_uint64(optarg, "nr thins");
fs.op = flags::METADATA_OP_CREATE_THIN;
break;
case 4:
fs.blocks_per_thin = parse_uint64(optarg, "blocks per thin");
case 101:
fs.data_block_size = parse_uint64(optarg, "data block size");
break;
case 5:
fs.run_lengths = parse_uint64(optarg, "run lengths");
case 102:
fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
break;
case 301:
fs.dev_id = parse_uint64(optarg, "dev id");
break;
case 'V':
@@ -202,14 +245,12 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
}
}
if (!fs.output) {
cerr << "No output file provided.\n\n";
if (!fs.check_conformance()) {
usage(cerr);
return 1;
} else
check_output_file_requirements(*fs.output);
}
return create_metadata(fs);
return generate_metadata(fs);
}
//----------------------------------------------------------------

338
thin-provisioning/thin_metadata_pack.cc
View File

@@ -1,338 +0,0 @@
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/filter/zlib.hpp>
#include <boost/optional.hpp>
#include <fcntl.h>
#include <fstream>
#include <getopt.h>
#include <sys/types.h>
#include <unistd.h>
#include <vector>
#include "persistent-data/file_utils.h"
#include "persistent-data/space-maps/disk.h"
#include "persistent-data/checksum.h"
#include "thin-provisioning/commands.h"
#include "thin-provisioning/superblock.h"
#include "version.h"
using namespace thin_provisioning;
using namespace persistent_data;
using boost::optional;
//---------------------------------------------------------------------------
namespace {
using namespace std;
constexpr uint64_t MAGIC = 0xa537a0aa6309ef77;
constexpr uint64_t PACK_VERSION = 1;
uint32_t const SUPERBLOCK_CSUM_SEED = 160774;
uint32_t const BITMAP_CSUM_XOR = 240779;
uint32_t const INDEX_CSUM_XOR = 160478;
uint32_t const BTREE_CSUM_XOR = 121107;
// Pack file format
// ----------------
//
// file := <file-header> <entry>*
// file-header := MAGIC BLOCK_SIZE NR_BLOCKS NR_ENTRIES
// entry := BLOCK_NR BYTES class flags {
struct flags {
optional<string> input_file_;
optional<string> output_file_;
};
class is_metadata_functor {
public:
is_metadata_functor() {
}
bool operator() (void const *raw) const {
uint32_t const *cksum = reinterpret_cast<uint32_t const*>(raw);
base::crc32c sum(*cksum);
sum.append(cksum + 1, MD_BLOCK_SIZE - sizeof(uint32_t));
switch (sum.get_sum()) {
case SUPERBLOCK_CSUM_SEED:
case INDEX_CSUM_XOR:
case BITMAP_CSUM_XOR:
case BTREE_CSUM_XOR:
return true;
default:
return false;
}
}
};
void prealloc_file(string const &file, off_t len) {
int fd = ::open(file.c_str(), O_TRUNC | O_CREAT | O_RDWR, 0666);
if (fd < 0)
throw runtime_error("couldn't open output file");
if (::fallocate(fd, 0, 0, len))
throw runtime_error("couldn't fallocate");
::close(fd);
}
uint64_t read_u64(istream &in) {
base::le64 n;
in.read(reinterpret_cast<char *>(&n), sizeof(n));
if (!in)
throw runtime_error("couldn't read u64");
return base::to_cpu<uint64_t>(n);
}
void write_u64(ostream &out, uint64_t n) {
base::le64 n_le = base::to_disk<base::le64>(n);
out.write(reinterpret_cast<char *>(&n_le), sizeof(n_le));
if (!out)
throw runtime_error("couldn't write u64");
}
int pack(flags const &f) {
using namespace boost::iostreams;
std::ofstream out_file(*f.output_file_, ios_base::binary);
write_u64(out_file, MAGIC);
write_u64(out_file, PACK_VERSION);
boost::iostreams::filtering_ostreambuf out_buf;
out_buf.push(zlib_compressor());
out_buf.push(out_file);
std::ostream out(&out_buf);
block_manager::ptr bm = open_bm(*f.input_file_, block_manager::READ_ONLY, true);
uint64_t block_size = 4096;
auto nr_blocks = bm->get_nr_blocks();
cerr << "nr_blocks = " << nr_blocks << "\n";
write_u64(out, block_size);
write_u64(out, nr_blocks);
is_metadata_functor is_metadata;
for (block_address b = 0; b < nr_blocks; b++) {
auto rr = bm->read_lock(b);
if (is_metadata(rr.data())) {
write_u64(out, b);
out.write(reinterpret_cast<const char *>(rr.data()), block_size);
}
}
return 0;
}
int unpack(flags const &f)
{
using namespace boost::iostreams;
ifstream in_file(*f.input_file_, ios_base::binary);
if (!in_file)
throw runtime_error("Couldn't open pack file");
if (read_u64(in_file) != MAGIC)
throw runtime_error("Not a pack file.");
if (read_u64(in_file) != PACK_VERSION)
throw runtime_error("unknown pack file format version");
filtering_istreambuf in_buf;
in_buf.push(zlib_decompressor());
in_buf.push(in_file);
std::istream in(&in_buf);
auto block_size = read_u64(in);
auto nr_blocks = read_u64(in);
prealloc_file(*f.output_file_, nr_blocks * block_size);
block_manager bm(*f.output_file_, nr_blocks, 6, block_manager::READ_WRITE, true);
uint8_t bytes[block_size];
while (true) {
uint64_t block_nr;
try {
block_nr = read_u64(in);
} catch (...) {
break;
}
if (block_nr >= nr_blocks)
throw runtime_error("block nr out of bounds");
in.read(reinterpret_cast<char *>(bytes), block_size);
if (!in)
throw runtime_error("couldn't read data");
auto wr = bm.write_lock(block_nr);
memcpy(wr.data(), bytes, block_size);
}
return 0;
}
}
//---------------------------------------------------------------------------
thin_metadata_pack_cmd::thin_metadata_pack_cmd()
: command("thin_metadata_pack")
{
}
void
thin_metadata_pack_cmd::usage(ostream &out) const {
out << "Usage: " << get_name() << " [options]\n"
<< "Options:\n"
<< " {-i|--input} <input metadata (binary format)>\n"
<< " {-o|--output} <output packed metadata>\n"
<< " {-h|--help}\n"
<< " {-V|--version}" << endl;
}
int
thin_metadata_pack_cmd::run(int argc, char **argv)
{
const char shortopts[] = "hi:o:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "input", required_argument, NULL, 'i'},
{ "output", required_argument, NULL, 'o'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
flags f;
int c;
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'i':
f.input_file_ = optarg;
break;
case 'o':
f.output_file_ = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr);
return 1;
}
}
if (!f.input_file_) {
cerr << "No input file provided." << endl;
usage(cerr);
return 1;
}
if (!f.output_file_) {
cerr << "No output file provided." << endl;
usage(cerr);
return 1;
}
return pack(f);
}
//---------------------------------------------------------------------------
thin_metadata_unpack_cmd::thin_metadata_unpack_cmd()
: command("thin_metadata_unpack")
{
}
void
thin_metadata_unpack_cmd::usage(ostream &out) const {
out << "Usage: " << get_name() << " [options]\n"
<< "Options:\n"
<< " {-i|--input} <input packed metadata>\n"
<< " {-o|--output} <output metadata (binary format)>\n"
<< " {-h|--help}\n"
<< " {-V|--version}" << endl;
}
int
thin_metadata_unpack_cmd::run(int argc, char **argv)
{
const char shortopts[] = "hi:o:V";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "input", required_argument, NULL, 'i'},
{ "output", required_argument, NULL, 'o'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
flags f;
int c;
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'i':
f.input_file_ = optarg;
break;
case 'o':
f.output_file_ = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr);
return 1;
}
}
if (!f.input_file_) {
cerr << "No input file provided." << endl;
usage(cerr);
return 1;
}
if (!f.output_file_) {
cerr << "No output file provided." << endl;
usage(cerr);
return 1;
}
return unpack(f);
}
//---------------------------------------------------------------------------