From 0c6d132774fb6fdcfd3df9d3f45fabb4b186a297 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 27 May 2020 12:01:59 +0800 Subject: [PATCH 1/4] [thin_generate_metadata] Preparation for revising the features --- thin-provisioning/thin_generate_metadata.cc | 114 ++------------------ 1 file changed, 10 insertions(+), 104 deletions(-) diff --git a/thin-provisioning/thin_generate_metadata.cc b/thin-provisioning/thin_generate_metadata.cc index 0891864..7a7c990 100644 --- a/thin-provisioning/thin_generate_metadata.cc +++ b/thin-provisioning/thin_generate_metadata.cc @@ -17,24 +17,12 @@ // . #include "base/output_file_requirements.h" -#include "persistent-data/file_utils.h" #include "thin-provisioning/commands.h" -#include "thin-provisioning/emitter.h" -#include "thin-provisioning/human_readable_format.h" #include "thin-provisioning/metadata.h" -#include "thin-provisioning/restore_emitter.h" -#include "thin-provisioning/xml_format.h" #include "version.h" #include -#include #include -#include -#include -#include -#include -#include -#include #include using namespace boost; @@ -48,78 +36,14 @@ namespace { struct flags { flags() : data_block_size(128), - nr_data_blocks(10240), - nr_thins(1), - blocks_per_thin(1024), - run_lengths(1024) { + nr_data_blocks(10240) + { } block_address data_block_size; block_address nr_data_blocks; - unsigned nr_thins; - block_address blocks_per_thin; - block_address run_lengths; optional output; }; - - // This is how we stir in some entropy. It mixes up the data - // device. - class shuffler { - public: - shuffler(block_address nr_blocks, unsigned run_lengths) - : nr_blocks_(nr_blocks / run_lengths), - run_lengths_(run_lengths) { - } - - block_address map(block_address b) const { - return reverse(b / run_lengths_) + (b % run_lengths_); - } - - private: - block_address reverse(block_address b) const { - return nr_blocks_ - b - 1ull; - } - - block_address nr_blocks_; - block_address run_lengths_; - }; - - void generate_device(emitter::ptr e, shuffler const &s, uint32_t dev_id, - block_address nr_blocks, block_address base) { - - e->begin_device(dev_id, nr_blocks, 0, 0, 0); - for (unsigned b = 0; b < nr_blocks; b++) - e->single_map(b, s.map(base + b), 0); - e->end_device(); - } - - void generate_metadata(flags const &fs, emitter::ptr e) { - e->begin_superblock("fake metadata", 0, 0, optional(), optional(), - fs.data_block_size, fs.nr_data_blocks, optional()); - - shuffler s(fs.nr_data_blocks, fs.run_lengths); - for (unsigned i = 0; i < fs.nr_thins; i++) - generate_device(e, s, i, fs.blocks_per_thin, i * fs.blocks_per_thin); - - e->end_superblock(); - } - - int create_metadata(flags const &fs) { - try { - // The block size gets updated by the restorer. - block_manager::ptr bm(open_bm(*fs.output, block_manager::READ_WRITE)); - metadata::ptr md(new metadata(bm, metadata::CREATE, 128, 0)); - emitter::ptr restorer = create_restore_emitter(md); - - generate_metadata(fs, restorer); - - } catch (std::exception &e) { - cerr << e.what() << endl; - return 1; - } - - return 0; - } } //---------------------------------------------------------------- @@ -137,9 +61,6 @@ thin_generate_metadata_cmd::usage(std::ostream &out) const << " {-h|--help}\n" << " --data-block-size \n" << " --nr-data-blocks \n" - << " --nr-thins \n" - << " --blocks-per-thin \n" - << " --run-lengths \n" << " {-o|--output} \n" << " {-V|--version}" << endl; } @@ -151,14 +72,11 @@ thin_generate_metadata_cmd::run(int argc, char **argv) struct flags fs; const char *shortopts = "hi:o:qV"; const struct option longopts[] = { - { "help", no_argument, NULL, 'h'}, - { "output", required_argument, NULL, 'o'}, - { "data-block-size", required_argument, NULL, 1}, - { "nr-data-blocks", required_argument, NULL, 2}, - { "nr-thins", required_argument, NULL, 3}, - { "blocks-per-thin", required_argument, NULL, 4}, - { "run-lengths", required_argument, NULL, 5}, - { "version", no_argument, NULL, 'V'}, + { "help", no_argument, NULL, 'h' }, + { "output", required_argument, NULL, 'o' }, + { "data-block-size", required_argument, NULL, 101 }, + { "nr-data-blocks", required_argument, NULL, 102 }, + { "version", no_argument, NULL, 'V' }, { NULL, no_argument, NULL, 0 } }; @@ -172,26 +90,14 @@ thin_generate_metadata_cmd::run(int argc, char **argv) fs.output = optarg; break; - case 1: + case 101: fs.data_block_size = parse_uint64(optarg, "data block size"); break; - case 2: + case 102: fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks"); break; - case 3: - fs.nr_thins = parse_uint64(optarg, "nr thins"); - break; - - case 4: - fs.blocks_per_thin = parse_uint64(optarg, "blocks per thin"); - break; - - case 5: - fs.run_lengths = parse_uint64(optarg, "run lengths"); - break; - case 'V': cout << THIN_PROVISIONING_TOOLS_VERSION << endl; return 0; @@ -209,7 +115,7 @@ thin_generate_metadata_cmd::run(int argc, char **argv) } else check_output_file_requirements(*fs.output); - return create_metadata(fs); + return 0; } //---------------------------------------------------------------- From 0f2918e98908ef073d849c1cf675854702f812cc Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 27 May 2020 14:48:35 +0800 Subject: [PATCH 2/4] [thin_generate_metadata] Implement format_metadata and create_thin --- thin-provisioning/thin_generate_metadata.cc | 149 +++++++++++++++++++- 1 file changed, 142 insertions(+), 7 deletions(-) diff --git a/thin-provisioning/thin_generate_metadata.cc b/thin-provisioning/thin_generate_metadata.cc index 7a7c990..f7127cc 100644 --- a/thin-provisioning/thin_generate_metadata.cc +++ b/thin-provisioning/thin_generate_metadata.cc @@ -17,6 +17,7 @@ // . #include "base/output_file_requirements.h" +#include "persistent-data/file_utils.h" #include "thin-provisioning/commands.h" #include "thin-provisioning/metadata.h" #include "version.h" @@ -34,16 +35,132 @@ using namespace thin_provisioning; namespace { struct flags { + enum metadata_operations { + METADATA_OP_NONE, + METADATA_OP_FORMAT, + METADATA_OP_OPEN, + METADATA_OP_CREATE_THIN, + METADATA_OP_LAST + }; + flags() - : data_block_size(128), + : op(METADATA_OP_NONE), + data_block_size(128), nr_data_blocks(10240) { } - block_address data_block_size; + bool check_conformance(); + + metadata_operations op; + sector_t data_block_size; block_address nr_data_blocks; + optional dev_id; optional output; }; + + // FIXME: modulize the conditions + bool flags::check_conformance() { + if (op == METADATA_OP_NONE || op >= METADATA_OP_LAST) { + cerr << "Invalid operation." << endl; + return false; + } + + if (!output) { + cerr << "No output file provided." << endl; + return false; + } else + check_output_file_requirements(*output); + + if (op == METADATA_OP_CREATE_THIN && !dev_id) { + cerr << "no device id provided." << endl; + return false; + } + + return true; + } + + //-------------------------------- + + single_mapping_tree::ptr new_mapping_tree(metadata::ptr md) { + return single_mapping_tree::ptr( + new single_mapping_tree(*md->tm_, + mapping_tree_detail::block_time_ref_counter(md->data_sm_))); + } + + bool is_device_exists(metadata::ptr md, uint64_t dev_id) { + uint64_t key[1] = {dev_id}; + + device_tree::maybe_value v1 = md->details_->lookup(key); + if (v1) + return true; + + dev_tree::maybe_value v2 = md->mappings_top_level_->lookup(key); + if (v2) + return true; + + return false; + } + + //-------------------------------- + + metadata::ptr format_metadata(block_manager::ptr bm, + sector_t data_block_size, + block_address nr_data_blocks) { + metadata::ptr md(new metadata(bm, + metadata::CREATE, + data_block_size, + nr_data_blocks)); + md->commit(); + return md; + } + + metadata::ptr open_metadata(block_manager::ptr bm) { + metadata::ptr md(new metadata(bm, true)); + return md; + } + + void create_thin(metadata::ptr md, uint64_t dev_id) { + uint64_t key[1] = {dev_id}; + + if (is_device_exists(md, dev_id)) + throw runtime_error("device already exists"); + + device_tree_detail::device_details details; + details.transaction_id_ = md->sb_.trans_id_; + details.creation_time_ = md->sb_.time_; + details.snapshotted_time_ = details.creation_time_; + md->details_->insert(key, details); + + single_mapping_tree::ptr subtree = new_mapping_tree(md); + md->mappings_top_level_->insert(key, subtree->get_root()); + md->mappings_->set_root(md->mappings_top_level_->get_root()); // FIXME: ugly + + md->commit(); + } + + metadata::ptr open_or_format_metadata(block_manager::ptr bm, flags const &fs) { + + if (fs.op == flags::METADATA_OP_FORMAT) + return format_metadata(bm, fs.data_block_size, fs.nr_data_blocks); + else + return open_metadata(bm); + } + + int generate_metadata(flags const &fs) { + block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE); + metadata::ptr md = open_or_format_metadata(bm, fs); + + switch (fs.op) { + case flags::METADATA_OP_CREATE_THIN: + create_thin(md, *fs.dev_id); + break; + default: + break; + } + + return 0; + } } //---------------------------------------------------------------- @@ -74,8 +191,12 @@ thin_generate_metadata_cmd::run(int argc, char **argv) const struct option longopts[] = { { "help", no_argument, NULL, 'h' }, { "output", required_argument, NULL, 'o' }, + { "format", no_argument, NULL, 1 }, + { "open", no_argument, NULL, 2 }, + { "create-thin", no_argument, NULL, 3 }, { "data-block-size", required_argument, NULL, 101 }, { "nr-data-blocks", required_argument, NULL, 102 }, + { "dev-id", required_argument, NULL, 301 }, { "version", no_argument, NULL, 'V' }, { NULL, no_argument, NULL, 0 } }; @@ -90,6 +211,18 @@ thin_generate_metadata_cmd::run(int argc, char **argv) fs.output = optarg; break; + case 1: + fs.op = flags::METADATA_OP_FORMAT; + break; + + case 2: + fs.op = flags::METADATA_OP_OPEN; + break; + + case 3: + fs.op = flags::METADATA_OP_CREATE_THIN; + break; + case 101: fs.data_block_size = parse_uint64(optarg, "data block size"); break; @@ -98,6 +231,10 @@ thin_generate_metadata_cmd::run(int argc, char **argv) fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks"); break; + case 301: + fs.dev_id = parse_uint64(optarg, "dev id"); + break; + case 'V': cout << THIN_PROVISIONING_TOOLS_VERSION << endl; return 0; @@ -108,14 +245,12 @@ thin_generate_metadata_cmd::run(int argc, char **argv) } } - if (!fs.output) { - cerr << "No output file provided.\n\n"; + if (!fs.check_conformance()) { usage(cerr); return 1; - } else - check_output_file_requirements(*fs.output); + } - return 0; + return generate_metadata(fs); } //---------------------------------------------------------------- From 4cdffafe88524028ae28fc183b5d10619ea8dc52 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 29 Mar 2017 14:58:12 +0100 Subject: [PATCH 3/4] [sm_recursive] Fix bug that allowed the same block to be allocated twice. See issue 70. Soln from mingnus. --- persistent-data/space-maps/recursive.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/persistent-data/space-maps/recursive.cc b/persistent-data/space-maps/recursive.cc index dadbe88..0977e61 100644 --- a/persistent-data/space-maps/recursive.cc +++ b/persistent-data/space-maps/recursive.cc @@ -97,6 +97,12 @@ namespace { add_op(b, block_op(SET, c)); else { recursing_lock lock(*this); + + // the inner set_count may trigger a find_free, + // so it's important we update the allocated + // blocks list before calling. + allocated_blocks_.add(b, b + 1); + return sm_->set_count(b, c); } } @@ -111,6 +117,12 @@ namespace { add_op(b, block_op(INC, count)); else { recursing_lock lock(*this); + + // the inner inc() may trigger a find_free, + // so it's important we update the allocated + // blocks list before calling. + allocated_blocks_.add(b, b + 1); + return sm_->inc(b, count); } } From 86139cf6bd7826241e7cb81e4a9bea2f8ef6e761 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 28 May 2020 13:38:48 +0800 Subject: [PATCH 4/4] [sm_recursive] Fix atomicity when applying BOPs Any BOP should be taken only once. Therefore, we should remove BOPs that are being processed or had been processed from the uncommitted list. --- persistent-data/space-maps/recursive.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/persistent-data/space-maps/recursive.cc b/persistent-data/space-maps/recursive.cc index 0977e61..a6643da 100644 --- a/persistent-data/space-maps/recursive.cc +++ b/persistent-data/space-maps/recursive.cc @@ -212,9 +212,11 @@ namespace { void flush_ops_() { recursing_lock lock(*this); - for (auto const &p : ops_) { - block_address b = p.first; - auto const &op = p.second; + while (!ops_.empty()) { + auto p = ops_.begin(); + block_address b = p->first; + auto op = p->second; + ops_.erase(p); switch (op.op_) { case INC: @@ -230,7 +232,6 @@ namespace { } } - ops_.clear(); allocated_blocks_.clear(); }