Merge pull request #136 from mingnus/thin-generate-metadata-wip

[thin_generate_metadata] Introduce the new transactional metadata generate tool

persistent-data/space-maps/recursive.cc

@@ -97,6 +97,12 @@ namespace {
 				add_op(b, block_op(SET, c));
 			else {
 				recursing_lock lock(*this);
+
+				// the inner set_count may trigger a find_free,
+				// so it's important we update the allocated
+				// blocks list before calling.
+				allocated_blocks_.add(b, b + 1);
+
 				return sm_->set_count(b, c);
 			}
 		}
@@ -111,6 +117,12 @@ namespace {
 				add_op(b, block_op(INC, count));
 			else {
 				recursing_lock lock(*this);
+
+				// the inner inc() may trigger a find_free,
+				// so it's important we update the allocated
+				// blocks list before calling.
+				allocated_blocks_.add(b, b + 1);
+
 				return sm_->inc(b, count);
 			}
 		}
@@ -200,9 +212,11 @@ namespace {
 		void flush_ops_() {
 			recursing_lock lock(*this);
 
-			for (auto const &p : ops_) {
+			while (!ops_.empty()) {
-				block_address b = p.first;
+				auto p = ops_.begin();
-				auto const &op = p.second;
+				block_address b = p->first;
+				auto op = p->second;
+				ops_.erase(p);
 
 				switch (op.op_) {
 				case INC:
@@ -218,7 +232,6 @@ namespace {
 				}
 			}
 
-			ops_.clear();
 			allocated_blocks_.clear();
 		}
 

thin-provisioning/thin_generate_metadata.cc

@@ -19,22 +19,11 @@
 #include "base/output_file_requirements.h"
 #include "persistent-data/file_utils.h"
 #include "thin-provisioning/commands.h"
-#include "thin-provisioning/emitter.h"
-#include "thin-provisioning/human_readable_format.h"
 #include "thin-provisioning/metadata.h"
-#include "thin-provisioning/restore_emitter.h"
-#include "thin-provisioning/xml_format.h"
 #include "version.h"
 
 #include <boost/optional.hpp>
-#include <fstream>
 #include <getopt.h>
-#include <iostream>
-#include <libgen.h>
-#include <linux/fs.h>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
 #include <unistd.h>
 
 using namespace boost;
@@ -46,76 +35,128 @@ using namespace thin_provisioning;
 
 namespace {
 	struct flags {
+		enum metadata_operations {
+			METADATA_OP_NONE,
+			METADATA_OP_FORMAT,
+			METADATA_OP_OPEN,
+			METADATA_OP_CREATE_THIN,
+			METADATA_OP_LAST
+		};
+
 		flags()
-			: data_block_size(128),
+			: op(METADATA_OP_NONE),
-			  nr_data_blocks(10240),
+			  data_block_size(128),
-			  nr_thins(1),
+			  nr_data_blocks(10240)
-			  blocks_per_thin(1024),
+		{
-			  run_lengths(1024) {
 		}
 
-		block_address data_block_size;
+		bool check_conformance();
+
+		metadata_operations op;
+		sector_t data_block_size;
 		block_address nr_data_blocks;
-		unsigned nr_thins;
+		optional<uint64_t> dev_id;
-		block_address blocks_per_thin;
-		block_address run_lengths;
 		optional<string> output;
 	};
 
-	// This is how we stir in some entropy.  It mixes up the data
+	// FIXME: modulize the conditions
-	// device.
+	bool flags::check_conformance() {
-	class shuffler {
+		if (op == METADATA_OP_NONE || op >= METADATA_OP_LAST) {
-	public:
+			cerr << "Invalid operation." << endl;
-		shuffler(block_address nr_blocks, unsigned run_lengths)
+			return false;
-			: nr_blocks_(nr_blocks / run_lengths),
-			  run_lengths_(run_lengths) {
 		}
 
-		block_address map(block_address b) const {
+		if (!output) {
-			return reverse(b / run_lengths_) + (b % run_lengths_);
+			cerr << "No output file provided." << endl;
+			return false;
+		} else
+			check_output_file_requirements(*output);
+
+		if (op == METADATA_OP_CREATE_THIN && !dev_id) {
+			cerr << "no device id provided." << endl;
+			return false;
 		}
 
-	private:
+		return true;
-		block_address reverse(block_address b) const {
-			return nr_blocks_ - b - 1ull;
-		}
-
-		block_address nr_blocks_;
-		block_address run_lengths_;
-	};
-
-	void generate_device(emitter::ptr e, shuffler const &s, uint32_t dev_id,
-			     block_address nr_blocks, block_address base) {
-
-		e->begin_device(dev_id, nr_blocks, 0, 0, 0);
-		for (unsigned b = 0; b < nr_blocks; b++)
-			e->single_map(b, s.map(base + b), 0);
-		e->end_device();
 	}
 
-	void generate_metadata(flags const &fs, emitter::ptr e) {
+	//--------------------------------
-		e->begin_superblock("fake metadata", 0, 0, optional<uint32_t>(), optional<uint32_t>(),
-				    fs.data_block_size, fs.nr_data_blocks, optional<uint64_t>());
 
-		shuffler s(fs.nr_data_blocks, fs.run_lengths);
+	single_mapping_tree::ptr new_mapping_tree(metadata::ptr md) {
-		for (unsigned i = 0; i < fs.nr_thins; i++)
+		return single_mapping_tree::ptr(
-			generate_device(e, s, i, fs.blocks_per_thin, i * fs.blocks_per_thin);
+			new single_mapping_tree(*md->tm_,
-
+						mapping_tree_detail::block_time_ref_counter(md->data_sm_)));
-		e->end_superblock();
 	}
 
-	int create_metadata(flags const &fs) {
+	bool is_device_exists(metadata::ptr md, uint64_t dev_id) {
-		try {
+		uint64_t key[1] = {dev_id};
-			// The block size gets updated by the restorer.
-			block_manager::ptr bm(open_bm(*fs.output, block_manager::READ_WRITE));
-			metadata::ptr md(new metadata(bm, metadata::CREATE, 128, 0));
-			emitter::ptr restorer = create_restore_emitter(md);
 
-			generate_metadata(fs, restorer);
+		device_tree::maybe_value v1 = md->details_->lookup(key);
+		if (v1)
+			return true;
 
-		} catch (std::exception &e) {
+		dev_tree::maybe_value v2 = md->mappings_top_level_->lookup(key);
-			cerr << e.what() << endl;
+		if (v2)
-			return 1;
+			return true;
+
+		return false;
+	}
+
+	//--------------------------------
+
+	metadata::ptr format_metadata(block_manager::ptr bm,
+				      sector_t data_block_size,
+				      block_address nr_data_blocks) {
+		metadata::ptr md(new metadata(bm,
+					      metadata::CREATE,
+					      data_block_size,
+					      nr_data_blocks));
+		md->commit();
+		return md;
+	}
+
+	metadata::ptr open_metadata(block_manager::ptr bm) {
+		metadata::ptr md(new metadata(bm, true));
+		return md;
+	}
+
+	void create_thin(metadata::ptr md, uint64_t dev_id) {
+		uint64_t key[1] = {dev_id};
+
+		if (is_device_exists(md, dev_id))
+			throw runtime_error("device already exists");
+
+		device_tree_detail::device_details details;
+		details.transaction_id_ = md->sb_.trans_id_;
+		details.creation_time_ = md->sb_.time_;
+		details.snapshotted_time_ = details.creation_time_;
+		md->details_->insert(key, details);
+
+		single_mapping_tree::ptr subtree = new_mapping_tree(md);
+		md->mappings_top_level_->insert(key, subtree->get_root());
+		md->mappings_->set_root(md->mappings_top_level_->get_root()); // FIXME: ugly
+
+		md->commit();
+	}
+
+	metadata::ptr open_or_format_metadata(block_manager::ptr bm, flags const &fs) {
+
+		if (fs.op == flags::METADATA_OP_FORMAT)
+			return format_metadata(bm, fs.data_block_size, fs.nr_data_blocks);
+		else
+			return open_metadata(bm);
+	}
+
+	int generate_metadata(flags const &fs) {
+		block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE);
+		metadata::ptr md = open_or_format_metadata(bm, fs);
+
+		switch (fs.op) {
+		case flags::METADATA_OP_CREATE_THIN:
+			create_thin(md, *fs.dev_id);
+			break;
+		default:
+			break;
 		}
 
 		return 0;
@@ -137,9 +178,6 @@ thin_generate_metadata_cmd::usage(std::ostream &out) const
 	    << "  {-h|--help}\n"
 	    << "  --data-block-size <block size>\n"
 	    << "  --nr-data-blocks <nr>\n"
-	    << "  --nr-thins <count>\n"
-	    << "  --blocks-per-thin <count>\n"
-	    << "  --run-lengths <count>\n"
 	    << "  {-o|--output} <output device or file>\n"
 	    << "  {-V|--version}" << endl;
 }
@@ -151,14 +189,15 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 	struct flags fs;
 	const char *shortopts = "hi:o:qV";
 	const struct option longopts[] = {
-		{ "help", no_argument, NULL, 'h'},
+		{ "help", no_argument, NULL, 'h' },
-		{ "output", required_argument, NULL, 'o'},
+		{ "output", required_argument, NULL, 'o' },
-		{ "data-block-size", required_argument, NULL, 1},
+		{ "format", no_argument, NULL, 1 },
-		{ "nr-data-blocks", required_argument, NULL, 2},
+		{ "open", no_argument, NULL, 2 },
-		{ "nr-thins", required_argument, NULL, 3},
+		{ "create-thin", no_argument, NULL, 3 },
-		{ "blocks-per-thin", required_argument, NULL, 4},
+		{ "data-block-size", required_argument, NULL, 101 },
-		{ "run-lengths", required_argument, NULL, 5},
+		{ "nr-data-blocks", required_argument, NULL, 102 },
-		{ "version", no_argument, NULL, 'V'},
+		{ "dev-id", required_argument, NULL, 301 },
+		{ "version", no_argument, NULL, 'V' },
 		{ NULL, no_argument, NULL, 0 }
 	};
 
@@ -173,23 +212,27 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 			break;
 
 		case 1:
-			fs.data_block_size = parse_uint64(optarg, "data block size");
+			fs.op = flags::METADATA_OP_FORMAT;
 			break;
 
 		case 2:
-			fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
+			fs.op = flags::METADATA_OP_OPEN;
 			break;
 
 		case 3:
-			fs.nr_thins = parse_uint64(optarg, "nr thins");
+			fs.op = flags::METADATA_OP_CREATE_THIN;
 			break;
 
-		case 4:
+		case 101:
-			fs.blocks_per_thin = parse_uint64(optarg, "blocks per thin");
+			fs.data_block_size = parse_uint64(optarg, "data block size");
 			break;
 
-		case 5:
+		case 102:
-			fs.run_lengths = parse_uint64(optarg, "run lengths");
+			fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
+			break;
+
+		case 301:
+			fs.dev_id = parse_uint64(optarg, "dev id");
 			break;
 
 		case 'V':
@@ -202,14 +245,12 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 		}
 	}
 
-	if (!fs.output) {
+	if (!fs.check_conformance()) {
-		cerr << "No output file provided.\n\n";
 		usage(cerr);
 		return 1;
-	} else
+	}
-		check_output_file_requirements(*fs.output);
 
-	return create_metadata(fs);
+	return generate_metadata(fs);
 }
 
 //----------------------------------------------------------------