Merge pull request #144 from mingnus/thin-generate-metadata-wip

Enhance thin_generate_mappings, and introduce thin_generate_damage

Makefile.in

@@ -123,11 +123,13 @@ SOURCE=\
 	thin-provisioning/xml_format.cc
 
 DEVTOOLS_SOURCE=\
+	thin-provisioning/damage_generator.cc \
 	thin-provisioning/thin_journal.cc \
 	thin-provisioning/thin_journal_check.cc \
 	thin-provisioning/thin_ll_dump.cc \
 	thin-provisioning/thin_ll_restore.cc \
 	thin-provisioning/thin_show_duplicates.cc \
+	thin-provisioning/thin_generate_damage.cc \
 	thin-provisioning/thin_generate_metadata.cc \
 	thin-provisioning/thin_generate_mappings.cc \
 	thin-provisioning/variable_chunk_stream.cc \

base/io_generator.cc

@@ -1,4 +1,5 @@
 #include "base/io_generator.h"
+#include "persistent-data/run_set.h"
 #include <stdexcept>
 #include <cstdlib>
 #include <cstring>
@@ -25,61 +26,146 @@ namespace {
 
 	//--------------------------------
 
-	class offset_generator {
+	class sequence_generator {
 	public:
-		typedef std::shared_ptr<offset_generator> ptr;
+		typedef std::shared_ptr<sequence_generator> ptr;
 
-		virtual base::sector_t next_offset() = 0;
+		virtual uint64_t next() = 0;
 	};
 
-	class sequential_offset_generator: public offset_generator {
+	// - The maximum generated value is not greater than (begin+size)
+	// - The generated values are not aligned to the step, if the begin
+	//   value is not aligned.
+	class forward_sequence_generator: public sequence_generator {
 	public:
-		sequential_offset_generator(base::sector_t offset,
+		forward_sequence_generator(uint64_t begin,
-					    base::sector_t size,
+					   uint64_t size,
-					    base::sector_t block_size)
+					   uint64_t step)
-			: block_size_(block_size),
+			: begin_(begin),
-			  begin_(offset),
+			  end_(begin + (size / step) * step),
-			  end_(offset + size),
+			  step_(step),
-			  current_(offset) {
+			  current_(begin),
-			if (size < block_size)
+			  rounds_(0) {
-				throw std::runtime_error("size must be greater than block_size");
+			if (size < step)
+				throw std::runtime_error("size must be greater than the step");
 		}
 
-		base::sector_t next_offset() {
+		forward_sequence_generator()
-			sector_t r = current_;
+			: begin_(0), end_(1), step_(1),
-			current_ += block_size_;
+			  current_(0), rounds_(0) {
-			if (current_ > end_) {
+		}
+
+		uint64_t next() {
+			uint64_t r = current_;
+			current_ += step_;
+			if (current_ >= end_) {
 				current_ = begin_;
-				return begin_;
+				++rounds_;
 			}
 			return r;
 		}
 
+		void reset(uint64_t begin, uint64_t size, uint64_t step = 1) {
+			begin_ = begin;
+			end_ = begin + (size / step) * step;
+			step_ = step;
+			current_ = begin;
+			rounds_ = 0;
+		}
+
+		uint64_t get_rounds() {
+			return rounds_;
+		}
+
 	private:
-		unsigned block_size_;
+		uint64_t begin_;
-		base::sector_t begin_;
+		uint64_t end_;
-		base::sector_t end_;
+		uint64_t step_;
-		base::sector_t current_;
+		uint64_t current_;
+		uint64_t rounds_;
 	};
 
-	class random_offset_generator: public offset_generator {
+	// - The maximum generated value is not greater than (begin+size)
+	// - The generated values are not aligned to the step, if the begin
+	//   value is not aligned.
+	class random_sequence_generator: public sequence_generator {
 	public:
-		random_offset_generator(sector_t offset,
+		// TODO: load random seeds
-					sector_t size,
+		random_sequence_generator(uint64_t begin,
-					sector_t block_size)
+					  uint64_t size,
-			: block_begin_(offset / block_size),
+					  uint64_t step,
-			  nr_blocks_(size / block_size),
+					  unsigned seq_nr = 1)
-			  block_size_(block_size) {
+			: begin_(begin),
+			  nr_steps_(size / step),
+			  step_(step),
+			  max_forward_steps_(seq_nr),
+			  nr_generated_(0)
+		{
+			if (!max_forward_steps_ || max_forward_steps_ > nr_steps_)
+				throw std::runtime_error("invalid number of forward steps");
+
+			if (max_forward_steps_ > 1)
+				reset_forward_generator();
 		}
 
-		sector_t next_offset() {
+		uint64_t next() {
-			return ((std::rand() % nr_blocks_) + block_begin_) * block_size_;
+			// FIXME: eliminate if-else
+			uint64_t step_idx = (max_forward_steps_ > 1) ?
+				next_forward_step() : next_random_step();
+			rand_map_.add(step_idx);
+			++nr_generated_;
+
+			// wrap-around
+			if (nr_generated_ == nr_steps_) {
+				rand_map_.clear();
+				nr_generated_ = 0;
+			}
+
+			return begin_ + step_idx * step_;
 		}
 
 	private:
-		uint64_t block_begin_;
+		void reset_forward_generator() {
-		uint64_t nr_blocks_;
+			uint64_t begin = next_random_step();
-		unsigned block_size_;
+			unsigned seq_nr = (std::rand() % max_forward_steps_) + 1;
+			forward_gen_.reset(begin, seq_nr);
+		}
+
+		uint64_t next_forward_step() {
+			uint64_t step_idx;
+
+			bool found = true;
+			while (found) {
+				step_idx = forward_gen_.next();
+				found = rand_map_.member(step_idx);
+
+				if (found || forward_gen_.get_rounds())
+					reset_forward_generator();
+			}
+
+			return step_idx;
+		}
+
+		uint64_t next_random_step() const {
+			uint64_t step_idx;
+
+			bool found = true;
+			while (found) {
+				step_idx = std::rand() % nr_steps_;
+				found = rand_map_.member(step_idx);
+			}
+
+			return step_idx;
+		}
+
+		uint64_t begin_;
+		uint64_t nr_steps_;
+		uint64_t step_;
+		unsigned max_forward_steps_;
+
+		base::run_set<uint64_t> rand_map_;
+		uint64_t nr_generated_;
+		forward_sequence_generator forward_gen_;
 	};
 
 	//--------------------------------
@@ -120,13 +206,13 @@ namespace {
 		virtual bool next(base::io &next_io);
 
 	private:
-		offset_generator::ptr
+		sequence_generator::ptr
 		create_offset_generator(io_generator_options const &opts);
 
 		op_generator::ptr
 		create_op_generator(io_generator_options const &opts);
 
-		offset_generator::ptr offset_gen_;
+		sequence_generator::ptr offset_gen_;
 		op_generator::ptr op_gen_;
 		sector_t block_size_;
 		size_t io_size_finished_;
@@ -146,7 +232,7 @@ namespace {
 			return false;
 
 		next_io.op_ = op_gen_->next_op();
-		next_io.sector_ = offset_gen_->next_offset();
+		next_io.sector_ = offset_gen_->next();
 		next_io.size_ = block_size_;
 
 		io_size_finished_ += block_size_;
@@ -154,18 +240,19 @@ namespace {
 		return true;
 	}
 
-	offset_generator::ptr
+	sequence_generator::ptr
 	base_io_generator::create_offset_generator(io_generator_options const &opts) {
-		if (opts.pattern_.is_random())
+		sequence_generator *gen;
-			return offset_generator::ptr(
-				new random_offset_generator(opts.offset_,
-							    opts.size_,
-							    opts.block_size_));
 
-		return offset_generator::ptr(
+		if (opts.pattern_.is_random())
-			new sequential_offset_generator(opts.offset_,
+			gen = new random_sequence_generator(opts.offset_,
-							opts.size_,
+					opts.size_, opts.block_size_,
-							opts.block_size_));
+					opts.nr_seq_blocks_);
+		else
+			gen = new forward_sequence_generator(opts.offset_,
+					opts.size_, opts.block_size_);
+
+		return sequence_generator::ptr(gen);
 	}
 
 	op_generator::ptr

base/io_generator.h

@@ -36,6 +36,7 @@ namespace base {
 		sector_t block_size_;
 		sector_t size_;
 		sector_t io_size_;
+		unsigned nr_seq_blocks_;
 	};
 
 	class io_generator {

thin-provisioning/commands.cc

@@ -22,6 +22,7 @@ thin_provisioning::register_thin_commands(base::application &app)
 	app.add_cmd(command::ptr(new thin_ll_dump_cmd()));
 	app.add_cmd(command::ptr(new thin_ll_restore_cmd()));
 	app.add_cmd(command::ptr(new thin_scan_cmd()));
+	app.add_cmd(command::ptr(new thin_generate_damage_cmd()));
 	app.add_cmd(command::ptr(new thin_generate_metadata_cmd()));
 	app.add_cmd(command::ptr(new thin_generate_mappings_cmd()));
 	app.add_cmd(command::ptr(new thin_show_duplicates_cmd()));

thin-provisioning/commands.h

@@ -103,6 +103,13 @@ namespace thin_provisioning {
 		virtual int run(int argc, char **argv);
 	};
 
+	class thin_generate_damage_cmd : public base::command {
+	public:
+		thin_generate_damage_cmd();
+		virtual void usage(std::ostream &out) const;
+		virtual int run(int argc, char **argv);
+	};
+
 	class thin_generate_metadata_cmd : public base::command {
 	public:
 		thin_generate_metadata_cmd();

thin-provisioning/damage_generator.cc

@@ -0,0 +1,66 @@
+#include "damage_generator.h"
+
+using namespace thin_provisioning;
+
+//----------------------------------------------------------------
+
+namespace {
+	void find_blocks(space_map::ptr sm,
+			 block_address nr_blocks,
+			 ref_t expected,
+			 std::set<block_address> &found) {
+		block_address sm_size = sm->get_nr_blocks();
+		base::run_set<block_address> visited;
+		block_address nr_visited = 0;
+
+		srand(time(NULL));
+		while (nr_blocks) {
+			if (nr_visited == sm_size)
+				break;
+
+			block_address b = rand() % sm_size;
+			if (visited.member(b))
+				continue;
+
+			ref_t c = sm->get_count(b);
+			if (c == expected) {
+				found.insert(b);
+				--nr_blocks;
+			}
+
+			visited.add(b);
+			++nr_visited;
+		}
+
+		if (nr_blocks) {
+			ostringstream out;
+			out << "cannot find " << (nr_blocks + found.size())
+			    << " blocks of ref-count " << expected;
+			throw runtime_error(out.str());
+		}
+	}
+}
+
+//----------------------------------------------------------------
+
+damage_generator::damage_generator(block_manager::ptr bm)
+{
+	md_ = metadata::ptr(new metadata(bm, true));
+}
+
+void damage_generator::commit()
+{
+	md_->commit();
+}
+
+void damage_generator::create_metadata_leaks(block_address nr_leaks,
+					     ref_t expected, ref_t actual)
+{
+	std::set<block_address> leaks;
+	find_blocks(md_->metadata_sm_, nr_leaks, expected, leaks);
+
+	for (auto const &b : leaks)
+		md_->metadata_sm_->set_count(b, actual);
+}
+
+//----------------------------------------------------------------

thin-provisioning/damage_generator.h

@@ -0,0 +1,22 @@
+#ifndef METADATA_DAMAGE_GENERATOR_H
+#define METADATA_DAMAGE_GENERATOR_H
+
+#include "metadata.h"
+
+//----------------------------------------------------------------
+
+class damage_generator {
+public:
+	typedef std::shared_ptr<damage_generator> ptr;
+
+	damage_generator(block_manager::ptr bm);
+	void commit();
+	void create_metadata_leaks(block_address nr_leaks, ref_t expected, ref_t actual);
+
+private:
+	thin_provisioning::metadata::ptr md_;
+};
+
+//----------------------------------------------------------------
+
+#endif

thin-provisioning/thin_generate_damage.cc

@@ -0,0 +1,144 @@
+#include "base/output_file_requirements.h"
+#include "persistent-data/file_utils.h"
+#include "thin-provisioning/commands.h"
+#include "thin-provisioning/damage_generator.h"
+#include "version.h"
+
+#include <getopt.h>
+#include <unistd.h>
+
+using namespace persistent_data;
+using namespace thin_provisioning;
+
+//----------------------------------------------------------------
+
+namespace {
+	struct flags {
+		enum damage_operations {
+			DAMAGE_OP_NONE,
+			DAMAGE_OP_CREATE_METADATA_LEAKS,
+			DAMAGE_OP_LAST
+		};
+
+		flags()
+			: op(DAMAGE_OP_NONE),
+			  nr_blocks(0),
+			  expected_rc(0),
+			  actual_rc(0) {
+		}
+
+		bool check_conformance();
+
+		damage_operations op;
+		string output;
+		block_address nr_blocks;
+		ref_t expected_rc;
+		ref_t actual_rc;
+	};
+
+	bool flags::check_conformance() {
+		if (op == DAMAGE_OP_NONE || op >= DAMAGE_OP_LAST) {
+			cerr << "Invalid operation." << endl;
+			return false;
+		}
+
+		if (!output.size()) {
+			cerr << "No output file provided." << endl;
+			return false;
+		}
+
+		if (!nr_blocks) {
+			cerr << "Invalid number of blocks" << endl;
+			return false;
+		}
+
+		if (op == DAMAGE_OP_CREATE_METADATA_LEAKS &&
+		    expected_rc == actual_rc) {
+			cerr << "Invalid reference count parameters" << endl;
+			return false;
+		}
+
+		check_output_file_requirements(output);
+
+		return true;
+	}
+
+	int generate_damage(flags const &fs) {
+		block_manager::ptr bm = open_bm(fs.output, block_manager::READ_WRITE);
+		damage_generator::ptr gen = damage_generator::ptr(new damage_generator(bm));
+
+		switch (fs.op) {
+		case flags::DAMAGE_OP_CREATE_METADATA_LEAKS:
+			gen->create_metadata_leaks(fs.nr_blocks, fs.expected_rc, fs.actual_rc);
+			break;
+		default:
+			break;
+		}
+
+		gen->commit();
+
+		return 0;
+	}
+}
+
+//----------------------------------------------------------------
+
+thin_generate_damage_cmd::thin_generate_damage_cmd()
+	: command("thin_generate_damage")
+{
+}
+
+void
+thin_generate_damage_cmd::usage(std::ostream &out) const
+{
+}
+
+int
+thin_generate_damage_cmd::run(int argc, char **argv)
+{
+	int c;
+	struct flags fs;
+	const char *shortopts = "ho:V";
+	const struct option longopts[] = {
+		{ "help", no_argument, NULL, 'h' },
+		{ "output", required_argument, NULL, 'o' },
+		{ "create-metadata-leaks", no_argument, NULL, 1 },
+		{ "nr-blocks", required_argument, NULL, 1001 },
+		{ "expected", required_argument, NULL, 1002 },
+		{ "actual", required_argument, NULL, 1003 },
+		{ "version", no_argument, NULL, 'V' },
+		{ NULL, no_argument, NULL, 0 }
+	};
+
+	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
+		switch(c) {
+		case 'h':
+			usage(cout);
+			break;
+		case 'o':
+			fs.output = optarg;
+			break;
+		case 1:
+			fs.op = flags::DAMAGE_OP_CREATE_METADATA_LEAKS;
+			break;
+		case 1001:
+			fs.nr_blocks = parse_uint64(optarg, "nr_blocks");
+			break;
+		case 1002:
+			fs.expected_rc = parse_uint64(optarg, "expected");
+			break;
+		case 1003:
+			fs.actual_rc = parse_uint64(optarg, "actual");
+			break;
+		}
+	}
+
+	if (!fs.check_conformance()) {
+		usage(cerr);
+		return 1;
+	}
+
+	return generate_damage(fs);
+}
+
+//----------------------------------------------------------------

thin-provisioning/thin_generate_mappings.cc

@@ -47,6 +47,7 @@ namespace {
 		base::sector_t offset;
 		boost::optional<base::sector_t> size;
 		boost::optional<base::sector_t> io_size;
+		boost::optional<unsigned> nr_seq_blocks;
 	};
 
 	bool flags::check_conformance() {
@@ -65,6 +66,14 @@ namespace {
 			return false;
 		}
 
+		if (nr_seq_blocks) {
+			if (!pattern.is_random()) {
+				cerr << "Cannot specify the sequence size"
+					" while doing non-random IO" << endl;
+				return false;
+			}
+		}
+
 		check_output_file_requirements(*output);
 
 		return true;
@@ -90,6 +99,7 @@ namespace {
 		opts.offset_ = fs.offset;
 		opts.size_ = *fs.size;
 		opts.io_size_ = !fs.io_size ? *fs.size : *fs.io_size;
+		opts.nr_seq_blocks_ = !fs.nr_seq_blocks ? 1 : *fs.nr_seq_blocks;
 		io_generator::ptr gen = create_io_generator(opts);
 
 		base::io io;
@@ -133,6 +143,7 @@ thin_generate_mappings_cmd::usage(std::ostream &out) const
 	    << "  {--io-size} <io-size in sectors>\n"
 	    << "  {--rw write|trim|randwrite|randtrim|randtw}\n"
 	    << "  {--size} <size in sectors>\n"
+	    << "  {--seq-nr} <max nr. of sequential ios>\n"
 	    << "  {-V|--version}" << endl;
 }
 
@@ -141,7 +152,7 @@ thin_generate_mappings_cmd::run(int argc, char **argv)
 {
 	int c;
 	struct flags fs;
-	const char *shortopts = "hi:o:qV";
+	const char *shortopts = "ho:V";
 	const struct option longopts[] = {
 		{ "help", no_argument, NULL, 'h' },
 		{ "output", required_argument, NULL, 'o' },
@@ -150,6 +161,7 @@ thin_generate_mappings_cmd::run(int argc, char **argv)
 		{ "offset", required_argument, NULL, 3 },
 		{ "size", required_argument, NULL, 4 },
 		{ "io-size", required_argument, NULL, 5 },
+		{ "seq-nr", required_argument, NULL, 6 },
 		{ "version", no_argument, NULL, 'V' },
 		{ NULL, no_argument, NULL, 0 }
 	};
@@ -184,6 +196,10 @@ thin_generate_mappings_cmd::run(int argc, char **argv)
 			fs.io_size = parse_uint64(optarg, "io_size");
 			break;
 
+		case 6:
+			fs.nr_seq_blocks = parse_uint64(optarg, "seq_nr");
+			break;
+
 		case 'V':
 			cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
 			return 0;

thin-provisioning/thin_generate_metadata.cc

@@ -43,6 +43,7 @@ namespace {
 			METADATA_OP_SET_TRANSACTION_ID,
 			METADATA_OP_RESERVE_METADATA_SNAP,
 			METADATA_OP_RELEASE_METADATA_SNAP,
+			METADATA_OP_SET_NEEDS_CHECK,
 			METADATA_OP_LAST
 		};
 
@@ -133,6 +134,9 @@ namespace {
 		case flags::METADATA_OP_RELEASE_METADATA_SNAP:
 			pool->release_metadata_snap();
 			break;
+		case flags::METADATA_OP_SET_NEEDS_CHECK:
+			pool->set_needs_check();
+			break;
 		default:
 			break;
 		}
@@ -163,6 +167,7 @@ thin_generate_metadata_cmd::usage(std::ostream &out) const
 	    << "  {--reserve-metadata-snap}\n"
 	    << "  {--release-metadata-snap}\n"
 	    << "  {--set-transaction-id} <tid>\n"
+	    << "  {--set-needs-check}\n"
 	    << "  {--data-block-size} <block size>\n"
 	    << "  {--nr-data-blocks} <nr>\n"
 	    << "  {--origin} <origin-id>\n"
@@ -175,7 +180,7 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 {
 	int c;
 	struct flags fs;
-	const char *shortopts = "hi:o:qV";
+	const char *shortopts = "ho:V";
 	const struct option longopts[] = {
 		{ "help", no_argument, NULL, 'h' },
 		{ "output", required_argument, NULL, 'o' },
@@ -187,9 +192,10 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 		{ "set-transaction-id", required_argument, NULL, 6 },
 		{ "reserve-metadata-snap", no_argument, NULL, 7 },
 		{ "release-metadata-snap", no_argument, NULL, 8 },
-		{ "data-block-size", required_argument, NULL, 101 },
+		{ "set-needs-check", no_argument, NULL, 9 },
-		{ "nr-data-blocks", required_argument, NULL, 102 },
+		{ "data-block-size", required_argument, NULL, 1001 },
-		{ "origin", required_argument, NULL, 401 },
+		{ "nr-data-blocks", required_argument, NULL, 1002 },
+		{ "origin", required_argument, NULL, 4001 },
 		{ "version", no_argument, NULL, 'V' },
 		{ NULL, no_argument, NULL, 0 }
 	};
@@ -240,15 +246,19 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 			fs.op = flags::METADATA_OP_RELEASE_METADATA_SNAP;
 			break;
 
-		case 101:
+		case 9:
+			fs.op = flags::METADATA_OP_SET_NEEDS_CHECK;
+			break;
+
+		case 1001:
 			fs.data_block_size = parse_uint64(optarg, "data block size");
 			break;
 
-		case 102:
+		case 1002:
 			fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
 			break;
 
-		case 401:
+		case 4001:
 			fs.origin = parse_uint64(optarg, "origin");
 			break;
 

thin-provisioning/thin_pool.cc

@@ -408,6 +408,12 @@ thin_pool::write_changed_details()
 	}
 }
 
+void
+thin_pool::set_needs_check()
+{
+	md_->sb_.set_needs_check_flag(true);
+}
+
 //----------------------------------------------------------------
 
 void

thin-provisioning/thin_pool.h

@@ -99,6 +99,9 @@ namespace thin_provisioning {
 		thin::ptr open_thin(thin_dev_t);
 		void close_thin(thin::ptr td);
 
+		// updates the superblock
+		void set_needs_check();
+
 	private:
 		friend class thin;
 		typedef std::map<thin_dev_t, thin::ptr> device_map;