Merge branch 'master' of github.com:jthornber/thin-provisioning-tools

Doubles performance

Makefile.in

@@ -87,6 +87,7 @@ SOURCE=\
 	persistent-data/file_utils.cc \
 	persistent-data/hex_dump.cc \
 	persistent-data/space-maps/careful_alloc.cc \
+	persistent-data/space-maps/core.cc \
 	persistent-data/space-maps/disk.cc \
 	persistent-data/space-maps/recursive.cc \
 	persistent-data/space_map.cc \

caching/metadata.cc

@@ -15,7 +15,7 @@ namespace {
 		if (!nr_blocks)
 			throw runtime_error("Metadata is not large enough for superblock.");
 
-		space_map::ptr sm(new core_map(nr_blocks));
+		space_map::ptr sm{create_core_map(nr_blocks)};
 		sm->inc(SUPERBLOCK_LOCATION);
 		transaction_manager::ptr tm(new transaction_manager(bm, sm));
 		return tm;

era/metadata.cc

@@ -13,7 +13,7 @@ namespace {
 		if (!nr_blocks)
 			throw runtime_error("Metadata is not large enough for superblock.");
 
-		space_map::ptr sm(new core_map(nr_blocks));
+		space_map::ptr sm{create_core_map(nr_blocks)};
 		sm->inc(SUPERBLOCK_LOCATION);
 		transaction_manager::ptr tm(new transaction_manager(bm, sm));
 		return tm;

persistent-data/file_utils.cc

@@ -59,7 +59,7 @@ persistent_data::open_tm(block_manager::ptr bm, block_address superblock_locatio
 	if (!nr_blocks)
 		throw runtime_error("Metadata is not large enough for superblock.");
 
-	space_map::ptr sm(new core_map(nr_blocks));
+	space_map::ptr sm{create_core_map(nr_blocks)};
 	sm->inc(superblock_location);
 	transaction_manager::ptr tm(new transaction_manager(bm, sm));
 	return tm;

persistent-data/space-maps/careful_alloc.cc

@@ -77,10 +77,6 @@ namespace {
 			return sm_->find_free(filtered_it);
 		}
 
-		virtual bool count_possibly_greater_than_one(block_address b) const {
-			return sm_->count_possibly_greater_than_one(b);
-		}
-
 		virtual void extend(block_address extra_blocks) {
 			return sm_->extend(extra_blocks);
 		}

persistent-data/space-maps/core.cc

@@ -0,0 +1,242 @@
+// Copyright (C) 2020 Red Hat, Inc. All rights reserved.
+//
+// This file is part of the thin-provisioning-tools source.
+//
+// thin-provisioning-tools is free software: you can redistribute it
+// and/or modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// thin-provisioning-tools is distributed in the hope that it will be
+// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with thin-provisioning-tools.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#include "persistent-data/space-maps/core.h"
+#include "persistent-data/math_utils.h"
+
+#include <stdexcept>
+
+using namespace persistent_data;
+using namespace std;
+
+//----------------------------------------------------------------
+
+namespace {
+	constexpr block_address ENTRIES_PER_WORD = 4 * sizeof(uint64_t);
+
+	class core_map : public checked_space_map {
+	public:
+		typedef std::shared_ptr<core_map> ptr;
+
+		core_map(block_address nr_blocks);
+
+		block_address get_nr_blocks() const override;
+		block_address get_nr_free() const override;
+		ref_t get_count(block_address b) const override;
+		void set_count(block_address b, ref_t c) override;
+		void commit() override;
+		void inc(block_address b, ref_t count) override;
+		void dec(block_address b, ref_t count) override;
+		maybe_block find_free(span_iterator &it) override;
+		void extend(block_address extra_blocks) override;
+		void count_metadata(block_counter &bc) const override;
+
+		// FIXME: meaningless, but this class is only used for testing
+		size_t root_size() const override;
+		
+		// FIXME: meaningless, but this class is only used for testing
+		virtual void copy_root(void *dest, size_t len) const override;
+
+		checked_space_map::ptr clone() const override;
+	private:
+		ref_t get_count_(block_address b) const;
+		void set_count_(block_address b, ref_t c);
+		void check_block_(block_address b) const;
+		ref_t lookup_bits_(block_address b) const;
+		void set_bits_(block_address b, ref_t c);
+		ref_t lookup_exception_(block_address b) const;
+
+		block_address nr_blocks_;
+		block_address nr_free_;
+		block_address search_start_;
+
+		std::vector<uint64_t> bits_;
+		std::map<block_address, ref_t> exceptions_;
+
+	};
+
+	core_map::core_map(block_address nr_blocks)
+		: nr_blocks_(nr_blocks),
+		  nr_free_(nr_blocks),
+		  search_start_(0),
+		  bits_(base::div_up(nr_blocks, ENTRIES_PER_WORD)) {
+	}
+
+	block_address
+	core_map::get_nr_blocks() const {
+		return nr_blocks_;
+	}
+
+	block_address
+	core_map::get_nr_free() const {
+		return nr_free_;
+	}
+
+	ref_t
+	core_map::get_count(block_address b) const {
+		check_block_(b);
+		return get_count_(b);
+	}
+
+	void
+	core_map::set_count(block_address b, ref_t c) {
+		check_block_(b);
+		set_count_(b, c);
+	}
+
+	void
+	core_map::commit() {
+	}
+
+	void
+	core_map::inc(block_address b, ref_t count) {
+		check_block_(b);
+		auto old_c = get_count_(b);
+		set_count_(b, old_c + count);
+	}
+
+	void
+	core_map::dec(block_address b, ref_t count) {
+		check_block_(b);
+		auto old_c = get_count_(b);
+		set_count_(b, old_c - count);
+	}
+
+	core_map::maybe_block
+	core_map::find_free(span_iterator &it) {
+		for (maybe_span ms = it.first(); ms; ms = it.next()) {
+			for (block_address b = std::max(search_start_, ms->first); b < ms->second; b++) {
+				check_block_(b);
+				if (!get_count_(b))
+					return maybe_block(b);
+			}
+		}
+
+		return maybe_block();
+	}
+
+	void
+	core_map::extend(block_address extra_blocks) {
+		throw std::runtime_error("'extend' not implemented");
+	}
+
+	void
+	core_map::count_metadata(block_counter &bc) const {
+	}
+
+	// FIXME: meaningless, but this class is only used for testing
+	size_t
+	core_map::root_size() const {
+		return 0;
+	}
+
+	// FIXME: meaningless, but this class is only used for testing
+	void
+	core_map::copy_root(void *dest, size_t len) const {
+		throw std::runtime_error("'copy root' not implemented");
+	}
+
+	checked_space_map::ptr
+	core_map::clone() const {
+		return ptr(new core_map(*this));
+	}
+
+	void
+	core_map::check_block_(block_address b) const {
+		if (b >= nr_blocks_)
+			throw std::runtime_error("block out of bounds");
+	}
+
+	ref_t
+	core_map::get_count_(block_address b) const {
+		auto c = lookup_bits_(b);
+		if (c == 3)
+			c = lookup_exception_(b);
+
+		return c;
+	}
+
+	void
+	core_map::set_count_(block_address b, ref_t c) {
+		auto old_c = get_count_(b);
+		
+		if (old_c > 2) {
+			if (c > 2)
+				exceptions_[b] = c;
+			else {
+				exceptions_.erase(b);
+				set_bits_(b, c);
+			}
+		} else {
+			if (c > 2) {
+				set_bits_(b, 3);
+				exceptions_.insert(make_pair(b, c));
+			} else
+				set_bits_(b, c);
+		}
+		
+		if (old_c == 0 && c > 0)
+			nr_free_--;
+
+		else if (old_c > 0 && c == 0) {
+			if (b < search_start_)
+				search_start_ = b;
+
+			nr_free_++;
+		}
+	}
+
+	ref_t
+	core_map::lookup_bits_(block_address b) const {
+		block_address word = b / ENTRIES_PER_WORD;
+		block_address shift = (b % ENTRIES_PER_WORD) * 2;
+
+		return (bits_[word] >> shift) & 0b11;
+	}
+
+	void
+	core_map::set_bits_(block_address b, ref_t c) {
+		if (c > 3)
+			throw runtime_error("bad count");
+
+		block_address word = b / ENTRIES_PER_WORD;
+		block_address shift = (b % ENTRIES_PER_WORD) * 2ull;
+
+		auto w = bits_[word] & ~(0b11ull << shift);
+		bits_[word] = w | (((uint64_t) c) << shift);
+	}
+	
+	ref_t
+	core_map::lookup_exception_(block_address b) const {
+		auto it = exceptions_.find(b);
+		if (it == exceptions_.end())
+			throw runtime_error("core space map exception entry missing");
+
+		return it->second;
+	}
+}
+
+//----------------------------------------------------------------
+
+checked_space_map::ptr
+persistent_data::create_core_map(block_address nr_blocks)
+{
+	return checked_space_map::ptr(new core_map(nr_blocks));
+}
+
+//----------------------------------------------------------------

persistent-data/space-maps/core.h

@@ -21,112 +21,12 @@
 
 #include "persistent-data/space_map.h"
 
+#include <map>
+
 //----------------------------------------------------------------
 
 namespace persistent_data {
-	class core_map : public checked_space_map {
-	public:
-		typedef std::shared_ptr<core_map> ptr;
-
-		core_map(block_address nr_blocks)
-			: counts_(nr_blocks, 0),
-			  nr_free_(nr_blocks),
-			  search_start_(0ull) {
-		}
-
-		block_address get_nr_blocks() const {
-			return counts_.size();
-		}
-
-		block_address get_nr_free() const {
-			return nr_free_;
-		}
-
-		ref_t get_count(block_address b) const {
-			return counts_[b];
-		}
-
-		void set_count(block_address b, ref_t c) {
-			if (counts_[b] == 0 && c > 0)
-				nr_free_--;
-
-			else if (counts_[b] > 0 && c == 0) {
-				if (b < search_start_)
-					search_start_ = b;
-
-				nr_free_++;
-			}
-
-			counts_[b] = c;
-		}
-
-		void commit() {
-		}
-
-		void inc(block_address b, ref_t count) override {
-			if (counts_[b] == 0)
-				nr_free_--;
-
-			counts_[b] += count;
-		}
-
-		void dec(block_address b, ref_t count) override {
-			counts_[b] -= count;
-
-			if (counts_[b] == 0) {
-				if (b < search_start_)
-					search_start_ = b;
-				nr_free_++;
-			}
-		}
-
-		maybe_block find_free(span_iterator &it) {
-			for (maybe_span ms = it.first(); ms; ms = it.next()) {
-				if (search_start_ >= ms->second)
-					continue;
-
-				for (block_address b = std::max(search_start_, ms->first); b < ms->second; b++) {
-					if (b >= counts_.size())
-						throw std::runtime_error("block out of bounds");
-
-					if (!counts_[b])
-						return maybe_block(b);
-				}
-			}
-
-			return maybe_block();
-		}
-
-		bool count_possibly_greater_than_one(block_address b) const {
-			return counts_[b] > 1;
-		}
-
-		void extend(block_address extra_blocks) {
-			throw std::runtime_error("'extend' not implemented");
-		}
-
-		void count_metadata(block_counter &bc) const {
-		}
-
-		// FIXME: meaningless, but this class is only used for testing
-		size_t root_size() const {
-			return 0;
-		}
-
-		// FIXME: meaningless, but this class is only used for testing
-		virtual void copy_root(void *dest, size_t len) const {
-			throw std::runtime_error("'copy root' not implemented");
-		}
-
-		checked_space_map::ptr clone() const {
-			return ptr(new core_map(*this));
-		}
-
-	private:
-		std::vector<ref_t> counts_;
-		unsigned nr_free_;
-		block_address search_start_;
-	};
+	checked_space_map::ptr create_core_map(block_address nr_blocks);
 }
 
 //----------------------------------------------------------------

persistent-data/space-maps/disk.cc

@@ -429,10 +429,6 @@ namespace {
 			return maybe_block();
 		}
 
-		bool count_possibly_greater_than_one(block_address b) const {
-			return get_count(b) > 1;
-		}
-
 		virtual void extend(block_address extra_blocks) {
 			block_address nr_blocks = nr_blocks_ + extra_blocks;
 

persistent-data/space-maps/noop.h

@@ -64,11 +64,6 @@ namespace persistent_data {
 			return boost::optional<block_address>(0);
 		}
 
-		bool count_possibly_greater_than_one(block_address b) const {
-			fail();
-			return false;
-		}
-
 		void extend(block_address extra_blocks) {
 			fail();
 		}

persistent-data/space-maps/recursive.cc

@@ -97,6 +97,12 @@ namespace {
 				add_op(b, block_op(SET, c));
 			else {
 				recursing_lock lock(*this);
+
+				// the inner set_count may trigger a find_free,
+				// so it's important we update the allocated
+				// blocks list before calling.
+				allocated_blocks_.add(b, b + 1);
+
 				return sm_->set_count(b, c);
 			}
 		}
@@ -111,6 +117,12 @@ namespace {
 				add_op(b, block_op(INC, count));
 			else {
 				recursing_lock lock(*this);
+
+				// the inner inc() may trigger a find_free,
+				// so it's important we update the allocated
+				// blocks list before calling.
+				allocated_blocks_.add(b, b + 1);
+
 				return sm_->inc(b, count);
 			}
 		}
@@ -132,16 +144,6 @@ namespace {
 			return sm_->find_free(filtered_it);
 		}
 
-		virtual bool count_possibly_greater_than_one(block_address b) const {
-			recursing_const_lock lock(*this);
-
-			bool gto = sm_->count_possibly_greater_than_one(b);
-			if (gto)
-				return true;
-
-			return modify_count(b, 1) > 1;
-		}
-
 		virtual void extend(block_address extra_blocks) {
 			cant_recurse("extend");
 			recursing_lock lock(*this);
@@ -210,9 +212,11 @@ namespace {
 		void flush_ops_() {
 			recursing_lock lock(*this);
 
-			for (auto const &p : ops_) {
-				block_address b = p.first;
-				auto const &op = p.second;
+			while (!ops_.empty()) {
+				auto p = ops_.begin();
+				block_address b = p->first;
+				auto op = p->second;
+				ops_.erase(p);
 
 				switch (op.op_) {
 				case INC:
@@ -228,7 +232,6 @@ namespace {
 				}
 			}
 
-			ops_.clear();
 			allocated_blocks_.clear();
 		}
 

persistent-data/space_map.h

@@ -88,8 +88,6 @@ namespace persistent_data {
 
 		virtual maybe_block find_free(span_iterator &it) = 0;
 
-		virtual bool count_possibly_greater_than_one(block_address b) const = 0;
-
 		virtual void extend(block_address extra_blocks) = 0;
 
 		struct iterator {

persistent-data/transaction_manager.cc

@@ -62,7 +62,7 @@ transaction_manager::new_block(validator v)
 pair<transaction_manager::write_ref, bool>
 transaction_manager::shadow(block_address orig, validator v)
 {
-	bool need_inc = sm_->count_possibly_greater_than_one(orig);
+	bool need_inc = sm_->get_count(orig) > 1;
 	if (is_shadow(orig) && !need_inc)
 		return make_pair(bm_->write_lock(orig, v), need_inc);
 

thin-provisioning/thin_generate_metadata.cc

@@ -19,22 +19,11 @@
 #include "base/output_file_requirements.h"
 #include "persistent-data/file_utils.h"
 #include "thin-provisioning/commands.h"
-#include "thin-provisioning/emitter.h"
-#include "thin-provisioning/human_readable_format.h"
 #include "thin-provisioning/metadata.h"
-#include "thin-provisioning/restore_emitter.h"
-#include "thin-provisioning/xml_format.h"
 #include "version.h"
 
 #include <boost/optional.hpp>
-#include <fstream>
 #include <getopt.h>
-#include <iostream>
-#include <libgen.h>
-#include <linux/fs.h>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
 #include <unistd.h>
 
 using namespace boost;
@@ -46,76 +35,128 @@ using namespace thin_provisioning;
 
 namespace {
 	struct flags {
+		enum metadata_operations {
+			METADATA_OP_NONE,
+			METADATA_OP_FORMAT,
+			METADATA_OP_OPEN,
+			METADATA_OP_CREATE_THIN,
+			METADATA_OP_LAST
+		};
+
 		flags()
-			: data_block_size(128),
-			  nr_data_blocks(10240),
-			  nr_thins(1),
-			  blocks_per_thin(1024),
-			  run_lengths(1024) {
+			: op(METADATA_OP_NONE),
+			  data_block_size(128),
+			  nr_data_blocks(10240)
+		{
 		}
 
-		block_address data_block_size;
+		bool check_conformance();
+
+		metadata_operations op;
+		sector_t data_block_size;
 		block_address nr_data_blocks;
-		unsigned nr_thins;
-		block_address blocks_per_thin;
-		block_address run_lengths;
+		optional<uint64_t> dev_id;
 		optional<string> output;
 	};
 
-	// This is how we stir in some entropy.  It mixes up the data
-	// device.
-	class shuffler {
-	public:
-		shuffler(block_address nr_blocks, unsigned run_lengths)
-			: nr_blocks_(nr_blocks / run_lengths),
-			  run_lengths_(run_lengths) {
+	// FIXME: modulize the conditions
+	bool flags::check_conformance() {
+		if (op == METADATA_OP_NONE || op >= METADATA_OP_LAST) {
+			cerr << "Invalid operation." << endl;
+			return false;
 		}
 
-		block_address map(block_address b) const {
-			return reverse(b / run_lengths_) + (b % run_lengths_);
+		if (!output) {
+			cerr << "No output file provided." << endl;
+			return false;
+		} else
+			check_output_file_requirements(*output);
+
+		if (op == METADATA_OP_CREATE_THIN && !dev_id) {
+			cerr << "no device id provided." << endl;
+			return false;
 		}
 
-	private:
-		block_address reverse(block_address b) const {
-			return nr_blocks_ - b - 1ull;
-		}
-
-		block_address nr_blocks_;
-		block_address run_lengths_;
-	};
-
-	void generate_device(emitter::ptr e, shuffler const &s, uint32_t dev_id,
-			     block_address nr_blocks, block_address base) {
-
-		e->begin_device(dev_id, nr_blocks, 0, 0, 0);
-		for (unsigned b = 0; b < nr_blocks; b++)
-			e->single_map(b, s.map(base + b), 0);
-		e->end_device();
+		return true;
 	}
 
-	void generate_metadata(flags const &fs, emitter::ptr e) {
-		e->begin_superblock("fake metadata", 0, 0, optional<uint32_t>(), optional<uint32_t>(),
-				    fs.data_block_size, fs.nr_data_blocks, optional<uint64_t>());
+	//--------------------------------
 
-		shuffler s(fs.nr_data_blocks, fs.run_lengths);
-		for (unsigned i = 0; i < fs.nr_thins; i++)
-			generate_device(e, s, i, fs.blocks_per_thin, i * fs.blocks_per_thin);
-
-		e->end_superblock();
+	single_mapping_tree::ptr new_mapping_tree(metadata::ptr md) {
+		return single_mapping_tree::ptr(
+			new single_mapping_tree(*md->tm_,
+						mapping_tree_detail::block_time_ref_counter(md->data_sm_)));
 	}
 
-	int create_metadata(flags const &fs) {
-		try {
-			// The block size gets updated by the restorer.
-			block_manager::ptr bm(open_bm(*fs.output, block_manager::READ_WRITE));
-			metadata::ptr md(new metadata(bm, metadata::CREATE, 128, 0));
-			emitter::ptr restorer = create_restore_emitter(md);
+	bool is_device_exists(metadata::ptr md, uint64_t dev_id) {
+		uint64_t key[1] = {dev_id};
 
-			generate_metadata(fs, restorer);
+		device_tree::maybe_value v1 = md->details_->lookup(key);
+		if (v1)
+			return true;
 
-		} catch (std::exception &e) {
-			cerr << e.what() << endl;
-			return 1;
+		dev_tree::maybe_value v2 = md->mappings_top_level_->lookup(key);
+		if (v2)
+			return true;
+
+		return false;
+	}
+
+	//--------------------------------
+
+	metadata::ptr format_metadata(block_manager::ptr bm,
+				      sector_t data_block_size,
+				      block_address nr_data_blocks) {
+		metadata::ptr md(new metadata(bm,
+					      metadata::CREATE,
+					      data_block_size,
+					      nr_data_blocks));
+		md->commit();
+		return md;
+	}
+
+	metadata::ptr open_metadata(block_manager::ptr bm) {
+		metadata::ptr md(new metadata(bm, true));
+		return md;
+	}
+
+	void create_thin(metadata::ptr md, uint64_t dev_id) {
+		uint64_t key[1] = {dev_id};
+
+		if (is_device_exists(md, dev_id))
+			throw runtime_error("device already exists");
+
+		device_tree_detail::device_details details;
+		details.transaction_id_ = md->sb_.trans_id_;
+		details.creation_time_ = md->sb_.time_;
+		details.snapshotted_time_ = details.creation_time_;
+		md->details_->insert(key, details);
+
+		single_mapping_tree::ptr subtree = new_mapping_tree(md);
+		md->mappings_top_level_->insert(key, subtree->get_root());
+		md->mappings_->set_root(md->mappings_top_level_->get_root()); // FIXME: ugly
+
+		md->commit();
+	}
+
+	metadata::ptr open_or_format_metadata(block_manager::ptr bm, flags const &fs) {
+
+		if (fs.op == flags::METADATA_OP_FORMAT)
+			return format_metadata(bm, fs.data_block_size, fs.nr_data_blocks);
+		else
+			return open_metadata(bm);
+	}
+
+	int generate_metadata(flags const &fs) {
+		block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE);
+		metadata::ptr md = open_or_format_metadata(bm, fs);
+
+		switch (fs.op) {
+		case flags::METADATA_OP_CREATE_THIN:
+			create_thin(md, *fs.dev_id);
+			break;
+		default:
+			break;
 		}
 
 		return 0;
@@ -137,9 +178,6 @@ thin_generate_metadata_cmd::usage(std::ostream &out) const
 	    << "  {-h|--help}\n"
 	    << "  --data-block-size <block size>\n"
 	    << "  --nr-data-blocks <nr>\n"
-	    << "  --nr-thins <count>\n"
-	    << "  --blocks-per-thin <count>\n"
-	    << "  --run-lengths <count>\n"
 	    << "  {-o|--output} <output device or file>\n"
 	    << "  {-V|--version}" << endl;
 }
@@ -151,14 +189,15 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 	struct flags fs;
 	const char *shortopts = "hi:o:qV";
 	const struct option longopts[] = {
-		{ "help", no_argument, NULL, 'h'},
-		{ "output", required_argument, NULL, 'o'},
-		{ "data-block-size", required_argument, NULL, 1},
-		{ "nr-data-blocks", required_argument, NULL, 2},
-		{ "nr-thins", required_argument, NULL, 3},
-		{ "blocks-per-thin", required_argument, NULL, 4},
-		{ "run-lengths", required_argument, NULL, 5},
-		{ "version", no_argument, NULL, 'V'},
+		{ "help", no_argument, NULL, 'h' },
+		{ "output", required_argument, NULL, 'o' },
+		{ "format", no_argument, NULL, 1 },
+		{ "open", no_argument, NULL, 2 },
+		{ "create-thin", no_argument, NULL, 3 },
+		{ "data-block-size", required_argument, NULL, 101 },
+		{ "nr-data-blocks", required_argument, NULL, 102 },
+		{ "dev-id", required_argument, NULL, 301 },
+		{ "version", no_argument, NULL, 'V' },
 		{ NULL, no_argument, NULL, 0 }
 	};
 
@@ -173,23 +212,27 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 			break;
 
 		case 1:
-			fs.data_block_size = parse_uint64(optarg, "data block size");
+			fs.op = flags::METADATA_OP_FORMAT;
 			break;
 
 		case 2:
-			fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
+			fs.op = flags::METADATA_OP_OPEN;
 			break;
 
 		case 3:
-			fs.nr_thins = parse_uint64(optarg, "nr thins");
+			fs.op = flags::METADATA_OP_CREATE_THIN;
 			break;
 
-		case 4:
-			fs.blocks_per_thin = parse_uint64(optarg, "blocks per thin");
+		case 101:
+			fs.data_block_size = parse_uint64(optarg, "data block size");
 			break;
 
-		case 5:
-			fs.run_lengths = parse_uint64(optarg, "run lengths");
+		case 102:
+			fs.nr_data_blocks = parse_uint64(optarg, "nr data blocks");
+			break;
+
+		case 301:
+			fs.dev_id = parse_uint64(optarg, "dev id");
 			break;
 
 		case 'V':
@@ -202,14 +245,12 @@ thin_generate_metadata_cmd::run(int argc, char **argv)
 		}
 	}
 
-	if (!fs.output) {
-		cerr << "No output file provided.\n\n";
+	if (!fs.check_conformance()) {
 		usage(cerr);
 		return 1;
-	} else
-		check_output_file_requirements(*fs.output);
+	}
 
-	return create_metadata(fs);
+	return generate_metadata(fs);
 }
 
 //----------------------------------------------------------------

thin-provisioning/thin_metadata_pack.cc

@@ -41,6 +41,8 @@ using boost::optional;
 namespace {
 	using namespace std;
 	constexpr uint64_t MAGIC = 0xa537a0aa6309ef77;
+	constexpr uint64_t PACK_VERSION = 1;
+	constexpr uint64_t PREFETCH_COUNT = 1024;
 
 	uint32_t const SUPERBLOCK_CSUM_SEED = 160774;
 	uint32_t const BITMAP_CSUM_XOR = 240779;
@@ -114,6 +116,7 @@ namespace {
 
 		std::ofstream out_file(*f.output_file_, ios_base::binary);
 		write_u64(out_file, MAGIC);
+		write_u64(out_file, PACK_VERSION);
 
 		boost::iostreams::filtering_ostreambuf out_buf;
 		out_buf.push(zlib_compressor());
@@ -130,6 +133,10 @@ namespace {
 		write_u64(out, block_size);
 		write_u64(out, nr_blocks);
 
+		// prefetch
+		for (block_address b = 0; b < PREFETCH_COUNT; b++)
+			bm->prefetch(b);
+
 		is_metadata_functor is_metadata;
 		for (block_address b = 0; b < nr_blocks; b++) {
 			auto rr = bm->read_lock(b);
@@ -138,6 +145,10 @@ namespace {
 				write_u64(out, b);
 				out.write(reinterpret_cast<const char *>(rr.data()), block_size);
 			}
+
+			auto prefetch_b = b + PREFETCH_COUNT;
+			if (prefetch_b < nr_blocks)
+				bm->prefetch(prefetch_b);
 		} 
 
 		return 0;
@@ -154,6 +165,9 @@ namespace {
 		if (read_u64(in_file) != MAGIC)
 			throw runtime_error("Not a pack file.");
 
+		if (read_u64(in_file) != PACK_VERSION)
+			throw runtime_error("unknown pack file format version");
+
 		filtering_istreambuf in_buf;
 		in_buf.push(zlib_decompressor());
 		in_buf.push(in_file);

unit-tests/array_block_t.cc

@@ -87,7 +87,7 @@ namespace {
 	transaction_manager::ptr
 	create_tm() {
 		block_manager::ptr bm = create_bm(NR_BLOCKS);
-		space_map::ptr sm(new core_map(NR_BLOCKS));
+		space_map::ptr sm{create_core_map(NR_BLOCKS)};
 		transaction_manager::ptr tm(new transaction_manager(bm, sm));
 		return tm;
 	}

unit-tests/array_t.cc

@@ -38,7 +38,7 @@ namespace {
 	public:
 		ArrayTests()
 			: bm_(new block_manager("./test.data", NR_BLOCKS, 4, block_manager::READ_WRITE)),
-			  sm_(new core_map(NR_BLOCKS)),
+			  sm_(create_core_map(NR_BLOCKS)),
 			  tm_(bm_, sm_) {
 		}
 

unit-tests/bitset_t.cc

@@ -56,7 +56,7 @@ namespace {
 	public:
 		BitsetTests()
 			: bm_(new block_manager("./test.data", NR_BLOCKS, 4, block_manager::READ_WRITE)),
-			  sm_(new core_map(NR_BLOCKS)),
+			  sm_(create_core_map(NR_BLOCKS)),
 			  tm_(bm_, sm_) {
 		}
 

unit-tests/btree_counter_t.cc

@@ -45,7 +45,7 @@ namespace {
 
 	private:
 		space_map::ptr setup_core_map() {
-			space_map::ptr sm(new core_map(NR_BLOCKS));
+			space_map::ptr sm(create_core_map(NR_BLOCKS));
 			sm->inc(SUPERBLOCK);
 			return sm;
 		}

unit-tests/btree_damage_visitor_t.cc

@@ -302,7 +302,7 @@ namespace {
 
 	private:
 		space_map::ptr setup_core_map() {
-			space_map::ptr sm(new core_map(NR_BLOCKS));
+			space_map::ptr sm(create_core_map(NR_BLOCKS));
 			sm->inc(SUPERBLOCK);
 			return sm;
 		}

unit-tests/btree_t.cc

@@ -35,7 +35,7 @@ namespace {
 	public:
 		BtreeTests() 
 			: bm_(new block_manager("./test.data", NR_BLOCKS, 4, block_manager::READ_WRITE)),
-			  sm_(new core_map(NR_BLOCKS)),
+			  sm_(create_core_map(NR_BLOCKS)),
 			  tm_(bm_, sm_) {
 		}
 

unit-tests/space_map_t.cc

@@ -37,14 +37,14 @@ namespace {
 	public:
 		SpaceMapTests()
 			: bm_(new block_manager("./test.data", NR_BLOCKS, MAX_LOCKS, block_manager::READ_WRITE)),
-			  sm_(new core_map(NR_BLOCKS)),
+			  sm_(create_core_map(NR_BLOCKS)),
 			  tm_(bm_, sm_) {
 		}
 
 		struct sm_core_creator {
 			static space_map::ptr
 			create(transaction_manager &tm) {
-				return space_map::ptr(new persistent_data::core_map(NR_BLOCKS));
+				return create_core_map(NR_BLOCKS);
 			}
 		};
 
@@ -52,8 +52,7 @@ namespace {
 			static space_map::ptr
 			create(transaction_manager &tm) {
 				return create_careful_alloc_sm(
-					checked_space_map::ptr(
-						new core_map(NR_BLOCKS)));
+					create_core_map(NR_BLOCKS));
 			}
 		};
 
@@ -61,8 +60,7 @@ namespace {
 			static checked_space_map::ptr
 			create(transaction_manager &tm) {
 				return create_recursive_sm(
-					checked_space_map::ptr(
-						new core_map(NR_BLOCKS)));
+					create_core_map(NR_BLOCKS));
 			}
 		};
 
@@ -279,7 +277,7 @@ TEST_F(SpaceMapTests, test_metadata_and_disk)
 {
 	block_manager::ptr bm(
 		new block_manager("./test.data", NR_BLOCKS, MAX_LOCKS, block_manager::READ_WRITE));
-	space_map::ptr core_sm(new core_map(NR_BLOCKS));
+	space_map::ptr core_sm{create_core_map(NR_BLOCKS)};
 	transaction_manager::ptr tm(new transaction_manager(bm, core_sm));
 	persistent_space_map::ptr metadata_sm = persistent_data::create_metadata_sm(*tm, NR_BLOCKS);
 	copy_space_maps(metadata_sm, core_sm);

unit-tests/test_utils.cc

@@ -21,7 +21,7 @@ void test::zero_block(block_manager::ptr bm, block_address b)
 transaction_manager::ptr
 test::open_temporary_tm(block_manager::ptr bm)
 {
-	space_map::ptr sm(new core_map(bm->get_nr_blocks()));
+	space_map::ptr sm{create_core_map(bm->get_nr_blocks())};
 	transaction_manager::ptr tm(new transaction_manager(bm, sm));
 	return tm;
 }

unit-tests/transaction_manager_t.cc

@@ -34,7 +34,7 @@ namespace {
 	create_tm() {
 		block_manager::ptr bm(
 			new block_manager("./test.data", NR_BLOCKS, MAX_HELD_LOCKS, block_manager::READ_WRITE));
-		space_map::ptr sm(new core_map(NR_BLOCKS));
+		space_map::ptr sm{create_core_map(NR_BLOCKS)};
 		transaction_manager::ptr tm(new transaction_manager(bm, sm));
 		tm->get_sm()->inc(0);
 		return tm;