[era] era_superblock_t and era_check

.gitignore

@@ -21,6 +21,8 @@ cache_dump
 cache_restore
 cache_repair
 
+era_check
+
 *.metadata
 bad-metadata
 Makefile

Makefile.in

@@ -25,6 +25,8 @@ PROGRAMS=\
 	cache_restore \
 	cache_repair \
 	\
+	era_check \
+	\
 	thin_check \
 	thin_dump \
 	thin_restore \
@@ -289,6 +291,38 @@ cache_restore: $(CACHE_RESTORE_OBJECTS) caching/cache_restore.o
 	@echo "    [LD]  $@"
 	$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
 
+#----------------------------------------------------------------
+# Era tools
+
+ERA_CHECK_SOURCE=\
+	base/base64.cc \
+	base/error_state.cc \
+	base/endian_utils.cc \
+	\
+	era/superblock.cc \
+	era/era_detail.cc \
+	\
+	persistent-data/checksum.cc \
+	persistent-data/error_set.cc \
+	persistent-data/file_utils.cc \
+	persistent-data/hex_dump.cc \
+	persistent-data/lock_tracker.cc \
+	persistent-data/data-structures/btree.cc \
+	persistent-data/data-structures/bitset.cc \
+	persistent-data/space_map.cc \
+	persistent-data/space-maps/disk.cc \
+	persistent-data/space-maps/recursive.cc \
+	persistent-data/space-maps/careful_alloc.cc \
+	persistent-data/transaction_manager.cc \
+
+ERA_CHECK_OBJECTS=$(subst .cc,.o,$(ERA_CHECK_SOURCE))
+
+era_check: $(ERA_CHECK_OBJECTS) era/era_check.o
+	@echo "    [LD]  $@"
+	$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
+
+#----------------------------------------------------------------
+
 DEPEND_FILES=\
 	$(subst .cc,.d,$(SOURCE)) \
 	$(subst .cc,.d,$(TEST_SOURCE)) \

era/era_check.cc

@@ -0,0 +1,249 @@
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <stdexcept>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <libgen.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "base/error_state.h"
+#include "base/nested_output.h"
+#include "era/superblock.h"
+#include "persistent-data/block.h"
+#include "persistent-data/file_utils.h"
+#include "persistent-data/space_map.h"
+#include "persistent-data/space-maps/core.h"
+#include "persistent-data/transaction_manager.h"
+#include "version.h"
+
+using namespace base;
+using namespace boost;
+using namespace era;
+using namespace persistent_data;
+using namespace std;
+
+//----------------------------------------------------------------
+
+namespace {
+	class reporter_base {
+	public:
+		reporter_base(nested_output &o)
+		: out_(o),
+		  err_(NO_ERROR) {
+		}
+
+		virtual ~reporter_base() {}
+
+		nested_output &out() {
+			return out_;
+		}
+
+		nested_output::nest push() {
+			return out_.push();
+		}
+
+		base::error_state get_error() const {
+			return err_;
+		}
+
+		void mplus_error(error_state err) {
+			err_ = combine_errors(err_, err);
+		}
+
+	private:
+		nested_output &out_;
+		error_state err_;
+	};
+
+	class superblock_reporter : public superblock_damage::damage_visitor, reporter_base {
+	public:
+		superblock_reporter(nested_output &o)
+		: reporter_base(o) {
+		}
+
+		virtual void visit(superblock_damage::superblock_corrupt const &d) {
+			out() << "superblock is corrupt" << end_message();
+			{
+				nested_output::nest _ = push();
+				out() << d.get_desc() << end_message();
+			}
+
+			mplus_error(FATAL);
+		}
+
+		virtual void visit(superblock_damage::superblock_invalid const &d) {
+			out() << "superblock is invalid" << end_message();
+			{
+				nested_output::nest _ = push();
+				out() << d.get_desc() << end_message();
+			}
+
+			mplus_error(FATAL);
+		}
+
+		using reporter_base::get_error;
+	};
+
+	//--------------------------------
+
+	transaction_manager::ptr open_tm(block_manager<>::ptr bm) {
+		space_map::ptr sm(new core_map(bm->get_nr_blocks()));
+		sm->inc(SUPERBLOCK_LOCATION);
+		transaction_manager::ptr tm(new transaction_manager(bm, sm));
+		return tm;
+	}
+
+	//--------------------------------
+
+	struct flags {
+		flags()
+			: superblock_only_(false),
+			  quiet_(false) {
+		}
+
+		bool superblock_only_;
+		bool quiet_;
+	};
+
+	struct stat guarded_stat(string const &path) {
+		struct stat info;
+
+		int r = ::stat(path.c_str(), &info);
+		if (r) {
+			ostringstream msg;
+			char buffer[128], *ptr;
+
+			ptr = ::strerror_r(errno, buffer, sizeof(buffer));
+			msg << path << ": " << ptr;
+			throw runtime_error(msg.str());
+		}
+
+		return info;
+	}
+
+	error_state metadata_check(block_manager<>::ptr bm, flags const &fs) {
+		nested_output out(cerr, 2);
+		if (fs.quiet_)
+			out.disable();
+
+		superblock_reporter sb_rep(out);
+
+		out << "examining superblock" << end_message();
+		{
+			nested_output::nest _ = out.push();
+			check_superblock(bm, bm->get_nr_blocks(), sb_rep);
+		}
+
+		if (sb_rep.get_error() == FATAL)
+			return FATAL;
+
+#if 0
+		superblock sb = read_superblock(bm);
+		transaction_manager::ptr tm = open_tm(bm);
+
+		// FIXME: make an error class that's an instance of mplus
+		return combine_errors(sb_rep.get_error(),
+				      combine_errors(mapping_rep.get_error(),
+						     combine_errors(hint_rep.get_error(),
+								    discard_rep.get_error())));
+#endif
+
+		return FATAL;
+	}
+
+	int check(string const &path, flags const &fs) {
+		error_state err;
+		struct stat info = guarded_stat(path);
+
+		if (!S_ISREG(info.st_mode) && !S_ISBLK(info.st_mode)) {
+			ostringstream msg;
+			msg << path << ": " << "Not a block device or regular file";
+			throw runtime_error(msg.str());
+		}
+
+		block_manager<>::ptr bm = open_bm(path, block_io<>::READ_ONLY);
+		err = metadata_check(bm, fs);
+
+		return err == NO_ERROR ? 0 : 1;
+	}
+
+	int check_with_exception_handling(string const &path, flags const &fs) {
+		int r;
+		try {
+			r = check(path, fs);
+
+		} catch (std::exception &e) {
+			if (!fs.quiet_)
+				cerr << e.what() << endl;
+			r = 1;
+		}
+
+		return r;
+
+	}
+
+	void usage(ostream &out, string const &cmd) {
+		out << "Usage: " << cmd << " [options] {device|file}" << endl
+		    << "Options:" << endl
+		    << "  {-q|--quiet}" << endl
+		    << "  {-h|--help}" << endl
+		    << "  {-V|--version}" << endl
+		    << "  {--super-block-only}" << endl;
+	}
+}
+
+//----------------------------------------------------------------
+
+int main(int argc, char **argv)
+{
+	int c;
+	flags fs;
+	const char shortopts[] = "qhV";
+	const struct option longopts[] = {
+		{ "quiet", no_argument, NULL, 'q' },
+		{ "super-block-only", no_argument, NULL, 1 },
+		{ "help", no_argument, NULL, 'h' },
+		{ "version", no_argument, NULL, 'V' },
+		{ NULL, no_argument, NULL, 0 }
+	};
+
+	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
+		switch(c) {
+		case 1:
+			fs.superblock_only_ = true;
+			break;
+
+		case 'h':
+			usage(cout, basename(argv[0]));
+			return 0;
+
+		case 'q':
+			fs.quiet_ = true;
+			break;
+
+		case 'V':
+			cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
+			return 0;
+
+		default:
+			usage(cerr, basename(argv[0]));
+			return 1;
+		}
+	}
+
+	if (argc == optind) {
+		cerr << "No input file provided." << endl;
+		usage(cerr, basename(argv[0]));
+		return 1;
+	}
+
+	return check_with_exception_handling(argv[optind], fs);
+}
+
+//----------------------------------------------------------------

era/superblock.cc

@@ -5,6 +5,7 @@
 
 using namespace base;
 using namespace era;
+using namespace superblock_damage;
 using namespace persistent_data;
 
 //----------------------------------------------------------------
@@ -53,6 +54,58 @@ namespace  {
 
 //----------------------------------------------------------------
 
+superblock_flags::superblock_flags()
+	: unhandled_flags_(0)
+{
+}
+
+superblock_flags::superblock_flags(uint32_t bits)
+{
+	if (bits & (1 << CLEAN_SHUTDOWN_BIT)) {
+		flags_.insert(CLEAN_SHUTDOWN);
+		bits &= ~(1 << CLEAN_SHUTDOWN_BIT);
+	}
+
+	unhandled_flags_ = bits;
+}
+
+void
+superblock_flags::set_flag(flag f)
+{
+	flags_.insert(f);
+}
+
+void
+superblock_flags::clear_flag(flag f)
+{
+	flags_.erase(f);
+}
+
+bool
+superblock_flags::get_flag(flag f) const
+{
+	return flags_.find(f) != flags_.end();
+}
+
+uint32_t
+superblock_flags::encode() const
+{
+	uint32_t r = 0;
+
+	if (get_flag(CLEAN_SHUTDOWN))
+		r = r | (1 << CLEAN_SHUTDOWN_BIT);
+
+	return r;
+}
+
+uint32_t
+superblock_flags::get_unhandled_flags() const
+{
+	return unhandled_flags_;
+}
+
+//----------------------------------------------------------------
+
 superblock::superblock()
 	: csum(0),
 	  blocknr(0),
@@ -112,9 +165,33 @@ superblock_traits::pack(value_type const &value, disk_type &disk)
 	disk.era_array_root = to_disk<le64>(value.era_array_root);
 }
 
+//--------------------------------
+
+superblock_corrupt::superblock_corrupt(std::string const &desc)
+	: damage(desc)
+{
+}
+
+void
+superblock_corrupt::visit(damage_visitor &v) const
+{
+	v.visit(*this);
+}
+
+superblock_invalid::superblock_invalid(std::string const &desc)
+	: damage(desc)
+{
+}
+
+void
+superblock_invalid::visit(damage_visitor &v) const
+{
+	v.visit(*this);
+}
+
 //----------------------------------------------------------------
 
-namespace validator {
+namespace era_validator {
 	using namespace persistent_data;
 
         uint32_t const VERSION = 1;
@@ -150,7 +227,7 @@ superblock
 era::read_superblock(block_manager<>::ptr bm, block_address location)
 {
 	superblock sb;
-	block_manager<>::read_ref r = bm->read_lock(location, validator::mk_v());
+	block_manager<>::read_ref r = bm->read_lock(location, era_validator::mk_v());
 	superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(&r.data());
 	superblock_traits::unpack(*sbd, sb);
 
@@ -160,8 +237,90 @@ era::read_superblock(block_manager<>::ptr bm, block_address location)
 void
 era::write_superblock(block_manager<>::ptr bm, superblock const &sb, block_address location)
 {
-	block_manager<>::write_ref w = bm->superblock_zero(location, validator::mk_v());
+	block_manager<>::write_ref w = bm->superblock_zero(location, era_validator::mk_v());
 	superblock_traits::pack(sb, *reinterpret_cast<superblock_disk *>(w.data().raw()));
 }
 
+void
+era::check_superblock(superblock const &sb,
+		      block_address nr_metadata_blocks,
+		      damage_visitor &visitor)
+{
+	if (sb.flags.get_unhandled_flags()) {
+		ostringstream msg;
+		msg << "invalid flags: " << sb.flags.get_unhandled_flags();
+		visitor.visit(superblock_invalid(msg.str()));
+	}
+
+	if (sb.blocknr >= nr_metadata_blocks) {
+		ostringstream msg;
+		msg << "blocknr out of bounds: " << sb.blocknr << " >= " << nr_metadata_blocks;
+		visitor.visit(superblock_invalid(msg.str()));
+	}
+
+	if (sb.magic != SUPERBLOCK_MAGIC) {
+		ostringstream msg;
+		msg << "magic in incorrect: " << sb.magic;
+		visitor.visit(superblock_invalid(msg.str()));
+	}
+
+	if (sb.version >= VERSION_END) {
+		ostringstream msg;
+		msg << "version incorrect: " << sb.version;
+		visitor.visit(superblock_invalid(msg.str()));
+	}
+
+	if (sb.version < VERSION_BEGIN) {
+		ostringstream msg;
+		msg << "version incorrect: " << sb.version;
+		visitor.visit(superblock_invalid(msg.str()));
+	}
+
+	if (sb.metadata_block_size != 8) {
+		ostringstream msg;
+		msg << "metadata block size incorrect: " << sb.metadata_block_size;
+		visitor.visit(superblock_invalid(msg.str()));
+	}
+
+	if (sb.bloom_tree_root == SUPERBLOCK_LOCATION) {
+		string msg("bloom tree root points back to the superblock");
+		visitor.visit(superblock_invalid(msg));
+	}
+
+	if (sb.era_array_root == SUPERBLOCK_LOCATION) {
+		string msg("era array root points back to the superblock");
+		visitor.visit(superblock_invalid(msg));
+	}
+
+	if (sb.bloom_tree_root == sb.era_array_root) {
+		ostringstream msg;
+		msg << "bloom tree root and era array both point to the same block: "
+		    << sb.era_array_root;
+		visitor.visit(superblock_invalid(msg.str()));
+	}
+}
+
+void
+era::check_superblock(persistent_data::block_manager<>::ptr bm,
+		      block_address nr_metadata_blocks,
+		      damage_visitor &visitor)
+{
+	superblock sb;
+
+	try {
+		sb = read_superblock(bm, SUPERBLOCK_LOCATION);
+
+	} catch (std::exception const &e) {
+
+		// FIXME: what if it fails due to a zero length file?  Not
+		// really a corruption, so much as an io error.  Should we
+		// separate these?
+
+		visitor.visit(superblock_corrupt(e.what()));
+	}
+
+	check_superblock(sb, nr_metadata_blocks, visitor);
+}
+
+
 //----------------------------------------------------------------

era/superblock.h

@@ -67,16 +67,64 @@ namespace era {
 
 	//--------------------------------
 
+	namespace superblock_damage {
+
+		class damage_visitor;
+
+		class damage {
+		public:
+			damage(std::string const &desc)
+				: desc_(desc) {
+			}
+
+			virtual ~damage() {}
+			virtual void visit(damage_visitor &v) const = 0;
+
+			std::string const &get_desc() const {
+				return desc_;
+			}
+
+		private:
+			std::string desc_;
+		};
+
+		struct superblock_corrupt : public damage {
+			superblock_corrupt(std::string const &desc);
+			void visit(damage_visitor &v) const;
+		};
+
+		struct superblock_invalid : public damage {
+			superblock_invalid(std::string const &desc);
+			void visit(damage_visitor &v) const;
+		};
+
+		class damage_visitor {
+		public:
+			virtual ~damage_visitor() {}
+
+			void visit(damage const &d);
+
+			virtual void visit(superblock_corrupt const &d) = 0;
+			virtual void visit(superblock_invalid const &d) = 0;
+		};
+	}
+
+	//--------------------------------
+
 	superblock read_superblock(persistent_data::block_manager<>::ptr bm,
 				   persistent_data::block_address location = SUPERBLOCK_LOCATION);
 
 	void write_superblock(persistent_data::block_manager<>::ptr bm,
 			      superblock const &sb,
 			      persistent_data::block_address location = SUPERBLOCK_LOCATION);
-#if 0
+
 	void check_superblock(superblock const &sb,
+			      persistent_data::block_address nr_metadata_blocks,
+			      superblock_damage::damage_visitor &visitor);
+
+	void check_superblock(persistent_data::block_manager<>::ptr bm,
+			      persistent_data::block_address nr_metadata_blocks,
 			      superblock_damage::damage_visitor &visitor);
-#endif
 }
 
 //----------------------------------------------------------------

features/era_check.feature

@@ -0,0 +1,83 @@
+Feature: era_check
+  Scenario: print version (-V flag)
+    When I run `era_check -V`
+    
+    Then it should pass with version
+
+  Scenario: print version (--version flag)
+    When I run `era_check --version`
+
+    Then it should pass with version
+
+  Scenario: print help
+    When I run `era_check --help`
+
+    Then it should pass
+    And era_usage to stdout
+
+  Scenario: print help
+    When I run `era_check -h`
+
+    Then it should pass
+    And era_usage to stdout
+
+  Scenario: Metadata file must be specified
+    When I run `era_check`
+
+    Then it should fail
+    And era_usage to stderr
+    And the stderr should contain:
+
+    """
+    No input file provided.
+    """
+
+  Scenario: Metadata file doesn't exist
+    When I run `era_check /arbitrary/filename`
+
+    Then it should fail
+    And the stderr should contain:
+    """
+    /arbitrary/filename: No such file or directory
+    """
+
+  Scenario: Metadata file cannot be a directory
+    Given a directory called foo
+
+    When I run `era_check foo`
+
+    Then it should fail
+    And the stderr should contain:
+    """
+    foo: Not a block device or regular file
+    """
+
+  # This test will fail if you're running as root
+  Scenario: Metadata file exists, but can't be opened
+    Given input without read permissions
+    When I run `era_check input`
+    Then it should fail
+    And the stderr should contain:
+    """
+    Permission denied
+    """
+
+  Scenario: Metadata file full of zeroes
+    Given input file
+    And block 1 is zeroed
+    When I run `era_check input`
+    Then it should fail
+
+  Scenario: --quiet is observed
+    Given input file
+    And block 1 is zeroed
+    When I run `era_check --quiet input`
+    Then it should fail
+    And it should give no output
+
+  Scenario: -q is observed
+    Given input file
+    And block 1 is zeroed
+    When I run `era_check -q input`
+    Then it should fail
+    And it should give no output

features/step_definitions/era_steps.rb

@@ -0,0 +1,16 @@
+ERA_USAGE =<<EOF
+Usage: era_check [options] {device|file}
+Options:
+  {-q|--quiet}
+  {-h|--help}
+  {-V|--version}
+  {--super-block-only}
+EOF
+
+Then /^era_usage to stdout$/ do
+  assert_partial_output(ERA_USAGE, all_stdout)
+end
+
+Then /^era_usage to stderr$/ do
+  assert_partial_output(ERA_USAGE, all_stderr)
+end

unit-tests/Makefile.in

@@ -59,6 +59,7 @@ TEST_SOURCE=\
 	unit-tests/cache_superblock_t.cc \
 	unit-tests/damage_tracker_t.cc \
 	unit-tests/endian_t.cc \
+	unit-tests/era_superblock_t.cc \
 	unit-tests/rmap_visitor_t.cc \
 	unit-tests/run_set_t.cc \
 	unit-tests/space_map_t.cc \

unit-tests/era_superblock_t.cc

@@ -0,0 +1,112 @@
+#include "gmock/gmock.h"
+#include "base/bits.h"
+#include "era/superblock.h"
+
+using namespace base;
+using namespace era;
+using namespace superblock_damage;
+using namespace testing;
+
+//----------------------------------------------------------------
+
+namespace {
+	unsigned const NR_METADATA_BLOCKS = 100;
+
+	class damage_visitor_mock : public damage_visitor {
+	public:
+		MOCK_METHOD1(visit, void (superblock_corrupt const &));
+		MOCK_METHOD1(visit, void (superblock_invalid const &));
+	};
+
+	class EraSuperblockTests : public Test {
+	public:
+		EraSuperblockTests() {
+			sb_.bloom_tree_root = 1;
+			sb_.era_array_root = 2;
+		}
+
+		void check() {
+			check_superblock(sb_, NR_METADATA_BLOCKS, visitor_);
+		}
+
+		void check_invalid() {
+			EXPECT_CALL(visitor_, visit(Matcher<superblock_invalid const &>(_))).Times(1);
+			check();
+		}
+
+		damage_visitor_mock visitor_;
+		superblock sb_;
+	};
+}
+
+//----------------------------------------------------------------
+
+TEST_F(EraSuperblockTests, default_constructed_superblock_is_valid)
+{
+	check();
+}
+
+TEST_F(EraSuperblockTests, clean_shutdown_flag_is_valid)
+{
+	sb_.flags.set_flag(superblock_flags::CLEAN_SHUTDOWN);
+	check();
+}
+
+TEST_F(EraSuperblockTests, unhandled_flags_get_set_correctly_and_is_invalid)
+{
+	uint32_t bad_flag = 1 << 12;
+	sb_.flags = superblock_flags(bad_flag | 1);
+	ASSERT_THAT(sb_.flags.get_unhandled_flags(), Eq(bad_flag));
+	check_invalid();
+}
+
+TEST_F(EraSuperblockTests, blocknr_is_in_range)
+{
+	sb_.blocknr = NR_METADATA_BLOCKS;
+	check_invalid();
+}
+
+TEST_F(EraSuperblockTests, magic_is_checked)
+{
+	sb_.magic = 12345;
+	check_invalid();
+}
+
+TEST_F(EraSuperblockTests, version_gt_1_is_checked)
+{
+	sb_.version = 2;
+	check_invalid();
+}
+
+TEST_F(EraSuperblockTests, version_lt_1_is_checked)
+{
+	sb_.version = 0;
+	check_invalid();
+}
+
+TEST_F(EraSuperblockTests, metadata_block_size_checked)
+{
+	sb_.metadata_block_size = 16;
+	check_invalid();
+}
+
+TEST_F(EraSuperblockTests, bloom_tree_root_isnt_0)
+{
+	sb_.bloom_tree_root = 0;
+	check_invalid();
+}
+
+TEST_F(EraSuperblockTests, era_array_root_isnt_0)
+{
+	sb_.era_array_root = 0;
+	check_invalid();
+}
+
+TEST_F(EraSuperblockTests, bloom_root_isnt_era_array_root)
+{
+	sb_.bloom_tree_root = 10;
+	sb_.era_array_root = 10;
+	check_invalid();
+}
+
+//----------------------------------------------------------------