[unit-tests] Add test cases for space maps

- Increase test metadata size
- Test block allocation among multiple transactions
- Test the interaction between sm_recursive and sm_careful_alloc
- Test intended in-place modification

unit-tests/space_map_t.cc

@@ -20,7 +20,9 @@
 #include "persistent-data/space-maps/disk.h"
 #include "persistent-data/space-maps/core.h"
 #include "persistent-data/space-maps/careful_alloc.h"
+#include "persistent-data/space-maps/disk_structures.h"
 #include "persistent-data/space-maps/recursive.h"
+#include "test_utils.h"
 
 using namespace std;
 using namespace persistent_data;
@@ -29,14 +31,16 @@ using namespace testing;
 //----------------------------------------------------------------
 
 namespace {
-	block_address const NR_BLOCKS = 1000; // FIXME: bump up
+	unsigned const ENTRIES_PER_BLOCK = (MD_BLOCK_SIZE - sizeof(sm_disk_detail::bitmap_header)) * 4;
+	unsigned const NR_BITMAPS = 2;
+	unsigned const NR_BLOCKS = ENTRIES_PER_BLOCK * NR_BITMAPS;
 	block_address const SUPERBLOCK = 0;
-	block_address const MAX_LOCKS = 8;
+	unsigned const MAX_LOCKS = 8;
 
 	class SpaceMapTests : public Test {
 	public:
 		SpaceMapTests()
-			: bm_(new block_manager("./test.data", NR_BLOCKS, MAX_LOCKS, block_manager::READ_WRITE)),
+			: bm_(test::create_bm(NR_BLOCKS)),
 			  sm_(create_core_map(NR_BLOCKS)),
 			  tm_(bm_, sm_) {
 		}
@@ -229,8 +233,89 @@ namespace {
 			test_high_ref_counts(SMCreator::create(tm_));
 		}
 
-		void
+
-		copy_space_maps(space_map::ptr lhs, space_map::ptr rhs) {
+		block_manager::ptr bm_;
+		space_map::ptr sm_;
+		transaction_manager tm_;
+	};
+
+	//--------------------------------
+
+	class MetadataSpaceMapTests : public Test {
+	public:
+		MetadataSpaceMapTests() {
+			memset(metadata_sm_root_, 0, sizeof(metadata_sm_root_));
+			create_metadata_sm(NR_BLOCKS);
+		}
+
+		void commit() {
+			metadata_sm_->commit();
+
+			ASSERT_THAT(metadata_sm_->root_size(), Le(sizeof(metadata_sm_root_)));
+			metadata_sm_->copy_root(metadata_sm_root_, sizeof(metadata_sm_root_));
+		}
+
+		void open() {
+			bm_ = block_manager::ptr(new block_manager("./test.data", NR_BLOCKS, MAX_LOCKS, block_manager::READ_WRITE));
+			space_map::ptr core_sm{create_core_map(NR_BLOCKS)};
+			tm_ = transaction_manager::ptr(new transaction_manager(bm_, core_sm));
+			metadata_sm_ = persistent_data::open_metadata_sm(*tm_, metadata_sm_root_);
+			tm_->set_sm(metadata_sm_);
+		}
+
+		void create_data_sm(block_address nr_blocks) {
+			data_sm_ = create_disk_sm(*tm_, nr_blocks);
+		}
+
+		void load_ies(std::vector<sm_disk_detail::index_entry> &entries) {
+			sm_disk_detail::sm_root v;
+			get_root(v);
+
+			block_address nr_indexes = (v.nr_blocks_ + ENTRIES_PER_BLOCK - 1) / ENTRIES_PER_BLOCK;
+			entries.resize(nr_indexes);
+			ASSERT_EQ(entries.size(), NR_BITMAPS);
+
+			block_manager::read_ref rr =
+				bm_->read_lock(v.bitmap_root_, index_validator());
+			sm_disk_detail::metadata_index const *mdi = reinterpret_cast<sm_disk_detail::metadata_index const *>(rr.data());
+			for (block_address i = 0; i < nr_indexes; i++) {
+				sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), entries[i]);
+				ASSERT_EQ(metadata_sm_->get_count(entries[i].blocknr_), 1u);
+			}
+		}
+
+		// note that the index_block is not shadowed until space map commit
+		void get_root(sm_disk_detail::sm_root &v) const {
+			sm_disk_detail::sm_root_disk d;
+			metadata_sm_->copy_root(&d, sizeof(d));
+			sm_disk_detail::sm_root_traits::unpack(d, v);
+		}
+
+		void mark_shadowed() {
+			block_counter bc(true);
+			metadata_sm_->count_metadata(bc);
+
+			block_address nr_blocks = metadata_sm_->get_nr_blocks();
+			for (block_address b = 0; b < nr_blocks; b++) {
+				if (bc.get_count(b))
+					tm_->mark_shadowed(b);
+			}
+		}
+
+		checked_space_map::ptr metadata_sm_;
+		checked_space_map::ptr data_sm_;
+
+	private:
+		void create_metadata_sm(block_address nr_blocks) {
+			bm_ = test::create_bm(NR_BLOCKS);
+			space_map::ptr core_sm{create_core_map(nr_blocks)};
+			tm_ = transaction_manager::ptr(new transaction_manager(bm_, core_sm));
+			metadata_sm_ = persistent_data::create_metadata_sm(*tm_, nr_blocks);
+			copy_space_maps(metadata_sm_, core_sm);
+			tm_->set_sm(metadata_sm_);
+		}
+
+		void copy_space_maps(space_map::ptr lhs, space_map::ptr rhs) {
 			for (block_address b = 0; b < rhs->get_nr_blocks(); b++) {
 				uint32_t count = rhs->get_count(b);
 				if (count > 0)
@@ -239,8 +324,9 @@ namespace {
 		}
 
 		block_manager::ptr bm_;
-		space_map::ptr sm_;
+		transaction_manager::ptr tm_;
-		transaction_manager tm_;
+
+		unsigned char metadata_sm_root_[128];
 	};
 }
 
@@ -273,17 +359,157 @@ TEST_F(SpaceMapTests, test_sm_metadata)
 	test_sm_reopen<sm_metadata_creator>();
 }
 
-TEST_F(SpaceMapTests, test_metadata_and_disk)
+//----------------------------------------------------------------
-{
-	block_manager::ptr bm(
-		new block_manager("./test.data", NR_BLOCKS, MAX_LOCKS, block_manager::READ_WRITE));
-	space_map::ptr core_sm{create_core_map(NR_BLOCKS)};
-	transaction_manager::ptr tm(new transaction_manager(bm, core_sm));
-	persistent_space_map::ptr metadata_sm = persistent_data::create_metadata_sm(*tm, NR_BLOCKS);
-	copy_space_maps(metadata_sm, core_sm);
-	tm->set_sm(metadata_sm);
 
-	persistent_space_map::ptr data_sm_ = create_disk_sm(*tm, NR_BLOCKS * 2);
+TEST_F(MetadataSpaceMapTests, test_metadata_and_disk)
+{
+	create_data_sm(NR_BLOCKS * 2);
+}
+
+// Test whether sm_recursive protected allocated blocks in a recursion
+// (github issue #70)
+TEST_F(MetadataSpaceMapTests, test_allocate_blocks)
+{
+	commit();
+	open();
+
+	std::vector<sm_disk_detail::index_entry> entries;
+	load_ies(entries);
+
+	block_address nr_allocatable = metadata_sm_->get_nr_free() - NR_BITMAPS;
+	block_address nr_allocated = 0;
+	while (true) {
+		space_map::maybe_block b = metadata_sm_->new_block();
+		if (!b)
+			break;
+		++nr_allocated;
+
+		for (auto const &e : entries)
+			ASSERT_NE(*b, e.blocknr_);
+	}
+
+	ASSERT_EQ(nr_allocated, nr_allocatable);
+	for (auto const &e : entries)
+		ASSERT_EQ(metadata_sm_->get_count(e.blocknr_), 0u);
+}
+
+// Test whether sm_careful_alloc protects released blocks in a recursion.
+// (github issue #97)
+TEST_F(MetadataSpaceMapTests, test_multiple_shadows_by_dec)
+{
+	// Occupy all the blocks belonging to the first bitmap,
+	// to change the position of further allocated blocks.
+	for (unsigned i = 0; i < ENTRIES_PER_BLOCK; i++)
+		metadata_sm_->new_block();
+
+	// the extra two blocks are the index block and the ref-count tree root
+	ASSERT_EQ(metadata_sm_->get_nr_free(), ENTRIES_PER_BLOCK - NR_BITMAPS - 2);
+
+	commit();
+	open();
+
+	std::vector<sm_disk_detail::index_entry> entries;
+	load_ies(entries);
+
+	// Releasing the block belonging to the second bitmap results in
+	// shadowing the second, then the first bitmap blocks.
+	// The shadow operations must not reuse the released block.
+	block_address last = metadata_sm_->get_nr_blocks() - metadata_sm_->get_nr_free() - 1;
+	ASSERT_EQ(metadata_sm_->get_count(last), 1u);
+	metadata_sm_->dec(last);
+
+	// assert that the released blocks are not used
+	ASSERT_EQ(metadata_sm_->get_count(last), 0u);
+	for (auto const &e : entries)
+		ASSERT_EQ(metadata_sm_->get_count(e.blocknr_), 0u);
+
+	ASSERT_EQ(metadata_sm_->get_nr_free(), ENTRIES_PER_BLOCK - NR_BITMAPS - 1);
+
+	// check ref-counts of shadowed bitmaps
+	commit();
+	open();
+	load_ies(entries);
+}
+
+// Test whether sm_recursive protects allocated blocks in a recursion
+// (github issue #70)
+TEST_F(MetadataSpaceMapTests, test_multiple_shadows_by_inc)
+{
+	// Occupy all the blocks belonging to the first bitmap,
+	// to change the position of further allocated blocks.
+	for (unsigned i = 0; i < ENTRIES_PER_BLOCK; i++)
+		metadata_sm_->new_block();
+
+	// the extra two blocks are the index block and the ref-count tree root
+	ASSERT_EQ(metadata_sm_->get_nr_free(), ENTRIES_PER_BLOCK - NR_BITMAPS - 2);
+
+	commit();
+	open();
+
+	std::vector<sm_disk_detail::index_entry> entries;
+	load_ies(entries);
+
+	// allocating a block results in shadowing the second,
+	// then the first bitmap
+	space_map::maybe_block b = metadata_sm_->new_block();
+
+	// assert reference counts
+	ASSERT_EQ(metadata_sm_->get_count(*b), 1u);
+	for (auto const &e : entries)
+		ASSERT_EQ(metadata_sm_->get_count(e.blocknr_), 0u);
+
+	ASSERT_EQ(metadata_sm_->get_nr_free(), ENTRIES_PER_BLOCK - NR_BITMAPS - 3);
+
+	// check ref-counts of shadowed bitmaps
+	commit();
+	open();
+	load_ies(entries);
+}
+
+// Test whether intended in-place modification works
+TEST_F(MetadataSpaceMapTests, test_intended_in_place_modification)
+{
+	sm_disk_detail::sm_root root;
+	get_root(root);
+	std::vector<sm_disk_detail::index_entry> entries;
+	load_ies(entries);
+
+	commit();
+	open();
+	mark_shadowed();
+
+	// the bitmaps won't be shadowed,
+	// all the free blocks therefore become allocatable
+	block_address nr_allocatable = metadata_sm_->get_nr_free();
+	block_address nr_allocated = 0;
+	while (true) {
+		space_map::maybe_block b = metadata_sm_->new_block();
+		if (!b)
+			break;
+		++nr_allocated;
+
+		for (auto const &e : entries)
+			ASSERT_NE(*b, e.blocknr_);
+	}
+
+	ASSERT_EQ(nr_allocated, nr_allocatable);
+
+	commit();
+	open();
+
+	sm_disk_detail::sm_root root2;
+	get_root(root2);
+	std::vector<sm_disk_detail::index_entry> entries2;
+	load_ies(entries2);
+
+	// assert the space map block locations are not changed
+	ASSERT_EQ(root.bitmap_root_, root2.bitmap_root_);
+	ASSERT_EQ(root.ref_count_root_, root2.ref_count_root_);
+	ASSERT_EQ(entries.size(), entries2.size());
+	for (unsigned i = 0; i < entries.size(); i++)
+		ASSERT_EQ(entries[i].blocknr_, entries2[i].blocknr_);
+
+	ASSERT_EQ(root2.nr_allocated_, root.nr_allocated_ + nr_allocated);
 }
 
 //----------------------------------------------------------------