From 05c4b7b4006fce45d6da0460d05483da5a273ab3 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 30 Sep 2016 11:21:20 -0400 Subject: [PATCH] [cache] a lot of format 2 changes --- caching/cache_check.cc | 8 +++++++- caching/cache_restore.cc | 39 ++++++++++++++++++++++++++------------ caching/mapping_array.cc | 14 ++++++++------ caching/mapping_array.h | 3 ++- caching/metadata.cc | 29 ++++++++++++++++++---------- caching/metadata.h | 6 ++++-- caching/restore_emitter.cc | 23 ++++++++++++++++------ caching/restore_emitter.h | 4 +++- caching/superblock.cc | 7 ++++++- caching/superblock.h | 5 +++++ 10 files changed, 98 insertions(+), 40 deletions(-) diff --git a/caching/cache_check.cc b/caching/cache_check.cc index 33d041b..b82b4f7 100644 --- a/caching/cache_check.cc +++ b/caching/cache_check.cc @@ -248,7 +248,12 @@ namespace { { nested_output::nest _ = out.push(); mapping_array ma(*tm, mapping_array::ref_counter(), sb.mapping_root, sb.cache_blocks); - check_mapping_array(ma, mapping_rep); + check_mapping_array(ma, mapping_rep, sb.version); + } + + if (sb.version >= 2) { + persistent_data::bitset dirty(*tm, *sb.dirty_root, sb.cache_blocks); + // FIXME: is there no bitset checker? } } @@ -275,6 +280,7 @@ namespace { { nested_output::nest _ = out.push(); persistent_data::bitset discards(*tm, sb.discard_root, sb.discard_nr_blocks); + // FIXME: is there no bitset checker? } } } diff --git a/caching/cache_restore.cc b/caching/cache_restore.cc index 6eb42b4..37be5c7 100644 --- a/caching/cache_restore.cc +++ b/caching/cache_restore.cc @@ -43,29 +43,33 @@ namespace { struct flags { flags() : metadata_version(1), - override_metadata_version(false), clean_shutdown(true), - quiet(false) { + quiet(false), + override_metadata_version(false), + override_version(1) { } optional input; optional output; uint32_t metadata_version; - bool override_metadata_version; bool clean_shutdown; bool quiet; + + bool override_metadata_version; + unsigned override_version; }; int restore(flags const &fs) { try { block_manager<>::ptr bm = open_bm(*fs.output, block_manager<>::READ_WRITE); metadata::ptr md(new metadata(bm, metadata::CREATE)); - emitter::ptr restorer = create_restore_emitter(md, fs.clean_shutdown); + emitter::ptr restorer = create_restore_emitter(md, fs.clean_shutdown, + fs.metadata_version); if (fs.override_metadata_version) { cerr << "overriding" << endl; - md->sb_.version = fs.metadata_version; + md->sb_.version = fs.override_version; } check_file_exists(*fs.input); @@ -95,12 +99,13 @@ cache_restore_cmd::usage(std::ostream &out) const { out << "Usage: " << get_name() << " [options]" << endl << "Options:" << endl - << " {-h|--help}" << endl - << " {-i|--input} " << endl - << " {-o|--output} " << endl - << " {-q|--quiet}" << endl - << " {-V|--version}" << endl - << endl + << " {-h|--help}\n" + << " {-i|--input} \n" + << " {-o|--output} \n" + << " {-q|--quiet}\n" + << " {--metadata-version} <1 or 2>\n" + << " {-V|--version}\n" + << "\n" << " {--debug-override-metadata-version} " << endl << " {--omit-clean-shutdown}" << endl; } @@ -114,6 +119,7 @@ cache_restore_cmd::run(int argc, char **argv) option const long_opts[] = { { "debug-override-metadata-version", required_argument, NULL, 0 }, { "omit-clean-shutdown", no_argument, NULL, 1 }, + { "metadata-version", required_argument, NULL, 2 }, { "help", no_argument, NULL, 'h'}, { "input", required_argument, NULL, 'i' }, { "output", required_argument, NULL, 'o'}, @@ -125,14 +131,23 @@ cache_restore_cmd::run(int argc, char **argv) while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) { switch(c) { case 0: - fs.metadata_version = lexical_cast(optarg); fs.override_metadata_version = true; + fs.override_version = lexical_cast(optarg); break; case 1: fs.clean_shutdown = false; break; + case 2: + fs.metadata_version = lexical_cast(optarg); + if ((fs.metadata_version < MIN_METADATA_VERSION) || + (fs.metadata_version > MAX_METADATA_VERSION)) { + cerr << "Bad metadata version\n\n"; + return 1; + } + break; + case 'h': usage(cout); return 0; diff --git a/caching/mapping_array.cc b/caching/mapping_array.cc index c6af4ef..0ecf110 100644 --- a/caching/mapping_array.cc +++ b/caching/mapping_array.cc @@ -60,8 +60,9 @@ invalid_mapping::visit(damage_visitor &v) const namespace { class check_mapping_visitor : public mapping_visitor { public: - check_mapping_visitor(damage_visitor &visitor) - : visitor_(visitor) { + check_mapping_visitor(damage_visitor &visitor, unsigned metadata_version) + : visitor_(visitor), + allowed_flags_(metadata_version == 1 ? (M_VALID | M_DIRTY) : M_VALID) { } virtual void visit(block_address cblock, mapping const &m) { @@ -90,12 +91,13 @@ namespace { seen_oblocks_.insert(m.oblock_); } - static bool unknown_flags(mapping const &m) { - return (m.flags_ & ~(M_VALID | M_DIRTY)); + bool unknown_flags(mapping const &m) { + return m.flags_ & ~allowed_flags_; } damage_visitor &visitor_; set seen_oblocks_; + uint32_t allowed_flags_; }; class ll_damage_visitor { @@ -123,9 +125,9 @@ caching::walk_mapping_array(mapping_array const &array, } void -caching::check_mapping_array(mapping_array const &array, damage_visitor &visitor) +caching::check_mapping_array(mapping_array const &array, damage_visitor &visitor, unsigned metadata_version) { - check_mapping_visitor mv(visitor); + check_mapping_visitor mv(visitor, metadata_version); walk_mapping_array(array, mv, visitor); } diff --git a/caching/mapping_array.h b/caching/mapping_array.h index 00d58d8..270400c 100644 --- a/caching/mapping_array.h +++ b/caching/mapping_array.h @@ -91,7 +91,8 @@ namespace caching { mapping_array_damage::damage_visitor &dv); void check_mapping_array(mapping_array const &array, - mapping_array_damage::damage_visitor &visitor); + mapping_array_damage::damage_visitor &visitor, + unsigned metadata_version); } //---------------------------------------------------------------- diff --git a/caching/metadata.cc b/caching/metadata.cc index 2368a3f..7d8bfae 100644 --- a/caching/metadata.cc +++ b/caching/metadata.cc @@ -3,6 +3,7 @@ #include "persistent-data/space-maps/core.h" using namespace caching; +namespace pd = persistent_data; //---------------------------------------------------------------- @@ -30,15 +31,11 @@ namespace { //---------------------------------------------------------------- -metadata::metadata(block_manager<>::ptr bm, open_type ot) +metadata::metadata(block_manager<>::ptr bm, open_type ot, unsigned metadata_version) { switch (ot) { case CREATE: - create_metadata(bm); - break; - - case OPEN: - open_metadata(bm); + create_metadata(bm, metadata_version); break; default: @@ -46,6 +43,11 @@ metadata::metadata(block_manager<>::ptr bm, open_type ot) } } +metadata::metadata(block_manager<>::ptr bm) +{ + open_metadata(bm); +} + void metadata::commit(bool clean_shutdown) { @@ -65,7 +67,7 @@ metadata::setup_hint_array(size_t width) } void -metadata::create_metadata(block_manager<>::ptr bm) +metadata::create_metadata(block_manager<>::ptr bm, unsigned metadata_version) { tm_ = open_tm(bm); @@ -79,7 +81,10 @@ metadata::create_metadata(block_manager<>::ptr bm) // We can't instantiate the hint array yet, since we don't know the // hint width. - discard_bits_ = persistent_data::bitset::ptr(new persistent_data::bitset(*tm_)); + discard_bits_ = pd::bitset::ptr(new pd::bitset(*tm_)); + + if (metadata_version >= 2) + dirty_bits_ = pd::bitset::ptr(new pd::bitset(*tm_)); } void @@ -100,8 +105,12 @@ metadata::open_metadata(block_manager<>::ptr bm) sb_.hint_root, sb_.cache_blocks)); if (sb_.discard_root) - discard_bits_ = persistent_data::bitset::ptr( - new persistent_data::bitset(*tm_, sb_.discard_root, sb_.discard_nr_blocks)); + discard_bits_ = pd::bitset::ptr( + new pd::bitset(*tm_, sb_.discard_root, sb_.discard_nr_blocks)); + + if (sb_.version >= 2) + dirty_bits_ = pd::bitset::ptr( + new pd::bitset(*tm_, *sb_.dirty_root, sb_.cache_blocks)); } void diff --git a/caching/metadata.h b/caching/metadata.h index 46c2b93..268b2f5 100644 --- a/caching/metadata.h +++ b/caching/metadata.h @@ -27,7 +27,8 @@ namespace caching { typedef block_manager<>::write_ref write_ref; typedef boost::shared_ptr ptr; - metadata(block_manager<>::ptr bm, open_type ot); + metadata(block_manager<>::ptr bm, open_type ot, unsigned metadata_version = 2); // Create only + metadata(block_manager<>::ptr bm); void commit(bool clean_shutdown = true); void setup_hint_array(size_t width); @@ -40,11 +41,12 @@ namespace caching { mapping_array::ptr mappings_; hint_array::ptr hints_; persistent_data::bitset::ptr discard_bits_; + persistent_data::bitset::ptr dirty_bits_; private: void init_superblock(); - void create_metadata(block_manager<>::ptr bm); + void create_metadata(block_manager<>::ptr bm, unsigned metadata_version); void open_metadata(block_manager<>::ptr bm); void commit_space_map(); diff --git a/caching/restore_emitter.cc b/caching/restore_emitter.cc index 12da592..34c5020 100644 --- a/caching/restore_emitter.cc +++ b/caching/restore_emitter.cc @@ -11,10 +11,11 @@ using namespace superblock_damage; namespace { class restorer : public emitter { public: - restorer(metadata::ptr md, bool clean_shutdown) + restorer(metadata::ptr md, bool clean_shutdown, unsigned metadata_version) : in_superblock_(false), md_(md), - clean_shutdown_(clean_shutdown) { + clean_shutdown_(clean_shutdown), + metadata_version_(metadata_version) { } virtual void begin_superblock(std::string const &uuid, @@ -24,6 +25,7 @@ namespace { size_t hint_width) { superblock &sb = md_->sb_; + sb.version = metadata_version_; strncpy((char *) sb.policy_name, policy.c_str(), sizeof(sb.policy_name)); memset(sb.policy_version, 0, sizeof(sb.policy_version)); // FIXME: should come from xml sb.policy_hint_size = hint_width; @@ -37,6 +39,10 @@ namespace { unmapped_value.flags_ = 0; md_->mappings_->grow(nr_cache_blocks, unmapped_value); + if (metadata_version_ > 1) + // make everything dirty by default + md_->dirty_bits_->grow(nr_cache_blocks, true); + vector hint_value(hint_width, '\0'); md_->hints_->grow(nr_cache_blocks, hint_value); } @@ -60,8 +66,11 @@ namespace { m.oblock_ = oblock; m.flags_ = M_VALID; - if (dirty) - m.flags_ = m.flags_ | M_DIRTY; + if (metadata_version_ == 1) { + if (dirty) + m.flags_ = m.flags_ | M_DIRTY; + } else + md_->dirty_bits_->set(cblock, dirty); md_->mappings_->set(cblock, m); } @@ -98,15 +107,17 @@ namespace { bool in_superblock_; metadata::ptr md_; bool clean_shutdown_; + unsigned metadata_version_; }; } //---------------------------------------------------------------- emitter::ptr -caching::create_restore_emitter(metadata::ptr md, bool clean_shutdown) +caching::create_restore_emitter(metadata::ptr md, unsigned metadata_version, + bool clean_shutdown) { - return emitter::ptr(new restorer(md, clean_shutdown)); + return emitter::ptr(new restorer(md, clean_shutdown, metadata_version)); } //---------------------------------------------------------------- diff --git a/caching/restore_emitter.h b/caching/restore_emitter.h index 9282895..5c077af 100644 --- a/caching/restore_emitter.h +++ b/caching/restore_emitter.h @@ -7,7 +7,9 @@ //---------------------------------------------------------------- namespace caching { - emitter::ptr create_restore_emitter(metadata::ptr md, bool clean_shutdown = true); + emitter::ptr create_restore_emitter(metadata::ptr md, + unsigned metadata_version, + bool clean_shutdown = true); } //---------------------------------------------------------------- diff --git a/caching/superblock.cc b/caching/superblock.cc index 3270b3f..0a2d8bb 100644 --- a/caching/superblock.cc +++ b/caching/superblock.cc @@ -45,6 +45,8 @@ namespace { le32 write_misses; le32 policy_version[CACHE_POLICY_VERSION_SIZE]; + + le64 dirty_root; // format 2 only } __attribute__ ((packed)); struct superblock_traits { @@ -57,7 +59,7 @@ namespace { uint32_t const SUPERBLOCK_MAGIC = 06142003; uint32_t const VERSION_BEGIN = 1; - uint32_t const VERSION_END = 2; + uint32_t const VERSION_END = 3; } //---------------------------------------------------------------- @@ -195,6 +197,9 @@ superblock_traits::unpack(superblock_disk const &disk, superblock &core) core.read_misses = to_cpu(disk.read_misses); core.write_hits = to_cpu(disk.write_hits); core.write_misses = to_cpu(disk.write_misses); + + if (core.version >= 2) + core.dirty_root = to_cpu(disk.dirty_root); } void diff --git a/caching/superblock.h b/caching/superblock.h index fdac263..2366bbc 100644 --- a/caching/superblock.h +++ b/caching/superblock.h @@ -4,6 +4,7 @@ #include "base/endian_utils.h" #include "persistent-data/data-structures/btree.h" +#include #include //---------------------------------------------------------------- @@ -11,6 +12,8 @@ namespace caching { typedef unsigned char __u8; + unsigned const MIN_METADATA_VERSION = 1; + unsigned const MAX_METADATA_VERSION = 2; unsigned const SPACE_MAP_ROOT_SIZE = 128; unsigned const CACHE_POLICY_NAME_SIZE = 16; unsigned const CACHE_POLICY_VERSION_SIZE = 3; @@ -78,6 +81,8 @@ namespace caching { uint32_t read_misses; uint32_t write_hits; uint32_t write_misses; + + boost::optional dirty_root; }; enum incompat_bits {