emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch '2016-14-08-cache-format2' into format2-merging

Browse Source 
Conflicts:
	VERSION
	unit-tests/Makefile.in
This commit is contained in:
Joe Thornber 2017-03-20 12:06:01 -04:00
commit 8c66660bb0
13 changed files with 109 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
caching/cache_check.cc
View File

@ -248,7 +248,12 @@ namespace {
{
nested_output::nest _ = out.push();
mapping_array ma(*tm, mapping_array::ref_counter(), sb.mapping_root, sb.cache_blocks);
check_mapping_array(ma, mapping_rep);
check_mapping_array(ma, mapping_rep, sb.version);
}
if (sb.version >= 2) {
persistent_data::bitset dirty(*tm, *sb.dirty_root, sb.cache_blocks);
// FIXME: is there no bitset checker?
}
}
@ -275,6 +280,7 @@ namespace {
{
nested_output::nest _ = out.push();
persistent_data::bitset discards(*tm, sb.discard_root, sb.discard_nr_blocks);
// FIXME: is there no bitset checker?
}
}
}

2
caching/cache_dump.cc
View File

@ -36,7 +36,7 @@ namespace {
int dump(string const &dev, string const &output, flags const &fs) {
try {
block_manager<>::ptr bm = open_bm(dev, block_manager<>::READ_ONLY);
metadata::ptr md(new metadata(bm, metadata::OPEN));
metadata::ptr md(new metadata(bm));
if (want_stdout(output)) {
emitter::ptr e = create_xml_emitter(cout);

2
caching/cache_repair.cc
View File

@ -19,7 +19,7 @@ using namespace caching;
namespace {
metadata::ptr open_metadata_for_read(string const &path) {
block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_ONLY);
return metadata::ptr(new metadata(bm, metadata::OPEN));
return metadata::ptr(new metadata(bm));
}
emitter::ptr output_emitter(string const &path) {

50
caching/cache_restore.cc
View File

@ -43,30 +43,34 @@ namespace {
struct flags {
flags()
: metadata_version(1),
override_metadata_version(false),
clean_shutdown(true),
quiet(false) {
quiet(false),
override_metadata_version(false),
override_version(1) {
}
optional<string> input;
optional<string> output;
uint32_t metadata_version;
bool override_metadata_version;
bool clean_shutdown;
bool quiet;
bool override_metadata_version;
unsigned override_version;
};
void override_version(metadata::ptr md, flags const &fs) {
md->sb_.version = fs.override_version;
md->commit(fs.clean_shutdown);
}
int restore(flags const &fs) {
try {
block_manager<>::ptr bm = open_bm(*fs.output, block_manager<>::READ_WRITE);
metadata::ptr md(new metadata(bm, metadata::CREATE));
emitter::ptr restorer = create_restore_emitter(md, fs.clean_shutdown);
if (fs.override_metadata_version) {
cerr << "overriding" << endl;
md->sb_.version = fs.metadata_version;
}
emitter::ptr restorer = create_restore_emitter(md, fs.clean_shutdown,
fs.metadata_version);
check_file_exists(*fs.input);
ifstream in(fs.input->c_str(), ifstream::in);
@ -74,6 +78,9 @@ namespace {
unique_ptr<progress_monitor> monitor = create_monitor(fs.quiet);
parse_xml(in, restorer, get_file_length(*fs.input), *monitor);
if (fs.override_metadata_version)
override_version(md, fs);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
@ -95,12 +102,13 @@ cache_restore_cmd::usage(std::ostream &out) const
{
out << "Usage: " << get_name() << " [options]" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-i|--input} <input xml file>" << endl
<< " {-o|--output} <output device or file>" << endl
<< " {-q|--quiet}" << endl
<< " {-V|--version}" << endl
<< endl
<< " {-h|--help}\n"
<< " {-i|--input} <input xml file>\n"
<< " {-o|--output} <output device or file>\n"
<< " {-q|--quiet}\n"
<< " {--metadata-version} <1 or 2>\n"
<< " {-V|--version}\n"
<< "\n"
<< " {--debug-override-metadata-version} <integer>" << endl
<< " {--omit-clean-shutdown}" << endl;
}
@ -114,6 +122,7 @@ cache_restore_cmd::run(int argc, char **argv)
option const long_opts[] = {
{ "debug-override-metadata-version", required_argument, NULL, 0 },
{ "omit-clean-shutdown", no_argument, NULL, 1 },
{ "metadata-version", required_argument, NULL, 2 },
{ "help", no_argument, NULL, 'h'},
{ "input", required_argument, NULL, 'i' },
{ "output", required_argument, NULL, 'o'},
@ -125,14 +134,23 @@ cache_restore_cmd::run(int argc, char **argv)
while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) {
switch(c) {
case 0:
fs.metadata_version = lexical_cast<uint32_t>(optarg);
fs.override_metadata_version = true;
fs.override_version = lexical_cast<uint32_t>(optarg);
break;
case 1:
fs.clean_shutdown = false;
break;
case 2:
fs.metadata_version = lexical_cast<uint32_t>(optarg);
if ((fs.metadata_version < MIN_METADATA_VERSION) ||
(fs.metadata_version > MAX_METADATA_VERSION)) {
cerr << "Bad metadata version\n\n";
return 1;
}
break;
case 'h':
usage(cout);
return 0;

2
caching/cache_writeback.cc
View File

@ -299,7 +299,7 @@ namespace {
int writeback_(flags const &f) {
block_manager<>::ptr bm = open_bm(*f.metadata_dev, block_manager<>::READ_WRITE);
metadata md(bm, metadata::OPEN);
metadata md(bm);
// FIXME: we're going to have to copy runs to get the through put with small block sizes
unsigned max_ios = f.cache_size / (md.sb_.data_block_size << SECTOR_SHIFT);

14
caching/mapping_array.cc
View File

@ -60,8 +60,9 @@ invalid_mapping::visit(damage_visitor &v) const
namespace {
class check_mapping_visitor : public mapping_visitor {
public:
check_mapping_visitor(damage_visitor &visitor)
: visitor_(visitor) {
check_mapping_visitor(damage_visitor &visitor, unsigned metadata_version)
: visitor_(visitor),
allowed_flags_(metadata_version == 1 ? (M_VALID | M_DIRTY) : M_VALID) {
}
virtual void visit(block_address cblock, mapping const &m) {
@ -90,12 +91,13 @@ namespace {
seen_oblocks_.insert(m.oblock_);
}
static bool unknown_flags(mapping const &m) {
return (m.flags_ & ~(M_VALID | M_DIRTY));
bool unknown_flags(mapping const &m) {
return m.flags_ & ~allowed_flags_;
}
damage_visitor &visitor_;
set<block_address> seen_oblocks_;
uint32_t allowed_flags_;
};
class ll_damage_visitor {
@ -123,9 +125,9 @@ caching::walk_mapping_array(mapping_array const &array,
}
void
caching::check_mapping_array(mapping_array const &array, damage_visitor &visitor)
caching::check_mapping_array(mapping_array const &array, damage_visitor &visitor, unsigned metadata_version)
{
check_mapping_visitor mv(visitor);
check_mapping_visitor mv(visitor, metadata_version);
walk_mapping_array(array, mv, visitor);
}

3
caching/mapping_array.h
View File

@ -91,7 +91,8 @@ namespace caching {
mapping_array_damage::damage_visitor &dv);
void check_mapping_array(mapping_array const &array,
mapping_array_damage::damage_visitor &visitor);
mapping_array_damage::damage_visitor &visitor,
unsigned metadata_version);
}
//----------------------------------------------------------------

29
caching/metadata.cc
View File

@ -3,6 +3,7 @@
#include "persistent-data/space-maps/core.h"
using namespace caching;
namespace pd = persistent_data;
//----------------------------------------------------------------
@ -30,15 +31,11 @@ namespace {
//----------------------------------------------------------------
metadata::metadata(block_manager<>::ptr bm, open_type ot)
metadata::metadata(block_manager<>::ptr bm, open_type ot, unsigned metadata_version)
{
switch (ot) {
case CREATE:
create_metadata(bm);
break;
case OPEN:
open_metadata(bm);
create_metadata(bm, metadata_version);
break;
default:
@ -46,6 +43,11 @@ metadata::metadata(block_manager<>::ptr bm, open_type ot)
}
}
metadata::metadata(block_manager<>::ptr bm)
{
open_metadata(bm);
}
void
metadata::commit(bool clean_shutdown)
{
@ -65,7 +67,7 @@ metadata::setup_hint_array(size_t width)
}
void
metadata::create_metadata(block_manager<>::ptr bm)
metadata::create_metadata(block_manager<>::ptr bm, unsigned metadata_version)
{
tm_ = open_tm(bm);
@ -79,7 +81,10 @@ metadata::create_metadata(block_manager<>::ptr bm)
// We can't instantiate the hint array yet, since we don't know the
// hint width.
discard_bits_ = persistent_data::bitset::ptr(new persistent_data::bitset(*tm_));
discard_bits_ = pd::bitset::ptr(new pd::bitset(*tm_));
if (metadata_version >= 2)
dirty_bits_ = pd::bitset::ptr(new pd::bitset(*tm_));
}
void
@ -100,8 +105,12 @@ metadata::open_metadata(block_manager<>::ptr bm)
sb_.hint_root, sb_.cache_blocks));
if (sb_.discard_root)
discard_bits_ = persistent_data::bitset::ptr(
new persistent_data::bitset(*tm_, sb_.discard_root, sb_.discard_nr_blocks));
discard_bits_ = pd::bitset::ptr(
new pd::bitset(*tm_, sb_.discard_root, sb_.discard_nr_blocks));
if (sb_.version >= 2)
dirty_bits_ = pd::bitset::ptr(
new pd::bitset(*tm_, *sb_.dirty_root, sb_.cache_blocks));
}
void

9
caching/metadata.h
View File

@ -19,15 +19,15 @@ namespace caching {
class metadata {
public:
enum open_type {
CREATE,
OPEN
CREATE
};
typedef block_manager<>::read_ref read_ref;
typedef block_manager<>::write_ref write_ref;
typedef boost::shared_ptr<metadata> ptr;
metadata(block_manager<>::ptr bm, open_type ot);
metadata(block_manager<>::ptr bm, open_type ot, unsigned metadata_version = 2); // Create only
metadata(block_manager<>::ptr bm);
void commit(bool clean_shutdown = true);
void setup_hint_array(size_t width);
@ -40,11 +40,12 @@ namespace caching {
mapping_array::ptr mappings_;
hint_array::ptr hints_;
persistent_data::bitset::ptr discard_bits_;
persistent_data::bitset::ptr dirty_bits_;
private:
void init_superblock();
void create_metadata(block_manager<>::ptr bm);
void create_metadata(block_manager<>::ptr bm, unsigned metadata_version);
void open_metadata(block_manager<>::ptr bm);
void commit_space_map();

23
caching/restore_emitter.cc
View File

@ -11,10 +11,11 @@ using namespace superblock_damage;
namespace {
class restorer : public emitter {
public:
restorer(metadata::ptr md, bool clean_shutdown)
restorer(metadata::ptr md, bool clean_shutdown, unsigned metadata_version)
: in_superblock_(false),
md_(md),
clean_shutdown_(clean_shutdown) {
clean_shutdown_(clean_shutdown),
metadata_version_(metadata_version) {
}
virtual void begin_superblock(std::string const &uuid,
@ -24,6 +25,7 @@ namespace {
size_t hint_width) {
superblock &sb = md_->sb_;
sb.version = metadata_version_;
strncpy((char *) sb.policy_name, policy.c_str(), sizeof(sb.policy_name));
memset(sb.policy_version, 0, sizeof(sb.policy_version)); // FIXME: should come from xml
sb.policy_hint_size = hint_width;
@ -37,6 +39,10 @@ namespace {
unmapped_value.flags_ = 0;
md_->mappings_->grow(nr_cache_blocks, unmapped_value);
if (metadata_version_ > 1)
// make everything dirty by default
md_->dirty_bits_->grow(nr_cache_blocks, true);
vector<unsigned char> hint_value(hint_width, '\0');
md_->hints_->grow(nr_cache_blocks, hint_value);
}
@ -60,8 +66,11 @@ namespace {
m.oblock_ = oblock;
m.flags_ = M_VALID;
if (dirty)
m.flags_ = m.flags_ | M_DIRTY;
if (metadata_version_ == 1) {
if (dirty)
m.flags_ = m.flags_ | M_DIRTY;
} else
md_->dirty_bits_->set(cblock, dirty);
md_->mappings_->set(cblock, m);
}
@ -98,15 +107,17 @@ namespace {
bool in_superblock_;
metadata::ptr md_;
bool clean_shutdown_;
unsigned metadata_version_;
};
}
//----------------------------------------------------------------
emitter::ptr
caching::create_restore_emitter(metadata::ptr md, bool clean_shutdown)
caching::create_restore_emitter(metadata::ptr md, unsigned metadata_version,
bool clean_shutdown)
{
return emitter::ptr(new restorer(md, clean_shutdown));
return emitter::ptr(new restorer(md, clean_shutdown, metadata_version));
}
//----------------------------------------------------------------

4
caching/restore_emitter.h
View File

@ -7,7 +7,9 @@
//----------------------------------------------------------------
namespace caching {
emitter::ptr create_restore_emitter(metadata::ptr md, bool clean_shutdown = true);
emitter::ptr create_restore_emitter(metadata::ptr md,
unsigned metadata_version,
bool clean_shutdown = true);
}
//----------------------------------------------------------------

7
caching/superblock.cc
View File

@ -45,6 +45,8 @@ namespace {
le32 write_misses;
le32 policy_version[CACHE_POLICY_VERSION_SIZE];
le64 dirty_root; // format 2 only
} __attribute__ ((packed));
struct superblock_traits {
@ -57,7 +59,7 @@ namespace {
uint32_t const SUPERBLOCK_MAGIC = 06142003;
uint32_t const VERSION_BEGIN = 1;
uint32_t const VERSION_END = 2;
uint32_t const VERSION_END = 3;
}
//----------------------------------------------------------------
@ -195,6 +197,9 @@ superblock_traits::unpack(superblock_disk const &disk, superblock &core)
core.read_misses = to_cpu<uint32_t>(disk.read_misses);
core.write_hits = to_cpu<uint32_t>(disk.write_hits);
core.write_misses = to_cpu<uint32_t>(disk.write_misses);
if (core.version >= 2)
core.dirty_root = to_cpu<uint64_t>(disk.dirty_root);
}
void

5
caching/superblock.h
View File

@ -4,6 +4,7 @@
#include "base/endian_utils.h"
#include "persistent-data/data-structures/btree.h"
#include <boost/optional.hpp>
#include <set>
//----------------------------------------------------------------
@ -11,6 +12,8 @@
namespace caching {
typedef unsigned char __u8;
unsigned const MIN_METADATA_VERSION = 1;
unsigned const MAX_METADATA_VERSION = 2;
unsigned const SPACE_MAP_ROOT_SIZE = 128;
unsigned const CACHE_POLICY_NAME_SIZE = 16;
unsigned const CACHE_POLICY_VERSION_SIZE = 3;
@ -78,6 +81,8 @@ namespace caching {
uint32_t read_misses;
uint32_t write_hits;
uint32_t write_misses;
boost::optional<uint64_t> dirty_root;
};
enum incompat_bits {