emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into space-map-checking

Browse Source
This commit is contained in:
Joe Thornber 2015-04-07 12:18:23 +01:00
commit ee200ba85b
168 changed files with 7851 additions and 2250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.gitignore vendored
View File

@ -8,19 +8,9 @@
test.data
cachegrind.*
\#*\#
core
thin_check
thin_dump
thin_restore
thin_repair
thin_rmap
thin_metadata_size
cache_check
cache_dump
cache_restore
cache_repair
cache_metadata_size
bin/pdata_tools
*.metadata
bad-metadata
@ -43,3 +33,5 @@ config.cache
config.log
config.status
configure
callgrind.*

24
CHANGES Normal file
View File

@ -0,0 +1,24 @@
v0.5
====
- thin_delta, thin_trim
v0.4
====
- All tools switch to using libaio. This gives a large performance
boost, especially to the write focused tools like thin_restore.
- Added a progress monitor to thin_restore, cache_restore and era_restore
- Added a --quiet/-q option to *_restore to turn off the progress bar
- Removed variable hint size support from cache tools. The kernel
still only supports a fixed 32bit width. This will have a side
effect of reducing the executable sizes due to less template
instatiation.
- Tools rolled into a single executable to save space.
- Fixed some bugs when walking bitsets (possibly effecting cache_dump
and cache_check).

30
Gemfile.lock
View File

@ -1,30 +1,32 @@
GEM
remote: https://rubygems.org/
specs:
aruba (0.5.3)
aruba (0.6.1)
childprocess (>= 0.3.6)
cucumber (>= 1.1.1)
rspec-expectations (>= 2.7.0)
builder (3.2.2)
childprocess (0.3.9)
childprocess (0.5.3)
ffi (~> 1.0, >= 1.0.11)
cucumber (1.3.8)
cucumber (1.3.16)
builder (>= 2.1.2)
diff-lcs (>= 1.1.3)
gherkin (~> 2.12.1)
gherkin (~> 2.12)
multi_json (>= 1.7.5, < 2.0)
multi_test (>= 0.0.2)
diff-lcs (1.2.4)
ejt_command_line (0.0.2)
ffi (1.9.0)
multi_test (>= 0.1.1)
diff-lcs (1.2.5)
ejt_command_line (0.0.4)
ffi (1.9.3)
gherkin (2.12.2)
multi_json (~> 1.3)
multi_json (1.8.2)
multi_test (0.0.2)
rspec-expectations (2.14.3)
diff-lcs (>= 1.1.3, < 2.0)
thinp_xml (0.0.12)
ejt_command_line (= 0.0.2)
multi_json (1.10.1)
multi_test (0.1.1)
rspec-expectations (3.0.4)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.0.0)
rspec-support (3.0.4)
thinp_xml (0.0.20)
ejt_command_line (>= 0.0.2)
PLATFORMS
ruby

286
Makefile.in
View File

@ -16,53 +16,60 @@
# with thin-provisioning-tools. If not, see
# <http://www.gnu.org/licenses/>.
.PHONY: all
V=@
PROGRAMS=\
cache_check \
cache_dump \
cache_restore \
cache_repair \
cache_metadata_size \
\
thin_check \
thin_dump \
thin_restore \
thin_repair \
thin_rmap \
thin_metadata_size
PROGRAMS=\
bin/pdata_tools
.PHONY: all
all: $(PROGRAMS)
SOURCE=\
base/application.cc \
base/base64.cc \
base/endian_utils.cc \
base/error_state.cc \
\
base/error_string.cc \
base/progress_monitor.cc \
base/xml_utils.cc \
block-cache/block_cache.cc \
caching/cache_check.cc \
caching/cache_dump.cc \
caching/cache_metadata_size.cc \
caching/cache_repair.cc \
caching/cache_restore.cc \
caching/hint_array.cc \
caching/superblock.cc \
caching/mapping_array.cc \
caching/metadata.cc \
caching/metadata_dump.cc \
caching/restore_emitter.cc \
caching/superblock.cc \
caching/xml_format.cc \
\
era/era_array.cc \
era/era_check.cc \
era/era_detail.cc \
era/era_dump.cc \
era/era_invalidate.cc \
era/era_restore.cc \
era/metadata.cc \
era/metadata_dump.cc \
era/restore_emitter.cc \
era/superblock.cc \
era/writeset_tree.cc \
era/xml_format.cc \
main.cc \
persistent-data/checksum.cc \
persistent-data/endian_utils.cc \
persistent-data/data-structures/bitset.cc \
persistent-data/data-structures/bloom_filter.cc \
persistent-data/data-structures/btree.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/transaction_manager.cc \
\
persistent-data/data-structures/bitset.cc \
persistent-data/data-structures/btree.cc \
\
persistent-data/space_map.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/space-maps/disk.cc \
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
\
persistent-data/space_map.cc \
persistent-data/transaction_manager.cc \
thin-provisioning/device_tree.cc \
thin-provisioning/human_readable_format.cc \
thin-provisioning/mapping_tree.cc \
@ -72,23 +79,16 @@ SOURCE=\
thin-provisioning/restore_emitter.cc \
thin-provisioning/rmap_visitor.cc \
thin-provisioning/superblock.cc \
thin-provisioning/thin_pool.cc \
thin-provisioning/xml_format.cc
PDATA_OBJECTS=$(subst .cc,.o,$(SOURCE))
CXX_PROGRAM_SOURCE=\
caching/cache_check.cc \
caching/cache_restore.cc \
\
thin-provisioning/thin_check.cc \
thin-provisioning/thin_delta.cc \
thin-provisioning/thin_dump.cc \
thin-provisioning/thin_restore.cc \
thin-provisioning/thin_metadata_size.cc \
thin-provisioning/thin_pool.cc \
thin-provisioning/thin_repair.cc \
thin-provisioning/thin_rmap.cc
C_PROGRAM_SOURCE=\
thin-provisioning/thin_metadata_size.c
thin-provisioning/thin_restore.cc \
thin-provisioning/thin_rmap.cc \
thin-provisioning/thin_trim.cc \
thin-provisioning/xml_format.cc
CC:=@CC@
CXX:=@CXX@
@ -99,18 +99,19 @@ CFLAGS+=-g -Wall -O3
CXXFLAGS+=-g -Wall -fno-strict-aliasing
CXXFLAGS+=@CXXOPTIMISE_FLAG@
CXXFLAGS+=@CXXDEBUG_FLAG@
CXXFLAGS+=@CXX_STRERROR_FLAG@
INCLUDES+=-I$(TOP_BUILDDIR) -I$(TOP_DIR) -I$(TOP_DIR)/thin-provisioning
LIBS:=-lstdc++
LIBEXPAT:=-lexpat
LIBS:=-lstdc++ -laio -lexpat
INSTALL:=@INSTALL@
PREFIX:=@prefix@
BINDIR:=$(DESTDIR)$(PREFIX)/sbin
MANPATH:=$(DESTDIR)$(MANDIR)
DATADIR:=$(DESTDIR)$(PREFIX)/share
MANPATH:=$(DATADIR)/man
vpath %.cc $(TOP_DIR)
INSTALL_DIR = $(INSTALL) -m 755 -d
INSTALL_PROGRAM = $(INSTALL) -m 755
INSTALL_PROGRAM = $(INSTALL) -m 755 -s
INSTALL_DATA = $(INSTALL) -p -m 644
ifeq ("@TESTING@", "yes")
@ -123,14 +124,6 @@ endif
.SUFFIXES: .d
%.o: %.c
@echo " [CC] $<"
$(V) $(CC) -c $(INCLUDES) $(CFLAGS) -o $@ $<
@echo " [DEP] $<"
$(V) $(CC) -MM -MT $(subst .c,.o,$<) $(INCLUDES) $(CFLAGS) $< > $*.$$$$; \
sed 's,\([^ :]*\)\.o[ :]*,\1.o \1.gmo $* : Makefile ,g' < $*.$$$$ > $*.d; \
$(RM) $*.$$$$
%.o: %.cc
@echo " [CXX] $<"
$(V) $(CXX) -c $(INCLUDES) $(CXXFLAGS) -o $@ $<
@ -141,149 +134,15 @@ endif
#----------------------------------------------------------------
lib/libpdata.a: $(PDATA_OBJECTS)
lib/libpdata.a: $(OBJECTS)
@echo " [AR] $<"
$(V)ar -rv $@ $(PDATA_OBJECTS) > /dev/null 2>&1
$(V)ar -rv $@ $(OBJECTS) > /dev/null 2>&1
bin/pdata_tools: $(OBJECTS)
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
#----------------------------------------------------------------
# Thin provisioning tools
THIN_DEBUG_SOURCE=$(SOURCE)
THIN_DUMP_SOURCE=$(SOURCE)
THIN_REPAIR_SOURCE=$(SOURCE)
THIN_RESTORE_SOURCE=$(SOURCE)
THIN_CHECK_SOURCE=\
base/error_state.cc \
persistent-data/checksum.cc \
persistent-data/endian_utils.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/data-structures/btree.cc \
persistent-data/space_map.cc \
persistent-data/space-maps/disk.cc \
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/transaction_manager.cc \
thin-provisioning/device_tree.cc \
thin-provisioning/mapping_tree.cc \
thin-provisioning/metadata.cc \
thin-provisioning/metadata_checker.cc \
thin-provisioning/superblock.cc
THIN_RMAP_SOURCE=\
persistent-data/checksum.cc \
persistent-data/endian_utils.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/data-structures/btree.cc \
persistent-data/space_map.cc \
persistent-data/space-maps/disk.cc \
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/transaction_manager.cc \
thin-provisioning/device_tree.cc \
thin-provisioning/mapping_tree.cc \
thin-provisioning/metadata.cc \
thin-provisioning/metadata_checker.cc \
thin-provisioning/rmap_visitor.cc \
thin-provisioning/superblock.cc
THIN_DEBUG_OBJECTS=$(subst .cc,.o,$(THIN_DEBUG_SOURCE))
THIN_DUMP_OBJECTS=$(subst .cc,.o,$(THIN_DUMP_SOURCE))
THIN_REPAIR_OBJECTS=$(subst .cc,.o,$(THIN_REPAIR_SOURCE))
THIN_RESTORE_OBJECTS=$(subst .cc,.o,$(THIN_RESTORE_SOURCE))
THIN_CHECK_OBJECTS=$(subst .cc,.o,$(THIN_CHECK_SOURCE))
THIN_RMAP_OBJECTS=$(subst .cc,.o,$(THIN_RMAP_SOURCE))
thin_debug: $(THIN_DEBUG_OBJECTS) thin-provisioning/thin_debug.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
thin_repair: $(THIN_REPAIR_OBJECTS) thin-provisioning/thin_repair.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
thin_dump: $(THIN_DUMP_OBJECTS) thin-provisioning/thin_dump.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
thin_restore: $(THIN_RESTORE_OBJECTS) thin-provisioning/thin_restore.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
thin_check: $(THIN_CHECK_OBJECTS) thin-provisioning/thin_check.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
thin_rmap: $(THIN_RMAP_OBJECTS) thin-provisioning/thin_rmap.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
thin_metadata_size: thin-provisioning/thin_metadata_size.o
@echo " [LD] $@"
$(V) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $+ -lm
#----------------------------------------------------------------
# Cache tools
CACHE_CHECK_SOURCE=\
base/base64.cc \
base/error_state.cc \
persistent-data/checksum.cc \
persistent-data/endian_utils.cc \
persistent-data/error_set.cc \
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/lock_tracker.cc \
persistent-data/data-structures/btree.cc \
persistent-data/data-structures/bitset.cc \
persistent-data/space_map.cc \
persistent-data/space-maps/disk.cc \
persistent-data/space-maps/recursive.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/transaction_manager.cc \
caching/hint_array.cc \
caching/superblock.cc \
caching/mapping_array.cc \
caching/metadata.cc \
caching/metadata_dump.cc \
caching/restore_emitter.cc \
caching/xml_format.cc
CACHE_CHECK_OBJECTS=$(subst .cc,.o,$(CACHE_CHECK_SOURCE))
CACHE_DUMP_SOURCE=$(SOURCE)
CACHE_DUMP_OBJECTS=$(subst .cc,.o,$(CACHE_DUMP_SOURCE))
CACHE_REPAIR_SOURCE=$(SOURCE)
CACHE_REPAIR_OBJECTS=$(subst .cc,.o,$(CACHE_REPAIR_SOURCE))
CACHE_RESTORE_SOURCE=$(SOURCE)
CACHE_RESTORE_OBJECTS=$(subst .cc,.o,$(CACHE_RESTORE_SOURCE))
cache_check: $(CACHE_CHECK_OBJECTS) caching/cache_check.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
cache_dump: $(CACHE_DUMP_OBJECTS) caching/cache_dump.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
cache_repair: $(CACHE_REPAIR_OBJECTS) caching/cache_repair.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
cache_restore: $(CACHE_RESTORE_OBJECTS) caching/cache_restore.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(LIBEXPAT)
cache_metadata_size: caching/cache_metadata_size.o
@echo " [LD] $@"
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
DEPEND_FILES=\
$(subst .cc,.d,$(SOURCE)) \
@ -302,29 +161,44 @@ clean:
distclean: clean
$(RM) config.cache config.log config.status configure.h version.h Makefile unit-tests/Makefile
install: $(PROGRAMS)
install: bin/pdata_tools
$(INSTALL_DIR) $(BINDIR)
$(INSTALL_PROGRAM) cache_check $(BINDIR)
$(INSTALL_PROGRAM) cache_dump $(BINDIR)
$(INSTALL_PROGRAM) cache_repair $(BINDIR)
$(INSTALL_PROGRAM) cache_restore $(BINDIR)
$(INSTALL_PROGRAM) thin_check $(BINDIR)
$(INSTALL_PROGRAM) thin_dump $(BINDIR)
$(INSTALL_PROGRAM) thin_repair $(BINDIR)
$(INSTALL_PROGRAM) thin_restore $(BINDIR)
$(INSTALL_PROGRAM) thin_rmap $(BINDIR)
$(INSTALL_PROGRAM) thin_metadata_size $(BINDIR)
$(INSTALL_PROGRAM) bin/pdata_tools $(BINDIR)
ln -s -f pdata_tools $(BINDIR)/cache_check
ln -s -f pdata_tools $(BINDIR)/cache_dump
ln -s -f pdata_tools $(BINDIR)/cache_metadata_size
ln -s -f pdata_tools $(BINDIR)/cache_repair
ln -s -f pdata_tools $(BINDIR)/cache_restore
ln -s -f pdata_tools $(BINDIR)/thin_check
ln -s -f pdata_tools $(BINDIR)/thin_delta
ln -s -f pdata_tools $(BINDIR)/thin_dump
ln -s -f pdata_tools $(BINDIR)/thin_repair
ln -s -f pdata_tools $(BINDIR)/thin_restore
ln -s -f pdata_tools $(BINDIR)/thin_rmap
ln -s -f pdata_tools $(BINDIR)/thin_trim
ln -s -f pdata_tools $(BINDIR)/thin_metadata_size
ln -s -f pdata_tools $(BINDIR)/era_check
ln -s -f pdata_tools $(BINDIR)/era_dump
ln -s -f pdata_tools $(BINDIR)/era_invalidate
ln -s -f pdata_tools $(BINDIR)/era_restore
$(INSTALL_DIR) $(MANPATH)/man8
$(INSTALL_DATA) man8/cache_check.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/cache_dump.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/cache_repair.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/cache_restore.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_check.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_delta.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_dump.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_repair.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_restore.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_rmap.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_trim.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_metadata_size.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/era_check.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/era_dump.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/era_invalidate.8 $(MANPATH)/man8
# $(INSTALL_DATA) man8/era_restore.8 $(MANPATH)/man8
.PHONY: install
@ -333,7 +207,7 @@ include unit-tests/Makefile
.PHONEY: features
features: $(PROGRAMS)
features: pdata_tools
cucumber --no-color --format progress
test: features unit-test

1
README.md
View File

@ -10,6 +10,7 @@ Requirements
A C++ compiler that supports the c++11 standard (eg, g++).
The [Boost C++ library](http://www.boost.org/).
The [expat](http://expat.sourceforge.net/) xml parser library (version 1).
The libaio library (note this is not the same as the aio library that you get by linking -lrt)
make, autoconf etc.
There are more requirements for testing, detailed below.

2
VERSION
View File

@ -1 +1 @@
0.2.8
0.4.1

63
base/application.cc Normal file
View File

@ -0,0 +1,63 @@
#include "base/application.h"
#include <libgen.h>
#include <linux/limits.h>
#include <string.h>
using namespace base;
using namespace std;
//----------------------------------------------------------------
int
application::run(int argc, char **argv)
{
string cmd = get_basename(argv[0]);
if (cmd == string("pdata_tools")) {
argc--;
argv++;
if (!argc) {
usage();
return 1;
}
cmd = argv[0];
}
std::list<command const *>::const_iterator it;
for (it = cmds_.begin(); it != cmds_.end(); ++it) {
if (cmd == (*it)->get_name())
return (*it)->run(argc, argv);
}
std::cerr << "Unknown command '" << cmd << "'\n";
usage();
return 1;
}
void
application::usage()
{
std::cerr << "Usage: <command> <args>\n"
<< "commands:\n";
std::list<command const *>::const_iterator it;
for (it = cmds_.begin(); it != cmds_.end(); ++it) {
std::cerr << " " << (*it)->get_name() << "\n";
}
}
std::string
application::get_basename(std::string const &path) const
{
char buffer[PATH_MAX + 1];
memset(buffer, 0, sizeof(buffer));
strncpy(buffer, path.c_str(), PATH_MAX);
return ::basename(buffer);
}
//----------------------------------------------------------------

52
base/application.h Normal file
View File

@ -0,0 +1,52 @@
#ifndef BASE_APPLICATION_H
#define BASE_APPLICATION_H
#include <iostream>
#include <list>
#include <string>
#include <stdexcept>
//----------------------------------------------------------------
namespace base {
class command {
public:
typedef int (*cmd_fn)(int, char **);
command(std::string const &name, cmd_fn fn)
: name_(name),
fn_(fn) {
}
std::string const &get_name() const {
return name_;
}
int run(int argc, char **argv) const {
return fn_(argc, argv);
}
private:
std::string name_;
cmd_fn fn_;
};
class application {
public:
void add_cmd(command const &c) {
cmds_.push_back(&c);
}
int run(int argc, char **argv);
private:
void usage();
std::string get_basename(std::string const &path) const;
std::list<command const *> cmds_;
};
}
//----------------------------------------------------------------
#endif

0
persistent-data/endian_utils.cc → base/endian_utils.cc
View File

25
persistent-data/endian_utils.h → base/endian_utils.h
View File

@ -16,8 +16,8 @@
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#ifndef ENDIAN_H
#define ENDIAN_H
#ifndef BASE_ENDIAN_H
#define BASE_ENDIAN_H
#include <endian.h>
#include <stdint.h>
@ -25,7 +25,26 @@
//----------------------------------------------------------------
// FIXME: rename to endian
/* An old glic doesn't provide these macros */
#if !defined(htole16) || !defined(le16toh) || !defined(htole32) || !defined(le32toh) || !defined(htole64) || !defined(le64toh)
#include <byteswap.h>
#if __BYTE_ORDER == __LITTLE_ENDIAN
#define htole16(x) (x)
#define le16toh(x) (x)
#define htole32(x) (x)
#define le32toh(x) (x)
#define htole64(x) (x)
#define le64toh(x) (x)
#else
#define htole16(x) __bswap_16(x)
#define le16toh(x) __bswap_16(x)
#define htole32(x) __bswap_32(x)
#define le32toh(x) __bswap_32(x)
#define htole64(x) __bswap_64(x)
#define le64toh(x) __bswap_64(x)
#endif
#endif
namespace base {
// These are just little wrapper types to make the compiler

39
base/error_string.cc Normal file
View File

@ -0,0 +1,39 @@
#include "base/error_string.h"
#include <errno.h>
#include <stdexcept>
#include <string.h>
using namespace std;
//----------------------------------------------------------------
#ifdef STRERROR_R_CHAR_P
string base::error_string(int err)
{
char *ptr;
char buffer[128];
ptr = strerror_r(errno, buffer, sizeof(buffer));
return string(ptr);
}
#else
string base::error_string(int err)
{
int r;
char buffer[128];
r = strerror_r(errno, buffer, sizeof(buffer));
if (r)
throw runtime_error("strerror_r failed");
return string(buffer);
}
#endif
//----------------------------------------------------------------

16
base/error_string.h Normal file
View File

@ -0,0 +1,16 @@
#ifndef BASE_ERROR_STRING_H
#define BASE_ERROR_STRING_H
#include <string>
//----------------------------------------------------------------
namespace base {
// There are a couple of version of strerror_r kicking around, so
// we wrap it.
std::string error_string(int err);
}
//----------------------------------------------------------------
#endif

48
base/indented_stream.h Normal file
View File

@ -0,0 +1,48 @@
#ifndef BASE_INDENTED_STREAM_H
#define BASE_INDENTED_STREAM_H
#include <iostream>
//----------------------------------------------------------------
namespace {
class indented_stream {
public:
indented_stream(std::ostream &out)
: out_(out),
indent_(0) {
}
void indent() {
for (unsigned i = 0; i < indent_ * 2; i++)
out_ << ' ';
}
void inc() {
indent_++;
}
void dec() {
indent_--;
}
template <typename T>
indented_stream &operator <<(T const &t) {
out_ << t;
return *this;
}
indented_stream &operator <<(std::ostream &(*fp)(std::ostream &)) {
out_ << fp;
return *this;
}
private:
std::ostream &out_;
unsigned indent_;
};
}
//----------------------------------------------------------------
#endif

78
base/progress_monitor.cc Normal file
View File

@ -0,0 +1,78 @@
#include "base/progress_monitor.h"
#include <iostream>
//----------------------------------------------------------------
namespace {
using namespace std;
class progress_bar : public base::progress_monitor {
public:
progress_bar(string const &title)
: title_(title),
progress_width_(50),
spinner_(0) {
update_percent(0);
}
~progress_bar() {
cout << "\n";
}
void update_percent(unsigned p) {
unsigned nr_equals = max<unsigned>(progress_width_ * p / 100, 1);
unsigned nr_spaces = progress_width_ - nr_equals;
cout << title_ << ": [";
for (unsigned i = 0; i < nr_equals - 1; i++)
cout << '=';
if (nr_equals < progress_width_)
cout << '>';
for (unsigned i = 0; i < nr_spaces; i++)
cout << ' ';
cout << "] " << spinner_char() << " " << p << "%\r" << flush;
spinner_++;
}
private:
char spinner_char() const {
char cs[] = {'|', '/', '-', '\\'};
unsigned index = spinner_ % sizeof(cs);
return cs[index];
}
std::string title_;
unsigned progress_width_;
unsigned spinner_;
};
class quiet_progress : public base::progress_monitor {
public:
void update_percent(unsigned p) {
}
};
}
//----------------------------------------------------------------
std::auto_ptr<base::progress_monitor>
base::create_progress_bar(std::string const &title)
{
return auto_ptr<progress_monitor>(new progress_bar(title));
}
std::auto_ptr<base::progress_monitor>
base::create_quiet_progress_monitor()
{
return auto_ptr<progress_monitor>(new quiet_progress());
}
//----------------------------------------------------------------

24
base/progress_monitor.h Normal file
View File

@ -0,0 +1,24 @@
#ifndef BASE_PROGRESS_MONITOR_H
#define BASE_PROGRESS_MONITOR_H
#include <boost/shared_ptr.hpp>
#include <memory>
#include <string>
//----------------------------------------------------------------
namespace base {
class progress_monitor {
public:
virtual ~progress_monitor() {}
virtual void update_percent(unsigned) = 0;
};
std::auto_ptr<progress_monitor> create_progress_bar(std::string const &title);
std::auto_ptr<progress_monitor> create_quiet_progress_monitor();
}
//----------------------------------------------------------------
#endif

86
base/xml_utils.cc Normal file
View File

@ -0,0 +1,86 @@
#include "xml_utils.h"
#include "persistent-data/file_utils.h"
#include <fstream>
#include <iostream>
using namespace xml_utils;
//----------------------------------------------------------------
void
xml_parser::parse(std::string const &backup_file, bool quiet)
{
persistent_data::check_file_exists(backup_file);
ifstream in(backup_file.c_str(), ifstream::in);
std::auto_ptr<base::progress_monitor> monitor = create_monitor(quiet);
size_t total = 0;
size_t input_length = get_file_length(backup_file);
while (!in.eof()) {
char buffer[4096];
in.read(buffer, sizeof(buffer));
size_t len = in.gcount();
int done = in.eof();
if (!XML_Parse(parser_, buffer, len, done)) {
ostringstream out;
out << "Parse error at line "
<< XML_GetCurrentLineNumber(parser_)
<< ":\n"
<< XML_ErrorString(XML_GetErrorCode(parser_))
<< endl;
throw runtime_error(out.str());
}
total += len;
monitor->update_percent(total * 100 / input_length);
}
}
size_t
xml_parser::get_file_length(string const &file) const
{
struct stat info;
int r;
r = ::stat(file.c_str(), &info);
if (r)
throw runtime_error("Couldn't stat backup path");
return info.st_size;
}
auto_ptr<base::progress_monitor>
xml_parser::create_monitor(bool quiet)
{
if (!quiet && isatty(fileno(stdout)))
return base::create_progress_bar("Restoring");
else
return base::create_quiet_progress_monitor();
}
//----------------------------------------------------------------
void
xml_utils::build_attributes(attributes &a, char const **attr)
{
while (*attr) {
char const *key = *attr;
attr++;
if (!*attr) {
ostringstream out;
out << "No value given for xml attribute: " << key;
throw runtime_error(out.str());
}
char const *value = *attr;
a.insert(make_pair(string(key), string(value)));
attr++;
}
}
//----------------------------------------------------------------

74
base/xml_utils.h Normal file
View File

@ -0,0 +1,74 @@
#ifndef BASE_XML_UTILS_H
#define BASE_XML_UTILS_H
#include <base/progress_monitor.h>
#include <boost/lexical_cast.hpp>
#include <boost/optional.hpp>
#include <expat.h>
#include <iosfwd>
#include <map>
#include <stdexcept>
using namespace std;
//----------------------------------------------------------------
namespace xml_utils {
// Simple wrapper to ensure the parser gets freed if an exception
// is thrown during parsing.
class xml_parser {
public:
xml_parser()
: parser_(XML_ParserCreate(NULL)) {
if (!parser_)
throw runtime_error("couldn't create xml parser");
}
~xml_parser() {
XML_ParserFree(parser_);
}
XML_Parser get_parser() {
return parser_;
}
void parse(std::string const &backup_file, bool quiet);
private:
size_t get_file_length(string const &file) const;
auto_ptr<base::progress_monitor> create_monitor(bool quiet);
XML_Parser parser_;
};
typedef std::map<std::string, std::string> attributes;
void build_attributes(attributes &a, char const **attr);
template <typename T>
T get_attr(attributes const &attr, string const &key) {
attributes::const_iterator it = attr.find(key);
if (it == attr.end()) {
ostringstream out;
out << "could not find attribute: " << key;
throw runtime_error(out.str());
}
return boost::lexical_cast<T>(it->second);
}
template <typename T>
boost::optional<T> get_opt_attr(attributes const &attr, string const &key) {
typedef boost::optional<T> rtype;
attributes::const_iterator it = attr.find(key);
if (it == attr.end())
return rtype();
return rtype(boost::lexical_cast<T>(it->second));
}
}
//----------------------------------------------------------------
#endif

1
bin/cache_check Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/cache_dump Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/cache_metadata_size Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/cache_repair Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/cache_restore Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/era_check Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/era_dump Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/era_invalidate Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/era_restore Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/thin_check Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/thin_delta Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/thin_dump Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/thin_metadata_size Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/thin_repair Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/thin_restore Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

1
bin/thin_rmap Symbolic link
View File

@ -0,0 +1 @@
pdata_tools

705
block-cache/block_cache.cc Normal file
View File

@ -0,0 +1,705 @@
#include "block-cache/block_cache.h"
#include <assert.h>
#include <libaio.h>
#include <errno.h>
#include <pthread.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <iostream>
#include <stdexcept>
#include <sstream>
using namespace bcache;
//----------------------------------------------------------------
// FIXME: get from linux headers
#define SECTOR_SHIFT 9
#define PAGE_SIZE 4096
#define MIN_BLOCKS 16
#define WRITEBACK_LOW_THRESHOLD_PERCENT 33
#define WRITEBACK_HIGH_THRESHOLD_PERCENT 66
//----------------------------------------------------------------
namespace {
void *alloc_aligned(size_t len, size_t alignment)
{
void *result = NULL;
int r = posix_memalign(&result, alignment, len);
if (r)
return NULL;
return result;
}
}
//----------------------------------------------------------------
int
block_cache::init_free_list(unsigned count)
{
size_t len;
block *blocks;
size_t block_size = block_size_ << SECTOR_SHIFT;
void *data;
unsigned i;
/* Allocate the block structures */
len = sizeof(block) * count;
blocks = static_cast<block *>(malloc(len));
if (!blocks)
return -ENOMEM;
blocks_memory_ = blocks;
/* Allocate the data for each block. We page align the data. */
data = alloc_aligned(count * block_size, PAGE_SIZE);
if (!data) {
free(blocks);
return -ENOMEM;
}
blocks_data_ = data;
for (i = 0; i < count; i++) {
block *b = new (blocks + i) block();
b->data_ = static_cast<unsigned char *>(data) + block_size * i;
list_add(&b->list_, &free_);
}
return 0;
}
void
block_cache::exit_free_list()
{
if (blocks_data_)
free(blocks_data_);
if (blocks_memory_) {
struct block *blocks = static_cast<block *>(blocks_memory_);
for (unsigned i = 0; i < nr_cache_blocks_; i++)
(blocks + i)->~block();
free(blocks_memory_);
}
}
block_cache::block *
block_cache::__alloc_block()
{
block *b;
if (list_empty(&free_))
return NULL;
b = list_first_entry(&free_, block, list_);
list_del(&b->list_);
return b;
}
/*----------------------------------------------------------------
* Low level IO handling
*
* We cannot have two concurrent writes on the same block.
* eg, background writeback, put with dirty, flush?
*
* To avoid this we introduce some restrictions:
*
* i) A held block can never be written back.
* ii) You cannot get a block until writeback has completed.
*
*--------------------------------------------------------------*/
/*
* This can be called from the context of the aio thread. So we have a
* separate 'top half' complete function that we know is only called by the
* main cache thread.
*/
void
block_cache::complete_io(block &b, int result)
{
b.error_ = result;
b.clear_flags(BF_IO_PENDING);
nr_io_pending_--;
if (b.error_)
list_move_tail(&b.list_, &errored_);
else {
if (b.test_flags(BF_DIRTY)) {
b.clear_flags(BF_DIRTY | BF_PREVIOUSLY_DIRTY);
nr_dirty_--;
}
list_move_tail(&b.list_, &clean_);
}
}
/*
* |b->list| should be valid (either pointing to itself, on one of the other
* lists.
*/
// FIXME: add batch issue
void
block_cache::issue_low_level(block &b, enum io_iocb_cmd opcode, const char *desc)
{
int r;
iocb *control_blocks[1];
assert(!b.test_flags(BF_IO_PENDING));
b.set_flags(BF_IO_PENDING);
nr_io_pending_++;
list_move_tail(&b.list_, &io_pending_);
b.control_block_.aio_lio_opcode = opcode;
control_blocks[0] = &b.control_block_;
r = io_submit(aio_context_, 1, control_blocks);
if (r != 1) {
complete_io(b, EIO);
std::ostringstream out;
out << "couldn't issue " << desc << " io for block " << b.index_;
if (r < 0)
out << ": io_submit failed with " << r;
else
out << ": io_submit succeeded, but queued no io";
throw std::runtime_error(out.str());
}
}
void
block_cache::issue_read(block &b)
{
assert(!b.test_flags(BF_IO_PENDING));
issue_low_level(b, IO_CMD_PREAD, "read");
}
void
block_cache::issue_write(block &b)
{
assert(!b.test_flags(BF_IO_PENDING));
b.v_->prepare(b.data_, b.index_);
issue_low_level(b, IO_CMD_PWRITE, "write");
}
void
block_cache::wait_io()
{
int r;
unsigned i;
// FIXME: use a timeout to prevent hanging
r = io_getevents(aio_context_, 1, nr_cache_blocks_, &events_[0], NULL);
if (r < 0) {
std::ostringstream out;
out << "io_getevents failed: " << r;
throw std::runtime_error(out.str());
}
for (i = 0; i < static_cast<unsigned>(r); i++) {
io_event const &e = events_[i];
block *b = container_of(e.obj, block, control_block_);
if (e.res == block_size_ << SECTOR_SHIFT)
complete_io(*b, 0);
else if (e.res < 0)
complete_io(*b, e.res);
else {
std::ostringstream out;
out << "incomplete io for block " << b->index_
<< ", e.res = " << e.res
<< ", e.res2 = " << e.res2
<< ", offset = " << b->control_block_.u.c.offset
<< ", nbytes = " << b->control_block_.u.c.nbytes;
throw std::runtime_error(out.str());
}
}
}
/*----------------------------------------------------------------
* Clean/dirty list management
*--------------------------------------------------------------*/
/*
* We're using lru lists atm, but I think it would be worth
* experimenting with a multiqueue approach.
*/
list_head *
block_cache::__categorise(block &b)
{
if (b.error_)
return &errored_;
return b.test_flags(BF_DIRTY) ? &dirty_ : &clean_;
}
void
block_cache::hit(block &b)
{
list_move_tail(&b.list_, __categorise(b));
}
/*----------------------------------------------------------------
* High level IO handling
*--------------------------------------------------------------*/
void
block_cache::wait_all()
{
while (!list_empty(&io_pending_))
wait_io();
}
void
block_cache::wait_specific(block &b)
{
while (b.test_flags(BF_IO_PENDING))
wait_io();
}
unsigned
block_cache::writeback(unsigned count)
{
block *b, *tmp;
unsigned actual = 0, dirty_length = 0;
list_for_each_entry_safe (b, tmp, &dirty_, list_) {
dirty_length++;
if (actual == count)
break;
// The block may be on the dirty list from a prior
// acquisition.
if (b->ref_count_)
continue;
issue_write(*b);
actual++;
}
return actual;
}
/*----------------------------------------------------------------
* Hash table
*---------------------------------------------------------------*/
/*
* |nr_buckets| must be a power of two.
*/
void
block_cache::hash_init(unsigned nr_buckets)
{
unsigned i;
nr_buckets_ = nr_buckets;
mask_ = nr_buckets - 1;
for (i = 0; i < nr_buckets; i++)
INIT_LIST_HEAD(&buckets_[i]);
}
unsigned
block_cache::hash(uint64_t index)
{
const unsigned BIG_PRIME = 4294967291UL;
return (((unsigned) index) * BIG_PRIME) & mask_;
}
block_cache::block *
block_cache::hash_lookup(block_address index)
{
block *b;
unsigned bucket = hash(index);
list_for_each_entry (b, &buckets_[bucket], hash_list_) {
if (b->index_ == index)
return b;
}
return NULL;
}
void
block_cache::hash_insert(block &b)
{
unsigned bucket = hash(b.index_);
list_move_tail(&b.hash_list_, &buckets_[bucket]);
}
void
block_cache::hash_remove(block &b)
{
list_del_init(&b.hash_list_);
}
/*----------------------------------------------------------------
* High level allocation
*--------------------------------------------------------------*/
void
block_cache::setup_control_block(block &b)
{
iocb *cb = &b.control_block_;
size_t block_size_bytes = block_size_ << SECTOR_SHIFT;
memset(cb, 0, sizeof(*cb));
cb->aio_fildes = fd_;
cb->u.c.buf = b.data_;
cb->u.c.offset = block_size_bytes * b.index_;
cb->u.c.nbytes = block_size_bytes;
}
block_cache::block *
block_cache::find_unused_clean_block()
{
struct block *b, *tmp;
list_for_each_entry_safe (b, tmp, &clean_, list_) {
if (b->ref_count_)
continue;
hash_remove(*b);
list_del(&b->list_);
return b;
}
return NULL;
}
block_cache::block *
block_cache::new_block(block_address index)
{
block *b;
b = __alloc_block();
if (!b) {
if (list_empty(&clean_)) {
if (list_empty(&io_pending_))
writeback(16);
wait_io();
}
b = find_unused_clean_block();
}
if (b) {
INIT_LIST_HEAD(&b->list_);
INIT_LIST_HEAD(&b->hash_list_);
b->bc_ = this;
b->ref_count_ = 0;
b->error_ = 0;
b->flags_ = 0;
b->v_ = noop_validator_;
b->index_ = index;
setup_control_block(*b);
hash_insert(*b);
}
return b;
}
/*----------------------------------------------------------------
* Block reference counting
*--------------------------------------------------------------*/
unsigned
block_cache::calc_nr_cache_blocks(size_t mem, sector_t block_size)
{
size_t space_per_block = (block_size << SECTOR_SHIFT) + sizeof(block);
unsigned r = mem / space_per_block;
return (r < MIN_BLOCKS) ? MIN_BLOCKS : r;
}
unsigned
block_cache::calc_nr_buckets(unsigned nr_blocks)
{
unsigned r = 8;
unsigned n = nr_blocks / 4;
if (n < 8)
n = 8;
while (r < n)
r <<= 1;
return r;
}
block_cache::block_cache(int fd, sector_t block_size, uint64_t on_disk_blocks, size_t mem)
: nr_locked_(0),
nr_dirty_(0),
nr_io_pending_(0),
read_hits_(0),
read_misses_(0),
write_zeroes_(0),
write_hits_(0),
write_misses_(0),
prefetches_(0),
noop_validator_(new noop_validator())
{
int r;
unsigned nr_cache_blocks = calc_nr_cache_blocks(mem, block_size);
unsigned nr_buckets = calc_nr_buckets(nr_cache_blocks);
buckets_.resize(nr_buckets);
fd_ = fd;
block_size_ = block_size;
nr_data_blocks_ = on_disk_blocks;
nr_cache_blocks_ = nr_cache_blocks;
events_.resize(nr_cache_blocks);
aio_context_ = 0; /* needed or io_setup will fail */
r = io_setup(nr_cache_blocks, &aio_context_);
if (r < 0) {
perror("io_setup failed");
throw std::runtime_error("io_setup failed");
}
hash_init(nr_buckets);
INIT_LIST_HEAD(&free_);
INIT_LIST_HEAD(&errored_);
INIT_LIST_HEAD(&dirty_);
INIT_LIST_HEAD(&clean_);
INIT_LIST_HEAD(&io_pending_);
r = init_free_list(nr_cache_blocks);
if (r)
throw std::runtime_error("couldn't allocate blocks");
}
block_cache::~block_cache()
{
assert(!nr_locked_);
flush();
wait_all();
exit_free_list();
if (aio_context_)
io_destroy(aio_context_);
::close(fd_);
#if 0
std::cerr << "\nblock cache stats\n"
<< "=================\n"
<< "prefetches:\t" << prefetches_ << "\n"
<< "read hits:\t" << read_hits_ << "\n"
<< "read misses:\t" << read_misses_ << "\n"
<< "write hits:\t" << write_hits_ << "\n"
<< "write misses:\t" << write_misses_ << "\n"
<< "write zeroes:\t" << write_zeroes_ << std::endl;
#endif
}
uint64_t
block_cache::get_nr_blocks() const
{
return nr_data_blocks_;
}
uint64_t
block_cache::get_nr_locked() const
{
return nr_locked_;
}
void
block_cache::zero_block(block &b)
{
write_zeroes_++;
memset(b.data_, 0, block_size_ << SECTOR_SHIFT);
b.mark_dirty();
}
void
block_cache::inc_hit_counter(unsigned flags)
{
if (flags & (GF_ZERO | GF_DIRTY))
write_hits_++;
else
read_hits_++;
}
void
block_cache::inc_miss_counter(unsigned flags)
{
if (flags & (GF_ZERO | GF_DIRTY))
write_misses_++;
else
read_misses_++;
}
block_cache::block *
block_cache::lookup_or_read_block(block_address index, unsigned flags,
validator::ptr v)
{
block *b = hash_lookup(index);
if (b) {
if (b->test_flags(BF_IO_PENDING)) {
inc_miss_counter(flags);
wait_specific(*b);
} else
inc_hit_counter(flags);
if (flags & GF_ZERO)
zero_block(*b);
else {
if (b->v_.get() != v.get()) {
if (b->test_flags(BF_DIRTY))
b->v_->prepare(b->data_, b->index_);
v->check(b->data_, b->index_);
}
}
b->v_ = v;
} else {
inc_miss_counter(flags);
b = new_block(index);
if (b) {
if (flags & GF_ZERO)
zero_block(*b);
else {
issue_read(*b);
wait_specific(*b);
v->check(b->data_, b->index_);
}
b->v_ = v;
}
}
return (!b || b->error_) ? NULL : b;
}
block_cache::block &
block_cache::get(block_address index, unsigned flags, validator::ptr v)
{
check_index(index);
block *b = lookup_or_read_block(index, flags, v);
if (b) {
if (b->ref_count_ && flags & (GF_DIRTY | GF_ZERO))
throw std::runtime_error("attempt to write lock block concurrently");
// FIXME: this gets called even for new blocks
hit(*b);
if (!b->ref_count_)
nr_locked_++;
b->ref_count_++;
if (flags & GF_BARRIER)
b->set_flags(BF_FLUSH);
if (flags & GF_DIRTY)
b->set_flags(BF_DIRTY);
return *b;
}
throw std::runtime_error("couldn't get block");
}
void
block_cache::preemptive_writeback()
{
unsigned nr_available = nr_cache_blocks_ - (nr_dirty_ - nr_io_pending_);
if (nr_available < (WRITEBACK_LOW_THRESHOLD_PERCENT * nr_cache_blocks_ / 100))
writeback((WRITEBACK_HIGH_THRESHOLD_PERCENT * nr_cache_blocks_ / 100) - nr_available);
}
void
block_cache::release(block_cache::block &b)
{
assert(!b.ref_count_);
nr_locked_--;
if (b.test_flags(BF_FLUSH))
flush();
if (b.test_flags(BF_DIRTY)) {
if (!b.test_flags(BF_PREVIOUSLY_DIRTY)) {
list_move_tail(&b.list_, &dirty_);
nr_dirty_++;
b.set_flags(BF_PREVIOUSLY_DIRTY);
}
if (b.test_flags(BF_FLUSH))
flush();
else
preemptive_writeback();
b.clear_flags(BF_FLUSH);
}
}
int
block_cache::flush()
{
block *b, *tmp;
list_for_each_entry_safe (b, tmp, &dirty_, list_) {
if (b->ref_count_ || b->test_flags(BF_IO_PENDING))
// The superblock may well be still locked.
continue;
issue_write(*b);
}
wait_all();
return list_empty(&errored_) ? 0 : -EIO;
}
void
block_cache::prefetch(block_address index)
{
check_index(index);
block *b = hash_lookup(index);
if (!b) {
prefetches_++;
b = new_block(index);
if (b)
issue_read(*b);
}
}
void
block_cache::check_index(block_address index) const
{
if (index >= nr_data_blocks_) {
std::ostringstream out;
out << "block out of bounds ("
<< index << " >= " << nr_data_blocks_ << ")\n";
throw std::runtime_error(out.str());
}
}
//----------------------------------------------------------------

224
block-cache/block_cache.h Normal file
View File

@ -0,0 +1,224 @@
#ifndef BLOCK_CACHE_H
#define BLOCK_CACHE_H
#include "block-cache/list.h"
#include <boost/shared_ptr.hpp>
#include <boost/noncopyable.hpp>
#include <stdexcept>
#include <libaio.h>
#include <memory>
#include <stdint.h>
#include <stdlib.h>
#include <vector>
//----------------------------------------------------------------
namespace bcache {
typedef uint64_t block_address;
typedef uint64_t sector_t;
class validator {
public:
typedef boost::shared_ptr<validator> ptr;
virtual ~validator() {}
virtual void check(void const *data, block_address location) const = 0;
virtual void prepare(void *data, block_address location) const = 0;
};
class noop_validator : public validator {
public:
void check(void const *data, block_address location) const {}
void prepare(void *data, block_address location) const {}
};
//----------------------------------------------------------------
class block_cache : private boost::noncopyable {
public:
enum block_flags {
BF_IO_PENDING = (1 << 0),
BF_DIRTY = (1 << 1),
BF_FLUSH = (1 << 2),
BF_PREVIOUSLY_DIRTY = (1 << 3)
};
class block : private boost::noncopyable {
public:
block()
: v_() {
INIT_LIST_HEAD(&list_);
}
// Do not give this class a destructor, it wont get
// called because we manage allocation ourselves.
uint64_t get_index() const {
return index_;
}
void *get_data() const {
return data_;
}
void mark_dirty() {
set_flags(BF_DIRTY);
}
void set_flags(unsigned flags) {
flags_ |= flags;
}
unsigned test_flags(unsigned flags) const {
return flags_ & flags;
}
void clear_flags(unsigned flags) {
flags_ &= ~flags;
}
void get() {
ref_count_++;
};
void put() {
if (!ref_count_)
throw std::runtime_error("bad put");
if (!--ref_count_)
bc_->release(*this);
}
private:
friend class block_cache;
block_cache *bc_;
uint64_t index_;
void *data_;
list_head list_;
list_head hash_list_;
unsigned ref_count_;
int error_;
unsigned flags_;
iocb control_block_;
validator::ptr v_;
};
//--------------------------------
block_cache(int fd, sector_t block_size,
uint64_t max_nr_blocks, size_t mem);
~block_cache();
uint64_t get_nr_blocks() const;
uint64_t get_nr_locked() const;
enum get_flags {
GF_ZERO = (1 << 0),
GF_DIRTY = (1 << 1),
GF_BARRIER = (1 << 2)
};
block_cache::block &get(block_address index, unsigned flags, validator::ptr v);
/*
* Flush can fail if an earlier write failed. You do not know which block
* failed. Make sure you build your recovery with this in mind.
*/
int flush();
void prefetch(block_address index);
private:
int init_free_list(unsigned count);
void exit_free_list();
block *__alloc_block();
void complete_io(block &b, int result);
void issue_low_level(block &b, enum io_iocb_cmd opcode, const char *desc);
void issue_read(block &b);
void issue_write(block &b);
void wait_io();
list_head *__categorise(block &b);
void hit(block &b);
void wait_all();
void wait_specific(block &b);
unsigned writeback(unsigned count);
void hash_init(unsigned nr_buckets);
unsigned hash(uint64_t index);
block *hash_lookup(block_address index);
void hash_insert(block &b);
void hash_remove(block &b);
void setup_control_block(block &b);
block *find_unused_clean_block();
block *new_block(block_address index);
void mark_dirty(block &b);
unsigned calc_nr_cache_blocks(size_t mem, sector_t block_size);
unsigned calc_nr_buckets(unsigned nr_blocks);
void zero_block(block &b);
block *lookup_or_read_block(block_address index, unsigned flags, validator::ptr v);
void preemptive_writeback();
void release(block_cache::block &block);
void check_index(block_address index) const;
void inc_hit_counter(unsigned flags);
void inc_miss_counter(unsigned flags);
//--------------------------------
int fd_;
sector_t block_size_;
uint64_t nr_data_blocks_;
uint64_t nr_cache_blocks_;
// We can't use auto_ptr or unique_ptr because the memory is allocated with malloc
void *blocks_memory_;
void *blocks_data_;
io_context_t aio_context_;
std::vector<io_event> events_;
/*
* Blocks on the free list are not initialised, apart from the
* b.data field.
*/
list_head free_;
list_head errored_;
list_head dirty_;
list_head clean_;
unsigned nr_locked_;
unsigned nr_dirty_;
unsigned nr_io_pending_;
struct list_head io_pending_;
/*
* Hash table fields.
*/
unsigned nr_buckets_;
unsigned mask_;
std::vector<list_head> buckets_;
// Stats
unsigned read_hits_;
unsigned read_misses_;
unsigned write_zeroes_;
unsigned write_hits_;
unsigned write_misses_;
unsigned prefetches_;
validator::ptr noop_validator_;
};
}
//----------------------------------------------------------------
#endif

216
block-cache/list.h Normal file
View File

@ -0,0 +1,216 @@
#ifndef LIB_BLOCK_CACHE_LIST_H
#define LIB_BLOCK_CACHE_LIST_H
#include <stddef.h>
/*----------------------------------------------------------------*/
/*
* Simple intrusive linked list code. Lifted from Linux kernel.
*/
/**
* container_of - cast a member of a structure out to the containing structure
* @ptr: the pointer to the member.
* @type: the type of the container struct this is embedded in.
* @member: the name of the member within the struct.
*
*/
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
struct list_head {
struct list_head *next, *prev;
};
static inline void INIT_LIST_HEAD(struct list_head *list)
{
list->next = list;
list->prev = list;
}
static inline void __list_add(struct list_head *new_,
struct list_head *prev,
struct list_head *next)
{
next->prev = new_;
new_->next = next;
new_->prev = prev;
prev->next = new_;
}
/**
* list_add - add a new entry
* @new_: new entry to be added
* @head: list head to add it after
*
* Insert a new entry after the specified head.
* This is good for implementing stacks.
*/
static inline void list_add(struct list_head *new_, struct list_head *head)
{
__list_add(new_, head, head->next);
}
/**
* list_add_tail - add a new entry
* @new_: new entry to be added
* @head: list head to add it before
*
* Insert a new entry before the specified head.
* This is useful for implementing queues.
*/
static inline void list_add_tail(struct list_head *new_, struct list_head *head)
{
__list_add(new_, head->prev, head);
}
/*
* Delete a list entry by making the prev/next entries
* point to each other.
*
* This is only for internal list manipulation where we know
* the prev/next entries already!
*/
static inline void __list_del(struct list_head * prev, struct list_head * next)
{
next->prev = prev;
prev->next = next;
}
/**
* list_del - deletes entry from list.
* @entry: the element to delete from the list.
* Note: list_empty() on entry does not return true after this, the entry is
* in an undefined state.
*/
static inline void __list_del_entry(struct list_head *entry)
{
__list_del(entry->prev, entry->next);
}
static inline void list_del(struct list_head *entry)
{
__list_del(entry->prev, entry->next);
entry->next = NULL;
entry->prev = NULL;
}
/**
* list_del_init - deletes entry from list and reinitialize it.
* @entry: the element to delete from the list.
*/
static inline void list_del_init(struct list_head *entry)
{
__list_del_entry(entry);
INIT_LIST_HEAD(entry);
}
/**
* list_move - delete from one list and add as another's head
* @list: the entry to move
* @head: the head that will precede our entry
*/
static inline void list_move(struct list_head *list, struct list_head *head)
{
__list_del_entry(list);
list_add(list, head);
}
/**
* list_move_tail - delete from one list and add as another's tail
* @list: the entry to move
* @head: the head that will follow our entry
*/
static inline void list_move_tail(struct list_head *list,
struct list_head *head)
{
__list_del_entry(list);
list_add_tail(list, head);
}
/**
* list_empty - tests whether a list is empty
* @head: the list to test.
*/
static inline int list_empty(const struct list_head *head)
{
return head->next == head;
}
/**
* list_entry - get the struct for this entry
* @ptr: the &struct list_head pointer.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_struct within the struct.
*/
#define list_entry(ptr, type, member) \
container_of(ptr, type, member)
/**
* list_first_entry - get the first element from a list
* @ptr: the list head to take the element from.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_struct within the struct.
*
* Note, that list is expected to be not empty.
*/
#define list_first_entry(ptr, type, member) \
list_entry((ptr)->next, type, member)
/**
* list_next_entry - get the next element in list
* @pos: the type * to cursor
* @member: the name of the list_struct within the struct.
*/
#define list_next_entry(pos, member) \
list_entry((pos)->member.next, typeof(*(pos)), member)
/**
* list_for_each - iterate over a list
* @pos: the &struct list_head to use as a loop cursor.
* @head: the head for your list.
*/
#define list_for_each(pos, head) \
for (pos = (head)->next; pos != (head); pos = pos->next)
/**
* list_for_each_safe - iterate over a list safe against removal of list entry
* @pos: the &struct list_head to use as a loop cursor.
* @n: another &struct list_head to use as temporary storage
* @head: the head for your list.
*/
#define list_for_each_safe(pos, n, head) \
for (pos = (head)->next, n = pos->next; pos != (head); \
pos = n, n = pos->next)
/**
* list_for_each_entry - iterate over list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_struct within the struct.
*/
#define list_for_each_entry(pos, head, member) \
for (pos = list_first_entry(head, typeof(*pos), member); \
&pos->member != (head); \
pos = list_next_entry(pos, member))
/**
* list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
* @pos: the type * to use as a loop cursor.
* @n: another type * to use as temporary storage
* @head: the head for your list.
* @member: the name of the list_struct within the struct.
*/
#define list_for_each_entry_safe(pos, n, head, member) \
for (pos = list_first_entry(head, typeof(*pos), member), \
n = list_next_entry(pos, member); \
&pos->member != (head); \
pos = n, n = list_next_entry(n, member))
/*----------------------------------------------------------------*/
#endif

21
caching/cache_check.cc
View File

@ -13,7 +13,9 @@
#include <unistd.h>
#include "base/error_state.h"
#include "base/error_string.h"
#include "base/nested_output.h"
#include "caching/commands.h"
#include "caching/metadata.h"
#include "persistent-data/block.h"
#include "persistent-data/file_utils.h"
@ -201,10 +203,7 @@ namespace {
int r = ::stat(path.c_str(), &info);
if (r) {
ostringstream msg;
char buffer[128], *ptr;
ptr = ::strerror_r(errno, buffer, sizeof(buffer));
msg << path << ": " << ptr;
msg << path << ": " << error_string(errno);
throw runtime_error(msg.str());
}
@ -237,7 +236,7 @@ namespace {
out << "examining mapping array" << end_message();
{
nested_output::nest _ = out.push();
mapping_array ma(tm, mapping_array::ref_counter(), sb.mapping_root, sb.cache_blocks);
mapping_array ma(*tm, mapping_array::ref_counter(), sb.mapping_root, sb.cache_blocks);
check_mapping_array(ma, mapping_rep);
}
}
@ -250,7 +249,7 @@ namespace {
out << "examining hint array" << end_message();
{
nested_output::nest _ = out.push();
hint_array ha(tm, sb.policy_hint_size, sb.hint_root, sb.cache_blocks);
hint_array ha(*tm, sb.policy_hint_size, sb.hint_root, sb.cache_blocks);
ha.check(hint_rep);
}
}
@ -264,7 +263,7 @@ namespace {
out << "examining discard bitset" << end_message();
{
nested_output::nest _ = out.push();
persistent_data::bitset discards(tm, sb.discard_root, sb.discard_nr_blocks);
persistent_data::bitset discards(*tm, sb.discard_root, sb.discard_nr_blocks);
}
}
}
@ -286,7 +285,7 @@ namespace {
throw runtime_error(msg.str());
}
block_manager<>::ptr bm = open_bm(path, block_io<>::READ_ONLY);
block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_ONLY);
err = metadata_check(bm, fs);
return err == NO_ERROR ? 0 : 1;
@ -322,14 +321,14 @@ namespace {
//----------------------------------------------------------------
int main(int argc, char **argv)
int cache_check_main(int argc, char **argv)
{
int c;
flags fs;
const char shortopts[] = "qhV";
const struct option longopts[] = {
{ "quiet", no_argument, NULL, 'q' },
{ "superblock-only", no_argument, NULL, 1 },
{ "super-block-only", no_argument, NULL, 1 },
{ "skip-mappings", no_argument, NULL, 2 },
{ "skip-hints", no_argument, NULL, 3 },
{ "skip-discards", no_argument, NULL, 4 },
@ -384,4 +383,6 @@ int main(int argc, char **argv)
return check_with_exception_handling(argv[optind], fs);
}
base::command caching::cache_check_cmd("cache_check", cache_check_main);
//----------------------------------------------------------------

7
caching/cache_dump.cc
View File

@ -4,6 +4,7 @@
#include <iostream>
#include "version.h"
#include "caching/commands.h"
#include "caching/mapping_array.h"
#include "caching/metadata.h"
#include "caching/metadata_dump.h"
@ -34,7 +35,7 @@ namespace {
int dump(string const &dev, string const &output, flags const &fs) {
try {
block_manager<>::ptr bm = open_bm(dev, block_io<>::READ_ONLY);
block_manager<>::ptr bm = open_bm(dev, block_manager<>::READ_ONLY);
metadata::ptr md(new metadata(bm, metadata::OPEN));
if (want_stdout(output)) {
@ -66,7 +67,7 @@ namespace {
//----------------------------------------------------------------
int main(int argc, char **argv)
int cache_dump_main(int argc, char **argv)
{
int c;
flags fs;
@ -114,4 +115,6 @@ int main(int argc, char **argv)
return dump(argv[optind], output, fs);
}
base::command caching::cache_dump_cmd("cache_dump", cache_dump_main);
//----------------------------------------------------------------

67
caching/cache_metadata_size.cc
View File

@ -1,5 +1,7 @@
#include "version.h"
#include "caching/commands.h"
#include <boost/lexical_cast.hpp>
#include <boost/optional.hpp>
#include <iostream>
@ -7,7 +9,6 @@
#include <stdint.h>
#include <stdexcept>
using namespace boost;
using namespace std;
//----------------------------------------------------------------
@ -16,11 +17,20 @@ namespace {
struct flags {
flags()
: max_hint_width(4) {
// Dance around some spurious compiler warnings
device_size = 0;
block_size = 0;
nr_blocks = 0;
device_size.reset();
block_size.reset();
nr_blocks.reset();
}
optional<uint64_t> device_size;
optional<uint32_t> block_size;
optional<uint64_t> nr_blocks;
boost::optional<uint64_t> device_size;
boost::optional<uint32_t> block_size;
boost::optional<uint64_t> nr_blocks;
uint32_t max_hint_width;
};
@ -58,19 +68,19 @@ namespace {
while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) {
switch (c) {
case 0:
fs.block_size = lexical_cast<uint32_t>(optarg);
fs.block_size = boost::lexical_cast<uint32_t>(optarg);
break;
case 1:
fs.device_size = lexical_cast<uint64_t>(optarg);
fs.device_size = boost::lexical_cast<uint64_t>(optarg);
break;
case 2:
fs.nr_blocks = lexical_cast<uint64_t>(optarg);
fs.nr_blocks = boost::lexical_cast<uint64_t>(optarg);
break;
case 3:
fs.max_hint_width = lexical_cast<uint32_t>(optarg);
fs.max_hint_width = boost::lexical_cast<uint32_t>(optarg);
break;
case 'h':
@ -93,44 +103,47 @@ namespace {
return CONTINUE;
}
void expand_flags(flags &fs) {
if (!fs.device_size && !fs.nr_blocks)
throw runtime_error("Please specify either --device-size and --block-size, or --nr-blocks.");
uint64_t get_nr_blocks(flags &fs) {
if (fs.device_size) {
if (!fs.block_size)
throw runtime_error("If you specify --device-size you must also give --block-size.");
if (fs.device_size && !fs.block_size)
throw runtime_error("If you specify --device-size you must also give --block-size.");
if (fs.block_size && !fs.device_size)
throw runtime_error("If you specify --block-size you must also give --device-size.");
if (fs.device_size && fs.block_size) {
uint64_t nr_blocks = *fs.device_size / *fs.block_size;
if (fs.nr_blocks) {
if (nr_blocks != *fs.nr_blocks)
throw runtime_error(
"Contradictory arguments given, --nr-blocks doesn't match the --device-size and --block-size.");
} else
fs.nr_blocks = nr_blocks;
}
return nr_blocks;
}
if (fs.block_size && !fs.device_size)
throw runtime_error("If you specify --block-size you must also give --device-size.");
if (fs.nr_blocks)
return *fs.nr_blocks;
throw runtime_error("Please specify either --device-size and --block-size, or --nr-blocks.");
}
uint64_t meg(uint64_t n) {
return n * 2048;
}
uint64_t calc_size(flags const &fs) {
uint64_t calc_size(uint64_t nr_blocks, uint32_t max_hint_width) {
uint64_t const SECTOR_SIZE = 512;
uint64_t const TRANSACTION_OVERHEAD = meg(4);
uint64_t const BYTES_PER_BLOCK = 16;
uint64_t const HINT_OVERHEAD_PER_BLOCK = 8;
uint64_t mapping_size = (*fs.nr_blocks * BYTES_PER_BLOCK) / SECTOR_SIZE;
uint64_t hint_size = (*fs.nr_blocks * (fs.max_hint_width + HINT_OVERHEAD_PER_BLOCK)) / SECTOR_SIZE;
uint64_t mapping_size = (nr_blocks * BYTES_PER_BLOCK) / SECTOR_SIZE;
uint64_t hint_size = (nr_blocks * (max_hint_width + HINT_OVERHEAD_PER_BLOCK)) / SECTOR_SIZE;
return TRANSACTION_OVERHEAD + mapping_size + hint_size;
}
}
int main(int argc, char **argv)
int cache_metadata_size_main(int argc, char **argv)
{
flags fs;
@ -143,8 +156,8 @@ int main(int argc, char **argv)
break;
}
expand_flags(fs);
cout << calc_size(fs) << " sectors" << endl;
uint64_t nr_blocks = get_nr_blocks(fs);
cout << calc_size(nr_blocks, fs.max_hint_width) << " sectors" << endl;
} catch (std::exception const &e) {
cerr << e.what();
@ -154,4 +167,6 @@ int main(int argc, char **argv)
return 0;
}
base::command caching::cache_metadata_size_cmd("cache_metadata_size", cache_metadata_size_main);
//----------------------------------------------------------------

9
caching/cache_repair.cc
View File

@ -2,6 +2,7 @@
#include <getopt.h>
#include <libgen.h>
#include "caching/commands.h"
#include "caching/metadata.h"
#include "caching/metadata_dump.h"
#include "caching/restore_emitter.h"
@ -16,12 +17,12 @@ using namespace caching;
namespace {
metadata::ptr open_metadata_for_read(string const &path) {
block_manager<>::ptr bm = open_bm(path, block_io<>::READ_ONLY);
block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_ONLY);
return metadata::ptr(new metadata(bm, metadata::OPEN));
}
emitter::ptr output_emitter(string const &path) {
block_manager<>::ptr bm = open_bm(path, block_io<>::READ_WRITE);
block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_WRITE);
metadata::ptr md(new metadata(bm, metadata::CREATE));
return create_restore_emitter(md, true);
}
@ -52,7 +53,7 @@ namespace {
//----------------------------------------------------------------
int main(int argc, char **argv)
int cache_repair_main(int argc, char **argv)
{
int c;
boost::optional<string> input_path, output_path;
@ -105,4 +106,6 @@ int main(int argc, char **argv)
return repair(*input_path, *output_path);
}
base::command caching::cache_repair_cmd("cache_repair", cache_repair_main);
//----------------------------------------------------------------

41
caching/cache_restore.cc
View File

@ -1,5 +1,6 @@
#include "version.h"
#include "caching/commands.h"
#include "caching/metadata.h"
#include "caching/restore_emitter.h"
#include "caching/xml_format.h"
@ -20,11 +21,30 @@ using namespace std;
//----------------------------------------------------------------
namespace {
size_t get_file_length(string const &file) {
struct stat info;
int r;
r = ::stat(file.c_str(), &info);
if (r)
throw runtime_error("Couldn't stat backup path");
return info.st_size;
}
auto_ptr<progress_monitor> create_monitor(bool quiet) {
if (!quiet && isatty(fileno(stdout)))
return create_progress_bar("Restoring");
else
return create_quiet_progress_monitor();
}
struct flags {
flags()
: metadata_version(1),
override_metadata_version(false),
clean_shutdown(true) {
clean_shutdown(true),
quiet(false) {
}
optional<string> input;
@ -33,11 +53,12 @@ namespace {
uint32_t metadata_version;
bool override_metadata_version;
bool clean_shutdown;
bool quiet;
};
int restore(flags const &fs) {
try {
block_manager<>::ptr bm = open_bm(*fs.output, block_io<>::READ_WRITE);
block_manager<>::ptr bm = open_bm(*fs.output, block_manager<>::READ_WRITE);
metadata::ptr md(new metadata(bm, metadata::CREATE));
emitter::ptr restorer = create_restore_emitter(md, fs.clean_shutdown);
@ -48,7 +69,9 @@ namespace {
check_file_exists(*fs.input);
ifstream in(fs.input->c_str(), ifstream::in);
parse_xml(in, restorer);
auto_ptr<progress_monitor> monitor = create_monitor(fs.quiet);
parse_xml(in, restorer, get_file_length(*fs.input), *monitor);
} catch (std::exception &e) {
cerr << e.what() << endl;
@ -64,6 +87,7 @@ namespace {
<< " {-h|--help}" << endl
<< " {-i|--input} <input xml file>" << endl
<< " {-o|--output} <output device or file>" << endl
<< " {-q|--quiet}" << endl
<< " {-V|--version}" << endl
<< endl
<< " {--debug-override-metadata-version} <integer>" << endl
@ -72,18 +96,19 @@ namespace {
}
}
int main(int argc, char **argv)
int cache_restore_main(int argc, char **argv)
{
int c;
flags fs;
char const *prog_name = basename(argv[0]);
char const *short_opts = "hi:o:V";
char const *short_opts = "hi:o:qV";
option const long_opts[] = {
{ "debug-override-metadata-version", required_argument, NULL, 0 },
{ "omit-clean-shutdown", no_argument, NULL, 1 },
{ "help", no_argument, NULL, 'h'},
{ "input", required_argument, NULL, 'i' },
{ "output", required_argument, NULL, 'o'},
{ "quiet", no_argument, NULL, 'q'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
@ -111,6 +136,10 @@ int main(int argc, char **argv)
fs.output = optional<string>(string(optarg));
break;
case 'q':
fs.quiet = true;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
@ -141,4 +170,6 @@ int main(int argc, char **argv)
return restore(fs);
}
base::command caching::cache_restore_cmd("cache_restore", cache_restore_main);
//----------------------------------------------------------------

18
caching/commands.h Normal file
View File

@ -0,0 +1,18 @@
#ifndef CACHING_COMMANDS_H
#define CACHING_COMMANDS_H
#include "base/application.h"
//----------------------------------------------------------------
namespace caching {
extern base::command cache_check_cmd;
extern base::command cache_dump_cmd;
extern base::command cache_metadata_size_cmd;
extern base::command cache_restore_cmd;
extern base::command cache_repair_cmd;
}
//----------------------------------------------------------------
#endif

67
caching/hint_array.cc
View File

@ -35,22 +35,19 @@ namespace {
// use the appropriate one.
#define all_widths \
xx(4); xx(8); xx(12); xx(16); xx(20); xx(24); xx(28); xx(32);\
xx(36); xx(40); xx(44); xx(48); xx(52); xx(56); xx(60); xx(64); \
xx(68); xx(72); xx(76); xx(80); xx(84); xx(88); xx(92); xx(96); \
xx(100); xx(104); xx(108); xx(112); xx(116); xx(120); xx(124); xx(128);
xx(4);
template <uint32_t WIDTH>
shared_ptr<array_base> mk_array(transaction_manager::ptr tm) {
boost::shared_ptr<array_base> mk_array(transaction_manager &tm) {
typedef hint_traits<WIDTH> traits;
typedef array<traits> ha;
typedef persistent_data::array<traits> ha;
shared_ptr<array_base> r = typename ha::ptr(new ha(tm, typename traits::ref_counter()));
boost::shared_ptr<array_base> r = typename ha::ptr(new ha(tm, typename traits::ref_counter()));
return r;
}
shared_ptr<array_base> mk_array(transaction_manager::ptr tm, uint32_t width) {
boost::shared_ptr<array_base> mk_array(transaction_manager &tm, uint32_t width) {
switch (width) {
#define xx(n) case n: return mk_array<n>(tm)
@ -61,15 +58,15 @@ namespace {
}
// never get here
return shared_ptr<array_base>();
return boost::shared_ptr<array_base>();
}
//--------------------------------
template <typename HA>
shared_ptr<HA>
downcast_array(shared_ptr<array_base> base) {
shared_ptr<HA> a = dynamic_pointer_cast<HA>(base);
boost::shared_ptr<HA>
downcast_array(boost::shared_ptr<array_base> base) {
boost::shared_ptr<HA> a = dynamic_pointer_cast<HA>(base);
if (!a)
throw runtime_error("internal error: couldn't cast hint array");
@ -79,16 +76,16 @@ namespace {
//--------------------------------
template <uint32_t WIDTH>
shared_ptr<array_base> mk_array(transaction_manager::ptr tm, block_address root, unsigned nr_entries) {
boost::shared_ptr<array_base> mk_array(transaction_manager &tm, block_address root, unsigned nr_entries) {
typedef hint_traits<WIDTH> traits;
typedef array<traits> ha;
typedef persistent_data::array<traits> ha;
shared_ptr<array_base> r = typename ha::ptr(new ha(tm, typename traits::ref_counter(), root, nr_entries));
boost::shared_ptr<array_base> r = typename ha::ptr(new ha(tm, typename traits::ref_counter(), root, nr_entries));
return r;
}
shared_ptr<array_base> mk_array(transaction_manager::ptr tm, uint32_t width, block_address root, unsigned nr_entries) {
boost::shared_ptr<array_base> mk_array(transaction_manager &tm, uint32_t width, block_address root, unsigned nr_entries) {
switch (width) {
#define xx(n) case n: return mk_array<n>(tm, root, nr_entries)
all_widths
@ -98,21 +95,21 @@ namespace {
}
// never get here
return shared_ptr<array_base>();
return boost::shared_ptr<array_base>();
}
//--------------------------------
template <uint32_t WIDTH>
void get_hint(shared_ptr<array_base> base, unsigned index, vector<unsigned char> &data) {
void get_hint(boost::shared_ptr<array_base> base, unsigned index, vector<unsigned char> &data) {
typedef hint_traits<WIDTH> traits;
typedef array<traits> ha;
typedef persistent_data::array<traits> ha;
shared_ptr<ha> a = downcast_array<ha>(base);
boost::shared_ptr<ha> a = downcast_array<ha>(base);
data = a->get(index);
}
void get_hint_(uint32_t width, shared_ptr<array_base> base, unsigned index, vector<unsigned char> &data) {
void get_hint_(uint32_t width, boost::shared_ptr<array_base> base, unsigned index, vector<unsigned char> &data) {
switch (width) {
#define xx(n) case n: return get_hint<n>(base, index, data)
all_widths
@ -123,15 +120,15 @@ namespace {
//--------------------------------
template <uint32_t WIDTH>
void set_hint(shared_ptr<array_base> base, unsigned index, vector<unsigned char> const &data) {
void set_hint(boost::shared_ptr<array_base> base, unsigned index, vector<unsigned char> const &data) {
typedef hint_traits<WIDTH> traits;
typedef array<traits> ha;
typedef persistent_data::array<traits> ha;
shared_ptr<ha> a = downcast_array<ha>(base);
boost::shared_ptr<ha> a = downcast_array<ha>(base);
a->set(index, data);
}
void set_hint_(uint32_t width, shared_ptr<array_base> base,
void set_hint_(uint32_t width, boost::shared_ptr<array_base> base,
unsigned index, vector<unsigned char> const &data) {
switch (width) {
#define xx(n) case n: return set_hint<n>(base, index, data)
@ -143,15 +140,15 @@ namespace {
//--------------------------------
template <uint32_t WIDTH>
void grow(shared_ptr<array_base> base, unsigned new_nr_entries, vector<unsigned char> const &value) {
void grow(boost::shared_ptr<array_base> base, unsigned new_nr_entries, vector<unsigned char> const &value) {
typedef hint_traits<WIDTH> traits;
typedef array<traits> ha;
typedef persistent_data::array<traits> ha;
shared_ptr<ha> a = downcast_array<ha>(base);
boost::shared_ptr<ha> a = downcast_array<ha>(base);
a->grow(new_nr_entries, value);
}
void grow_(uint32_t width, shared_ptr<array_base> base,
void grow_(uint32_t width, boost::shared_ptr<array_base> base,
unsigned new_nr_entries, vector<unsigned char> const &value)
{
switch (width) {
@ -197,17 +194,17 @@ namespace {
};
template <uint32_t WIDTH>
void walk_hints(shared_ptr<array_base> base, hint_visitor &hv, damage_visitor &dv) {
void walk_hints(boost::shared_ptr<array_base> base, hint_visitor &hv, damage_visitor &dv) {
typedef hint_traits<WIDTH> traits;
typedef array<traits> ha;
typedef persistent_data::array<traits> ha;
shared_ptr<ha> a = downcast_array<ha>(base);
boost::shared_ptr<ha> a = downcast_array<ha>(base);
value_adapter vv(hv);
ll_damage_visitor ll(dv);
a->visit_values(vv, ll);
}
void walk_hints_(uint32_t width, shared_ptr<array_base> base,
void walk_hints_(uint32_t width, boost::shared_ptr<array_base> base,
hint_visitor &hv, damage_visitor &dv) {
switch (width) {
#define xx(n) case n: walk_hints<n>(base, hv, dv); break
@ -233,13 +230,13 @@ missing_hints::visit(damage_visitor &v) const
//----------------------------------------------------------------
hint_array::hint_array(tm_ptr tm, unsigned width)
hint_array::hint_array(transaction_manager &tm, unsigned width)
: width_(check_width(width)),
impl_(mk_array(tm, width))
{
}
hint_array::hint_array(hint_array::tm_ptr tm, unsigned width,
hint_array::hint_array(transaction_manager &tm, unsigned width,
block_address root, unsigned nr_entries)
: width_(check_width(width)),
impl_(mk_array(tm, width, root, nr_entries))

5
caching/hint_array.h
View File

@ -56,10 +56,9 @@ namespace caching {
class hint_array {
public:
typedef boost::shared_ptr<hint_array> ptr;
typedef persistent_data::transaction_manager::ptr tm_ptr;
hint_array(tm_ptr tm, unsigned width);
hint_array(tm_ptr tm, unsigned width, block_address root, unsigned nr_entries);
hint_array(transaction_manager &tm, unsigned width);
hint_array(transaction_manager &tm, unsigned width, block_address root, unsigned nr_entries);
unsigned get_nr_entries() const;

2
caching/mapping_array.cc
View File

@ -1,5 +1,5 @@
#include "base/endian_utils.h"
#include "caching/mapping_array.h"
#include "persistent-data/endian_utils.h"
#include <set>

14
caching/metadata.cc
View File

@ -61,7 +61,7 @@ metadata::setup_hint_array(size_t width)
{
if (width > 0)
hints_ = hint_array::ptr(
new hint_array(tm_, width));
new hint_array(*tm_, width));
}
void
@ -70,16 +70,16 @@ metadata::create_metadata(block_manager<>::ptr bm)
tm_ = open_tm(bm);
space_map::ptr core = tm_->get_sm();
metadata_sm_ = create_metadata_sm(tm_, tm_->get_bm()->get_nr_blocks());
metadata_sm_ = create_metadata_sm(*tm_, tm_->get_bm()->get_nr_blocks());
copy_space_maps(metadata_sm_, core);
tm_->set_sm(metadata_sm_);
mappings_ = mapping_array::ptr(new mapping_array(tm_, mapping_array::ref_counter()));
mappings_ = mapping_array::ptr(new mapping_array(*tm_, mapping_array::ref_counter()));
// We can't instantiate the hint array yet, since we don't know the
// hint width.
discard_bits_ = persistent_data::bitset::ptr(new persistent_data::bitset(tm_));
discard_bits_ = persistent_data::bitset::ptr(new persistent_data::bitset(*tm_));
}
void
@ -89,19 +89,19 @@ metadata::open_metadata(block_manager<>::ptr bm)
sb_ = read_superblock(tm_->get_bm());
mappings_ = mapping_array::ptr(
new mapping_array(tm_,
new mapping_array(*tm_,
mapping_array::ref_counter(),
sb_.mapping_root,
sb_.cache_blocks));
if (sb_.hint_root)
hints_ = hint_array::ptr(
new hint_array(tm_, sb_.policy_hint_size,
new hint_array(*tm_, sb_.policy_hint_size,
sb_.hint_root, sb_.cache_blocks));
if (sb_.discard_root)
discard_bits_ = persistent_data::bitset::ptr(
new persistent_data::bitset(tm_, sb_.discard_root, sb_.discard_nr_blocks));
new persistent_data::bitset(*tm_, sb_.discard_root, sb_.discard_nr_blocks));
}
void

3
caching/metadata.h
View File

@ -1,10 +1,11 @@
#ifndef CACHE_METADATA_H
#define CACHE_METADATA_H
#include "base/endian_utils.h"
#include "persistent-data/block.h"
#include "persistent-data/data-structures/array.h"
#include "persistent-data/data-structures/bitset.h"
#include "persistent-data/endian_utils.h"
#include "persistent-data/space-maps/disk.h"
#include "persistent-data/transaction_manager.h"

2
caching/metadata_dump.cc
View File

@ -15,7 +15,7 @@ namespace {
void raise_metadata_damage() {
throw std::runtime_error("metadata contains errors (run cache_check for details).\n"
"perhaps you wanted to run with --repair");
"perhaps you wanted to run with --repair ?");
}
//--------------------------------

3
caching/restore_emitter.cc
View File

@ -17,9 +17,6 @@ namespace {
clean_shutdown_(clean_shutdown) {
}
virtual ~restorer() {
}
virtual void begin_superblock(std::string const &uuid,
pd::block_address block_size,
pd::block_address nr_cache_blocks,

24
caching/superblock.cc
View File

@ -275,25 +275,25 @@ namespace validator {
unsigned const SECTOR_TO_BLOCK_SHIFT = 3;
uint32_t const SUPERBLOCK_CSUM_SEED = 9031977;
struct sb_validator : public block_manager<>::validator {
virtual void check(buffer<> const &b, block_address location) const {
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(&b);
struct sb_validator : public bcache::validator {
virtual void check(void const *raw, block_address location) const {
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);
sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(sbd->csum))
throw checksum_error("bad checksum in superblock");
}
virtual void prepare(buffer<> &b, block_address location) const {
superblock_disk *sbd = reinterpret_cast<superblock_disk *>(&b);
virtual void prepare(void *raw, block_address location) const {
superblock_disk *sbd = reinterpret_cast<superblock_disk *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);
sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
sbd->csum = to_disk<base::le32>(sum.get_sum());
}
};
block_manager<>::validator::ptr mk_v() {
return block_manager<>::validator::ptr(new sb_validator);
bcache::validator::ptr mk_v() {
return bcache::validator::ptr(new sb_validator);
}
}
@ -302,9 +302,10 @@ namespace validator {
superblock
caching::read_superblock(block_manager<>::ptr bm, block_address location)
{
using namespace validator;
superblock sb;
block_manager<>::read_ref r = bm->read_lock(location, validator::mk_v());
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(&r.data());
block_manager<>::read_ref r = bm->read_lock(location, mk_v());
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(r.data());
superblock_traits::unpack(*sbd, sb);
return sb;
@ -313,8 +314,9 @@ caching::read_superblock(block_manager<>::ptr bm, block_address location)
void
caching::write_superblock(block_manager<>::ptr bm, superblock const &sb, block_address location)
{
block_manager<>::write_ref w = bm->superblock_zero(location, validator::mk_v());
superblock_traits::pack(sb, *reinterpret_cast<superblock_disk *>(w.data().raw()));
using namespace validator;
block_manager<>::write_ref w = bm->superblock_zero(location, mk_v());
superblock_traits::pack(sb, *reinterpret_cast<superblock_disk *>(w.data()));
}
void

4
caching/superblock.h
View File

@ -1,7 +1,7 @@
#ifndef CACHE_SUPERBLOCK_H
#define CACHE_SUPERBLOCK_H
#include "persistent-data/endian_utils.h"
#include "base/endian_utils.h"
#include "persistent-data/data-structures/btree.h"
#include <set>
@ -128,7 +128,7 @@ namespace caching {
//--------------------------------
persistent_data::block_manager<>::validator::ptr superblock_validator();
bcache::validator::ptr superblock_validator();
superblock read_superblock(persistent_data::block_manager<>::ptr bm,
persistent_data::block_address location = SUPERBLOCK_LOCATION);

140
caching/xml_format.cc
View File

@ -1,13 +1,15 @@
#include "base/base64.h"
#include "caching/xml_format.h"
#include <boost/lexical_cast.hpp>
#include <expat.h>
#include "base/base64.h"
#include "base/indented_stream.h"
#include "base/xml_utils.h"
#include <boost/lexical_cast.hpp>
using namespace boost;
using namespace caching;
using namespace persistent_data;
using namespace std;
using namespace xml_utils;
//----------------------------------------------------------------
@ -18,8 +20,7 @@ namespace {
class xml_emitter : public emitter {
public:
xml_emitter(ostream &out)
: out_(out),
indent_(0) {
: out_(out) {
}
void begin_superblock(std::string const &uuid,
@ -27,37 +28,37 @@ namespace {
block_address nr_cache_blocks,
std::string const &policy,
size_t hint_width) {
indent();
out_.indent();
out_ << "<superblock uuid=\"" << uuid << "\""
<< " block_size=\"" << block_size << "\""
<< " nr_cache_blocks=\"" << nr_cache_blocks << "\""
<< " policy=\"" << policy << "\""
<< " hint_width=\"" << hint_width << "\">" << endl;
inc();
out_.inc();
}
virtual void end_superblock() {
dec();
indent();
out_.dec();
out_.indent();
out_ << "</superblock>" << endl;
}
virtual void begin_mappings() {
indent();
out_.indent();
out_ << "<mappings>" << endl;
inc();
out_.inc();
}
virtual void end_mappings() {
dec();
indent();
out_.dec();
out_.indent();
out_ << "</mappings>" << endl;
}
virtual void mapping(block_address cblock,
block_address oblock,
bool dirty) {
indent();
out_.indent();
out_ << "<mapping"
<< " cache_block=\"" << cblock << "\""
<< " origin_block=\"" << oblock << "\""
@ -66,14 +67,14 @@ namespace {
}
virtual void begin_hints() {
indent();
out_.indent();
out_ << "<hints>" << endl;
inc();
out_.inc();
}
virtual void end_hints() {
dec();
indent();
out_.dec();
out_.indent();
out_ << "</hints>" << endl;
}
@ -81,7 +82,7 @@ namespace {
vector<unsigned char> const &data) {
using namespace base;
indent();
out_.indent();
out_ << "<hint"
<< " cache_block=\"" << cblock << "\""
<< " data=\"" << base64_encode(data) << "\""
@ -89,19 +90,19 @@ namespace {
}
virtual void begin_discards() {
indent();
out_.indent();
out_ << "<discards>" << endl;
inc();
out_.inc();
}
virtual void end_discards() {
dec();
indent();
out_.dec();
out_.indent();
out_ << "</discards>" << endl;
}
virtual void discard(block_address dblock_b, block_address dblock_e) {
indent();
out_.indent();
out_ << "<discard dbegin=\"" << dblock_b << "\""
<< " dend=\"" << dblock_e << "\"/>" << endl;
}
@ -111,70 +112,12 @@ namespace {
return v ? "true" : "false";
}
// FIXME: factor out a common class with the thin_provisioning emitter
void indent() {
for (unsigned i = 0; i < indent_ * 2; i++)
out_ << ' ';
}
void inc() {
indent_++;
}
void dec() {
indent_--;
}
ostream &out_;
unsigned indent_;
indented_stream out_;
};
//--------------------------------
// Parser
//--------------------------------
// FIXME: factor out common code with thinp one
typedef std::map<string, string> attributes;
void build_attributes(attributes &a, char const **attr) {
while (*attr) {
char const *key = *attr;
attr++;
if (!*attr) {
ostringstream out;
out << "No value given for xml attribute: " << key;
throw runtime_error(out.str());
}
char const *value = *attr;
a.insert(make_pair(string(key), string(value)));
attr++;
}
}
template <typename T>
T get_attr(attributes const &attr, string const &key) {
attributes::const_iterator it = attr.find(key);
if (it == attr.end()) {
ostringstream out;
out << "could not find attribute: " << key;
throw runtime_error(out.str());
}
return boost::lexical_cast<T>(it->second);
}
template <typename T>
boost::optional<T> get_opt_attr(attributes const &attr, string const &key) {
typedef boost::optional<T> rtype;
attributes::const_iterator it = attr.find(key);
if (it == attr.end())
return rtype();
return rtype(boost::lexical_cast<T>(it->second));
}
void parse_superblock(emitter *e, attributes const &attr) {
e->begin_superblock(get_attr<string>(attr, "uuid"),
get_attr<uint64_t>(attr, "block_size"),
@ -204,14 +147,14 @@ namespace {
block_address cblock = get_attr<uint64_t>(attr, "cache_block");
decoded_or_error doe = base64_decode(get_attr<string>(attr, "data"));
if (!get<vector<unsigned char> >(&doe)) {
if (!boost::get<vector<unsigned char> >(&doe)) {
ostringstream msg;
msg << "invalid base64 encoding of hint for cache block "
<< cblock << ": " << get<string>(doe);
<< cblock << ": " << boost::get<string>(doe);
throw runtime_error(msg.str());
}
e->hint(cblock, get<vector<unsigned char> >(doe));
e->hint(cblock, boost::get<vector<unsigned char> >(doe));
}
// FIXME: why passing e by ptr?
@ -293,14 +236,15 @@ caching::create_xml_emitter(ostream &out)
}
void
caching::parse_xml(istream &in, emitter::ptr e)
caching::parse_xml(istream &in, emitter::ptr e,
size_t input_length, base::progress_monitor &monitor)
{
XML_Parser parser = XML_ParserCreate(NULL);
if (!parser)
throw runtime_error("couldn't create xml parser");
xml_parser p;
XML_SetUserData(parser, e.get());
XML_SetElementHandler(parser, start_tag, end_tag);
XML_SetUserData(p.get_parser(), e.get());
XML_SetElementHandler(p.get_parser(), start_tag, end_tag);
size_t total = 0;
while (!in.eof()) {
char buffer[4096];
@ -308,17 +252,19 @@ caching::parse_xml(istream &in, emitter::ptr e)
size_t len = in.gcount();
int done = in.eof();
if (!XML_Parse(parser, buffer, len, done)) {
if (!XML_Parse(p.get_parser(), buffer, len, done)) {
ostringstream out;
out << "Parse error at line "
<< XML_GetCurrentLineNumber(parser)
<< XML_GetCurrentLineNumber(p.get_parser())
<< ":\n"
<< XML_ErrorString(XML_GetErrorCode(parser))
<< XML_ErrorString(XML_GetErrorCode(p.get_parser()))
<< endl;
throw runtime_error(out.str());
}
}
total += len;
monitor.update_percent(total * 100 / input_length);
}
}
//----------------------------------------------------------------

4
caching/xml_format.h
View File

@ -1,6 +1,7 @@
#ifndef CACHE_XML_FORMAT_H
#define CACHE_XML_FORMAT_H
#include "base/progress_monitor.h"
#include "emitter.h"
#include <iosfwd>
@ -9,7 +10,8 @@
namespace caching {
emitter::ptr create_xml_emitter(std::ostream &out);
void parse_xml(std::istream &in, emitter::ptr e);
void parse_xml(std::istream &in, emitter::ptr e,
size_t input_len, base::progress_monitor &monitor);
}
//----------------------------------------------------------------

11
configure.in → configure.ac
View File

@ -42,16 +42,22 @@ AC_PROG_MAKE_SET
AC_PROG_MKDIR_P
AC_PROG_INSTALL
################################################################
dnl -- Checks for functions.
AC_FUNC_STRERROR_R
if test x$ac_cv_func_strerror_r_char_p = xyes; then
CXX_STRERROR_FLAG="-DSTRERROR_R_CHAR_P"
fi
################################################################################
dnl -- Prefix is /usr by default, the exec_prefix default is setup later
AC_PREFIX_DEFAULT(/usr)
AC_CHECK_HEADERS([expat.h \
iostream \
libaio.h \
boost/bind.hpp \
boost/crc.hpp \
boost/intrusive/circular_list_algorithms.hpp \
boost/intrusive/rbtree_algorithms.hpp \
boost/lexical_cast.hpp \
boost/noncopyable.hpp \
boost/optional.hpp \
@ -137,6 +143,7 @@ VERSION_PATCHLEVEL=`echo "$VER" | $AWK -F '[[(.]]' '{print $3}'`
################################################################
AC_SUBST(CXXDEBUG_FLAG)
AC_SUBST(CXXOPTIMISE_FLAG)
AC_SUBST(CXX_STRERROR_FLAG)
AC_SUBST(INSTALL)
AC_SUBST(prefix)
AC_SUBST(RELEASE_DATE)

17
era/commands.h Normal file
View File

@ -0,0 +1,17 @@
#ifndef ERA_COMMANDS_H
#define ERA_COMMANDS_H
#include "base/application.h"
//----------------------------------------------------------------
namespace era {
extern base::command era_check_cmd;
extern base::command era_dump_cmd;
extern base::command era_invalidate_cmd;
extern base::command era_restore_cmd;
}
//----------------------------------------------------------------
#endif

35
era/emitter.h Normal file
View File

@ -0,0 +1,35 @@
#ifndef ERA_EMITTER_H
#define ERA_EMITTER_H
#include "persistent-data/block.h"
//----------------------------------------------------------------
namespace era {
namespace pd = persistent_data;
class emitter {
public:
typedef boost::shared_ptr<emitter> ptr;
virtual ~emitter() {}
virtual void begin_superblock(std::string const &uuid,
uint32_t data_block_size,
pd::block_address nr_blocks,
uint32_t current_era) = 0;
virtual void end_superblock() = 0;
virtual void begin_writeset(uint32_t era, uint32_t nr_bits) = 0;
virtual void writeset_bit(uint32_t bit, bool value) = 0;
virtual void end_writeset() = 0;
virtual void begin_era_array() = 0;
virtual void era(pd::block_address block, uint32_t era) = 0;
virtual void end_era_array() = 0;
};
}
//----------------------------------------------------------------
#endif

87
era/era_array.cc Normal file
View File

@ -0,0 +1,87 @@
#include "era/era_array.h"
using namespace era;
using namespace era_array_detail;
using namespace std;
//----------------------------------------------------------------
missing_eras::missing_eras(string const &desc, run<uint32_t> const &eras)
: damage(desc),
eras_(eras)
{
}
void
missing_eras::visit(damage_visitor &v) const
{
v.visit(*this);
}
invalid_era::invalid_era(string const &desc, block_address block, uint32_t era)
: damage(desc),
block_(block),
era_(era)
{
}
void
invalid_era::visit(damage_visitor &v) const
{
v.visit(*this);
}
//----------------------------------------------------------------
namespace {
class check_era_visitor : public era_array_visitor {
public:
check_era_visitor(damage_visitor &visitor, uint32_t current_era)
: visitor_(visitor),
current_era_(current_era) {
}
virtual void visit(uint32_t cblock, uint32_t era) {
if (era > current_era_)
visitor_.visit(invalid_era("era too great", cblock, era));
}
private:
damage_visitor &visitor_;
uint32_t current_era_;
};
class ll_damage_visitor {
public:
ll_damage_visitor(damage_visitor &v)
: v_(v) {
}
virtual void visit(array_detail::damage const &d) {
v_.visit(missing_eras(d.desc_, d.lost_keys_));
}
private:
damage_visitor &v_;
};
}
void
era::walk_era_array(era_array const &array,
era_array_visitor &ev,
era_array_detail::damage_visitor &dv)
{
ll_damage_visitor ll(dv);
array.visit_values(ev, ll);
}
void
era::check_era_array(era_array const &array,
uint32_t current_era,
era_array_detail::damage_visitor &dv)
{
check_era_visitor cv(dv, current_era);
walk_era_array(array, cv, dv);
}
//----------------------------------------------------------------

78
era/era_array.h Normal file
View File

@ -0,0 +1,78 @@
#ifndef ERA_ARRAY_H
#define ERA_ARRAY_H
#include "persistent-data/data-structures/array.h"
#include "persistent-data/data-structures/simple_traits.h"
//----------------------------------------------------------------
namespace era {
namespace era_array_detail {
class damage_visitor;
class damage {
public:
damage(std::string const &desc)
: desc_(desc) {
}
virtual ~damage() {}
virtual void visit(damage_visitor &v) const = 0;
std::string get_desc() const {
return desc_;
}
private:
std::string desc_;
};
struct missing_eras : public damage {
missing_eras(std::string const &desc, run<uint32_t> const &eras);
virtual void visit(damage_visitor &v) const;
run<uint32_t> eras_;
};
struct invalid_era : public damage {
invalid_era(std::string const &desc, block_address block, uint32_t era);
virtual void visit(damage_visitor &v) const;
block_address block_;
uint32_t era_;
};
class damage_visitor {
public:
virtual ~damage_visitor() {}
void visit(era_array_detail::damage const &d) {
d.visit(*this);
}
virtual void visit(missing_eras const &d) = 0;
virtual void visit(invalid_era const &d) = 0;
};
}
typedef persistent_data::array<uint32_traits> era_array;
class era_array_visitor {
public:
virtual ~era_array_visitor() {}
virtual void visit(uint32_t index, uint32_t era) = 0;
};
void walk_era_array(era_array const &array,
era_array_visitor &ev,
era_array_detail::damage_visitor &dv);
void check_era_array(era_array const &array,
uint32_t current_era,
era_array_detail::damage_visitor &dv);
}
//----------------------------------------------------------------
#endif

326
era/era_check.cc Normal file
View File

@ -0,0 +1,326 @@
#include <iostream>
#include <sstream>
#include <string>
#include <stdexcept>
#include <errno.h>
#include <fcntl.h>
#include <getopt.h>
#include <libgen.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "base/error_state.h"
#include "base/error_string.h"
#include "base/nested_output.h"
#include "era/commands.h"
#include "era/writeset_tree.h"
#include "era/era_array.h"
#include "era/superblock.h"
#include "persistent-data/block.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/space_map.h"
#include "persistent-data/space-maps/core.h"
#include "persistent-data/transaction_manager.h"
#include "version.h"
using namespace base;
using namespace boost;
using namespace era;
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
namespace {
class reporter_base {
public:
reporter_base(nested_output &o)
: out_(o),
err_(NO_ERROR) {
}
virtual ~reporter_base() {}
nested_output &out() {
return out_;
}
nested_output::nest push() {
return out_.push();
}
base::error_state get_error() const {
return err_;
}
void mplus_error(error_state err) {
err_ = combine_errors(err_, err);
}
private:
nested_output &out_;
error_state err_;
};
class superblock_reporter : public superblock_damage::damage_visitor, reporter_base {
public:
superblock_reporter(nested_output &o)
: reporter_base(o) {
}
virtual void visit(superblock_damage::superblock_corrupt const &d) {
out() << "superblock is corrupt" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
}
mplus_error(FATAL);
}
virtual void visit(superblock_damage::superblock_invalid const &d) {
out() << "superblock is invalid" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
}
mplus_error(FATAL);
}
using reporter_base::get_error;
};
class writeset_tree_reporter : public writeset_tree_detail::damage_visitor, reporter_base {
public:
writeset_tree_reporter(nested_output &o)
: reporter_base(o) {
}
void visit(writeset_tree_detail::missing_eras const &d) {
out() << "missing eras from writeset tree" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
out() << "Effected eras: [" << d.eras_.begin_.get()
<< ", " << d.eras_.end_.get() << ")" << end_message();
}
mplus_error(FATAL);
}
void visit(writeset_tree_detail::damaged_writeset const &d) {
out() << "damaged writeset" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
out() << "Era: " << d.era_ << end_message();
out() << "Missing bits: [" << d.missing_bits_.begin_.get()
<< ", " << d.missing_bits_.end_.get() << ")" << end_message();
}
mplus_error(FATAL);
}
using reporter_base::get_error;
};
class era_array_reporter : public era_array_detail::damage_visitor, reporter_base {
public:
era_array_reporter(nested_output &o)
: reporter_base(o) {
}
void visit(era_array_detail::missing_eras const &d) {
out() << "missing eras from era array" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
out() << "Effected eras: [" << d.eras_.begin_.get()
<< ", " << d.eras_.end_.get() << ")" << end_message();
}
mplus_error(FATAL);
}
void visit(era_array_detail::invalid_era const &d) {
out() << "invalid era in era array" << end_message();
{
nested_output::nest _ = push();
out() << d.get_desc() << end_message();
out() << "block: " << d.block_ << ", era: " << d.era_ << end_message();
}
mplus_error(FATAL);
}
using reporter_base::get_error;
};
//--------------------------------
transaction_manager::ptr open_tm(block_manager<>::ptr bm) {
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
sm->inc(SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
}
//--------------------------------
struct flags {
flags()
: superblock_only_(false),
quiet_(false) {
}
bool superblock_only_;
bool quiet_;
};
struct stat guarded_stat(string const &path) {
struct stat info;
int r = ::stat(path.c_str(), &info);
if (r) {
ostringstream msg;
msg << path << ": " << error_string(errno);;
throw runtime_error(msg.str());
}
return info;
}
error_state metadata_check(block_manager<>::ptr bm, flags const &fs) {
nested_output out(cerr, 2);
if (fs.quiet_)
out.disable();
superblock_reporter sb_rep(out);
out << "examining superblock" << end_message();
{
nested_output::nest _ = out.push();
check_superblock(bm, bm->get_nr_blocks(), sb_rep);
}
if (sb_rep.get_error() == FATAL)
return FATAL;
superblock sb = read_superblock(bm);
transaction_manager::ptr tm = open_tm(bm);
writeset_tree_reporter wt_rep(out);
{
era_detail_traits::ref_counter rc(tm);
writeset_tree wt(*tm, sb.writeset_tree_root, rc);
check_writeset_tree(tm, wt, wt_rep);
}
era_array_reporter ea_rep(out);
{
uint32_traits::ref_counter rc;
era_array ea(*tm, rc, sb.era_array_root, sb.nr_blocks);
check_era_array(ea, sb.current_era, ea_rep);
}
return combine_errors(sb_rep.get_error(),
combine_errors(wt_rep.get_error(),
ea_rep.get_error()));
}
int check(string const &path, flags const &fs) {
error_state err;
struct stat info = guarded_stat(path);
if (!S_ISREG(info.st_mode) && !S_ISBLK(info.st_mode)) {
ostringstream msg;
msg << path << ": " << "Not a block device or regular file";
throw runtime_error(msg.str());
}
block_manager<>::ptr bm = open_bm(path, block_manager<>::READ_ONLY);
err = metadata_check(bm, fs);
return err == NO_ERROR ? 0 : 1;
}
int check_with_exception_handling(string const &path, flags const &fs) {
int r;
try {
r = check(path, fs);
} catch (std::exception &e) {
if (!fs.quiet_)
cerr << e.what() << endl;
r = 1;
}
return r;
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-q|--quiet}" << endl
<< " {-h|--help}" << endl
<< " {-V|--version}" << endl
<< " {--super-block-only}" << endl;
}
}
//----------------------------------------------------------------
int era_check_main(int argc, char **argv)
{
int c;
flags fs;
const char shortopts[] = "qhV";
const struct option longopts[] = {
{ "quiet", no_argument, NULL, 'q' },
{ "super-block-only", no_argument, NULL, 1 },
{ "help", no_argument, NULL, 'h' },
{ "version", no_argument, NULL, 'V' },
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 1:
fs.superblock_only_ = true;
break;
case 'h':
usage(cout, basename(argv[0]));
return 0;
case 'q':
fs.quiet_ = true;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr, basename(argv[0]));
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr, basename(argv[0]));
return 1;
}
return check_with_exception_handling(argv[optind], fs);
}
base::command era::era_check_cmd("era_check", era_check_main);
//----------------------------------------------------------------

24
era/era_detail.cc Normal file
View File

@ -0,0 +1,24 @@
#include "era/era_detail.h"
#include <stdexcept>
using namespace base;
using namespace era;
//----------------------------------------------------------------
void
era_detail_traits::unpack(disk_type const &disk, value_type &value)
{
value.nr_bits = to_cpu<uint32_t>(disk.nr_bits);
value.writeset_root = to_cpu<uint64_t>(disk.writeset_root);
}
void
era_detail_traits::pack(value_type const &value, disk_type &disk)
{
disk.nr_bits = to_disk<le32>(value.nr_bits);
disk.writeset_root = to_disk<le64>(value.writeset_root);
}
//----------------------------------------------------------------

55
era/era_detail.h Normal file
View File

@ -0,0 +1,55 @@
#ifndef ERA_DETAIL_H
#define ERA_DETAIL_H
#include "base/endian_utils.h"
#include "persistent-data/transaction_manager.h"
//----------------------------------------------------------------
namespace era {
struct era_detail_disk {
base::le32 nr_bits;
base::le64 writeset_root;
} __attribute__ ((packed));
struct era_detail {
era_detail()
: nr_bits(0),
writeset_root(0) {
}
uint32_t nr_bits;
uint64_t writeset_root;
};
struct era_detail_ref_counter {
era_detail_ref_counter(persistent_data::transaction_manager::ptr tm)
: tm_(tm) {
}
void inc(era_detail const &d) {
tm_->get_sm()->inc(d.writeset_root);
}
void dec(persistent_data::block_address b) {
// I don't think we ever do this in the tools
throw std::runtime_error("not implemented");
}
private:
persistent_data::transaction_manager::ptr tm_;
};
struct era_detail_traits {
typedef era_detail_disk disk_type;
typedef era_detail value_type;
typedef era_detail_ref_counter ref_counter;
static void unpack(disk_type const &disk, value_type &value);
static void pack(value_type const &value, disk_type &disk);
};
}
//----------------------------------------------------------------
#endif

129
era/era_dump.cc Normal file
View File

@ -0,0 +1,129 @@
#include <fstream>
#include <getopt.h>
#include <libgen.h>
#include <iostream>
#include "version.h"
#include "era/commands.h"
#include "era/era_array.h"
#include "era/writeset_tree.h"
#include "era/metadata.h"
#include "era/metadata_dump.h"
#include "era/xml_format.h"
#include "persistent-data/file_utils.h"
using namespace era;
using namespace std;
//----------------------------------------------------------------
namespace {
struct flags {
flags()
: repair_(false),
logical_(false) {
}
bool repair_;
bool logical_;
};
//--------------------------------
string const STDOUT_PATH("-");
bool want_stdout(string const &output) {
return output == STDOUT_PATH;
}
int dump(string const &dev, string const &output, flags const &fs) {
try {
block_manager<>::ptr bm = open_bm(dev, block_manager<>::READ_ONLY);
metadata::ptr md(new metadata(bm, metadata::OPEN));
if (want_stdout(output)) {
emitter::ptr e = create_xml_emitter(cout);
metadata_dump(md, e, fs.repair_, fs.logical_);
} else {
ofstream out(output.c_str());
emitter::ptr e = create_xml_emitter(out);
metadata_dump(md, e, fs.repair_, fs.logical_);
}
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options] {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-o <xml file>}" << endl
<< " {-V|--version}" << endl
<< " {--repair}" << endl
<< " {--logical}" << endl;
}
}
//----------------------------------------------------------------
int era_dump_main(int argc, char **argv)
{
int c;
flags fs;
string output("-");
char const shortopts[] = "ho:V";
option const longopts[] = {
{ "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' },
{ "version", no_argument, NULL, 'V' },
{ "repair", no_argument, NULL, 1 },
{ "logical", no_argument, NULL, 2 },
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 1:
fs.repair_ = true;
break;
case 2:
fs.logical_ = true;
break;
case 'h':
usage(cout, basename(argv[0]));
return 0;
case 'o':
output = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr, basename(argv[0]));
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr, basename(argv[0]));
return 1;
}
return dump(argv[optind], output, fs);
}
base::command era::era_dump_cmd("era_dump", era_dump_main);
//----------------------------------------------------------------

259
era/era_invalidate.cc Normal file
View File

@ -0,0 +1,259 @@
#include <fstream>
#include <getopt.h>
#include <libgen.h>
#include <iostream>
#include "version.h"
#include "base/indented_stream.h"
#include "era/commands.h"
#include "era/era_array.h"
#include "era/writeset_tree.h"
#include "era/metadata.h"
#include "era/xml_format.h"
#include "persistent-data/file_utils.h"
#include <boost/lexical_cast.hpp>
using namespace boost;
using namespace era;
using namespace std;
//----------------------------------------------------------------
namespace {
struct flags {
flags()
: metadata_snapshot_(false) {
}
bool metadata_snapshot_;
optional<uint32_t> era_threshold_;
};
//--------------------------------
void walk_array(era_array const &array, uint32_t nr_blocks,
uint32_t threshold, set<uint32_t> &blocks) {
for (uint32_t b = 0; b < nr_blocks; b++) {
uint32_t era = array.get(b);
if (era >= threshold)
blocks.insert(b);
}
}
class writesets_marked_since : public writeset_tree_detail::writeset_visitor {
public:
writesets_marked_since(uint32_t threshold, set<uint32_t> &blocks)
: current_era_(0),
threshold_(threshold),
blocks_(blocks) {
}
void writeset_begin(uint32_t era, uint32_t nr_bits) {
current_era_ = era;
}
void bit(uint32_t index, bool value) {
if (value && current_era_ >= threshold_)
blocks_.insert(index);
}
void writeset_end() {
}
private:
uint32_t current_era_;
uint32_t threshold_;
set<uint32_t> &blocks_;
};
void raise_metadata_damage() {
throw std::runtime_error("metadata contains errors (run era_check for details).");
}
struct fatal_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
void visit(writeset_tree_detail::missing_eras const &d) {
raise_metadata_damage();
}
void visit(writeset_tree_detail::damaged_writeset const &d) {
raise_metadata_damage();
}
};
void walk_writesets(metadata const &md, uint32_t threshold, set<uint32_t> &result) {
writesets_marked_since v(threshold, result);
fatal_writeset_tree_damage dv;
walk_writeset_tree(md.tm_, *md.writeset_tree_, v, dv);
}
void mark_blocks_since(metadata const &md, optional<uint32_t> const &threshold, set<uint32_t> &result) {
if (!threshold)
// Can't get here, just putting in to pacify the compiler
throw std::runtime_error("threshold not set");
else {
walk_array(*md.era_array_, md.sb_.nr_blocks, *threshold, result);
walk_writesets(md, *threshold, result);
}
}
//--------------------------------
template <typename Iterator>
pair<uint32_t, uint32_t> next_run(Iterator &it, Iterator end) {
uint32_t b, e;
b = *it++;
e = b + 1;
while (it != end && *it == e) {
e++;
it++;
}
return make_pair(b, e);
}
void emit_blocks(ostream &out, set<uint32_t> const &blocks) {
indented_stream o(out);
o.indent();
o << "<blocks>" << endl;
o.inc();
{
set<uint32_t>::const_iterator it = blocks.begin();
while (it != blocks.end()) {
o.indent();
pair<uint32_t, uint32_t> range = next_run(it, blocks.end());
if (range.second - range.first == 1)
o << "<block block=\"" << range.first << "\"/>" << endl;
else
o << "<range begin=\"" << range.first
<< "\" end = \"" << range.second << "\"/>" << endl;
}
}
o.dec();
o.indent();
o << "</blocks>" << endl;
}
//--------------------------------
string const STDOUT_PATH("-");
bool want_stdout(string const &output) {
return output == STDOUT_PATH;
}
int invalidate(string const &dev, string const &output, flags const &fs) {
try {
set<uint32_t> blocks;
block_manager<>::ptr bm = open_bm(dev, block_manager<>::READ_ONLY);
if (fs.metadata_snapshot_) {
superblock sb = read_superblock(bm);
if (!sb.metadata_snap)
throw runtime_error("no metadata snapshot taken.");
metadata::ptr md(new metadata(bm, *sb.metadata_snap));
mark_blocks_since(*md, fs.era_threshold_, blocks);
} else {
metadata::ptr md(new metadata(bm, metadata::OPEN));
mark_blocks_since(*md, fs.era_threshold_, blocks);
}
if (want_stdout(output))
emit_blocks(cout, blocks);
else {
ofstream out(output.c_str());
emit_blocks(out, blocks);
}
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options] --written-since <era> {device|file}\n"
<< "Options:\n"
<< " {-h|--help}\n"
<< " {-o <xml file>}\n"
<< " {--metadata-snapshot}\n"
<< " {-V|--version}" << endl;
}
}
//----------------------------------------------------------------
int era_invalidate_main(int argc, char **argv)
{
int c;
flags fs;
string output("-");
char const shortopts[] = "ho:V";
option const longopts[] = {
{ "help", no_argument, NULL, 'h' },
{ "output", required_argument, NULL, 'o' },
{ "version", no_argument, NULL, 'V' },
{ "metadata-snapshot", no_argument, NULL, 1},
{ "written-since", required_argument, NULL, 2},
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 1:
fs.metadata_snapshot_ = true;
break;
case 2:
fs.era_threshold_ = lexical_cast<uint32_t>(optarg);
break;
case 'h':
usage(cout, basename(argv[0]));
return 0;
case 'o':
output = optarg;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr, basename(argv[0]));
return 1;
}
}
if (argc == optind) {
cerr << "No input file provided." << endl;
usage(cerr, basename(argv[0]));
return 1;
}
if (!fs.era_threshold_) {
cerr << "Please specify --written-since" << endl;
usage(cerr, basename(argv[0]));
return 1;
}
return invalidate(argv[optind], output, fs);
}
base::command era::era_invalidate_cmd("era_invalidate", era_invalidate_main);
//----------------------------------------------------------------

126
era/era_restore.cc Normal file
View File

@ -0,0 +1,126 @@
#include "version.h"
#include "era/commands.h"
#include "era/metadata.h"
#include "era/restore_emitter.h"
#include "era/xml_format.h"
#include "persistent-data/file_utils.h"
#include <boost/lexical_cast.hpp>
#include <fstream>
#include <getopt.h>
#include <iostream>
#include <libgen.h>
#include <string>
using namespace boost;
using namespace era;
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
namespace {
struct flags {
flags()
: quiet(false) {
}
optional<string> input;
optional<string> output;
bool quiet;
};
int restore(flags const &fs, bool quiet) {
try {
block_manager<>::ptr bm = open_bm(*fs.output, block_manager<>::READ_WRITE);
metadata::ptr md(new metadata(bm, metadata::CREATE));
emitter::ptr restorer = create_restore_emitter(*md);
parse_xml(*fs.input, restorer, fs.quiet);
} catch (std::exception &e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
void usage(ostream &out, string const &cmd) {
out << "Usage: " << cmd << " [options]" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-i|--input} <input xml file>" << endl
<< " {-o|--output} <output device or file>" << endl
<< " {-q|--quiet}" << endl
<< " {-V|--version}" << endl;
}
}
int era_restore_main(int argc, char **argv)
{
int c;
flags fs;
char const *prog_name = basename(argv[0]);
char const *short_opts = "hi:o:qV";
option const long_opts[] = {
{ "help", no_argument, NULL, 'h'},
{ "input", required_argument, NULL, 'i' },
{ "output", required_argument, NULL, 'o'},
{ "quiet", no_argument, NULL, 'q'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout, prog_name);
return 0;
case 'i':
fs.input = optional<string>(string(optarg));
break;
case 'o':
fs.output = optional<string>(string(optarg));
break;
case 'q':
fs.quiet = true;
break;
case 'V':
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
default:
usage(cerr, prog_name);
return 1;
}
}
if (argc != optind) {
usage(cerr, prog_name);
return 1;
}
if (!fs.input) {
cerr << "No input file provided." << endl << endl;
usage(cerr, prog_name);
return 1;
}
if (!fs.output) {
cerr << "No output file provided." << endl << endl;
usage(cerr, prog_name);
return 1;
}
return restore(fs, fs.quiet);
}
base::command era::era_restore_cmd("era_restore", era_restore_main);
//----------------------------------------------------------------

113
era/metadata.cc Normal file
View File

@ -0,0 +1,113 @@
#include "era/metadata.h"
#include "persistent-data/space-maps/core.h"
using namespace era;
//----------------------------------------------------------------
namespace {
unsigned const METADATA_CACHE_SIZ = 1024;
// FIXME: duplication
transaction_manager::ptr
open_tm(block_manager<>::ptr bm) {
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
sm->inc(SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
}
void
copy_space_maps(space_map::ptr lhs, space_map::ptr rhs) {
for (block_address b = 0; b < rhs->get_nr_blocks(); b++) {
uint32_t count = rhs->get_count(b);
if (count > 0)
lhs->set_count(b, rhs->get_count(b));
}
}
}
metadata::metadata(block_manager<>::ptr bm, open_type ot)
{
switch (ot) {
case CREATE:
create_metadata(bm);
break;
case OPEN:
open_metadata(bm);
break;
}
}
metadata::metadata(block_manager<>::ptr bm, block_address metadata_snap)
{
open_metadata(bm);
}
void
metadata::create_metadata(block_manager<>::ptr bm)
{
tm_ = open_tm(bm);
space_map::ptr core = tm_->get_sm();
metadata_sm_ = create_metadata_sm(*tm_, tm_->get_bm()->get_nr_blocks());
copy_space_maps(metadata_sm_, core);
tm_->set_sm(metadata_sm_);
writeset_tree_ = writeset_tree::ptr(new writeset_tree(*tm_, era_detail_traits::ref_counter(tm_)));
era_array_ = era_array::ptr(new era_array(*tm_,
uint32_traits::ref_counter()));
}
void
metadata::open_metadata(block_manager<>::ptr bm, block_address loc)
{
tm_ = open_tm(bm);
sb_ = read_superblock(tm_->get_bm(), loc);
writeset_tree_ = writeset_tree::ptr(new writeset_tree(*tm_,
sb_.writeset_tree_root,
era_detail_traits::ref_counter(tm_)));
era_array_ = era_array::ptr(new era_array(*tm_,
uint32_traits::ref_counter(),
sb_.era_array_root,
sb_.nr_blocks));
}
void
metadata::commit()
{
commit_space_map();
commit_writesets();
commit_era_array();
commit_superblock();
}
void
metadata::commit_space_map()
{
metadata_sm_->commit();
metadata_sm_->copy_root(&sb_.metadata_space_map_root, sizeof(sb_.metadata_space_map_root));
}
void
metadata::commit_writesets()
{
sb_.writeset_tree_root = writeset_tree_->get_root();
}
void
metadata::commit_era_array()
{
sb_.era_array_root = era_array_->get_root();
}
void
metadata::commit_superblock()
{
write_superblock(tm_->get_bm(), sb_);
}
//----------------------------------------------------------------

55
era/metadata.h Normal file
View File

@ -0,0 +1,55 @@
#ifndef ERA_METADATA_H
#define ERA_METADATA_H
#include "base/endian_utils.h"
#include "persistent-data/block.h"
#include "persistent-data/data-structures/array.h"
#include "persistent-data/data-structures/bitset.h"
#include "persistent-data/space-maps/disk.h"
#include "persistent-data/transaction_manager.h"
#include "era/superblock.h"
#include "era/writeset_tree.h"
#include "era/era_array.h"
//----------------------------------------------------------------
namespace era {
class metadata {
public:
enum open_type {
CREATE,
OPEN
};
typedef block_manager<>::read_ref read_ref;
typedef block_manager<>::write_ref write_ref;
typedef boost::shared_ptr<metadata> ptr;
metadata(block_manager<>::ptr bm, open_type ot);
metadata(block_manager<>::ptr bm, block_address metadata_snap);
void commit();
typedef persistent_data::transaction_manager tm;
tm::ptr tm_;
superblock sb_;
checked_space_map::ptr metadata_sm_;
writeset_tree::ptr writeset_tree_;
era_array::ptr era_array_;
private:
void create_metadata(block_manager<>::ptr bm);
void open_metadata(block_manager<>::ptr bm,
block_address loc = SUPERBLOCK_LOCATION);
void commit_space_map();
void commit_writesets();
void commit_era_array();
void commit_superblock();
};
};
//----------------------------------------------------------------
#endif

212
era/metadata_dump.cc Normal file
View File

@ -0,0 +1,212 @@
#include "era/metadata_dump.h"
#include "era/era_array.h"
using namespace era;
using namespace std;
//----------------------------------------------------------------
namespace {
string to_string(unsigned char const *data) {
// FIXME: we're assuming the data is zero terminated here
return std::string(reinterpret_cast<char const *>(data));
}
void raise_metadata_damage() {
throw std::runtime_error("metadata contains errors (run era_check for details).\n"
"perhaps you wanted to run with --repair ?");
}
class writeset_tree_emitter : public writeset_tree_detail::writeset_visitor {
public:
writeset_tree_emitter(emitter::ptr e)
: e_(e) {
}
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) {
e_->begin_writeset(era, nr_bits);
}
virtual void bit(uint32_t bit, bool value) {
e_->writeset_bit(bit, value);
}
virtual void writeset_end() {
e_->end_writeset();
}
private:
emitter::ptr e_;
};
class writeset_tree_collator : public writeset_tree_detail::writeset_visitor {
public:
writeset_tree_collator(map<uint32_t, uint32_t> &exceptions)
: exceptions_(exceptions),
current_era_(0) {
}
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) {
current_era_ = era;
}
virtual void bit(uint32_t bit, bool value) {
if (value) {
map<uint32_t, uint32_t>::const_iterator it = exceptions_.find(bit);
if (it == exceptions_.end() || it->second < current_era_)
exceptions_.insert(make_pair(bit, current_era_));
}
}
virtual void writeset_end() {
}
private:
map<uint32_t, uint32_t> &exceptions_;
uint32_t current_era_;
};
struct ignore_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
void visit(writeset_tree_detail::missing_eras const &d) {
}
void visit(writeset_tree_detail::damaged_writeset const &d) {
}
};
struct fatal_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
void visit(writeset_tree_detail::missing_eras const &d) {
raise_metadata_damage();
}
void visit(writeset_tree_detail::damaged_writeset const &d) {
raise_metadata_damage();
}
};
//--------------------------------
class era_array_emitter : public era_array_visitor {
public:
era_array_emitter(emitter::ptr e, map<uint32_t, uint32_t> const &exceptions)
: e_(e),
exceptions_(exceptions) {
}
virtual void visit(uint32_t index, uint32_t era) {
map<uint32_t, uint32_t>::const_iterator it = exceptions_.find(index);
if (it != exceptions_.end() && it->second > era)
e_->era(index, it->second);
else
e_->era(index, era);
}
private:
emitter::ptr e_;
map<uint32_t, uint32_t> exceptions_;
};
struct ignore_era_array_damage : public era_array_detail::damage_visitor {
void visit(era_array_detail::missing_eras const &d) {
}
void visit(era_array_detail::invalid_era const &d) {
}
};
class fatal_era_array_damage : public era_array_detail::damage_visitor {
void visit(era_array_detail::missing_eras const &d) {
raise_metadata_damage();
}
void visit(era_array_detail::invalid_era const &d) {
raise_metadata_damage();
}
};
void
dump(metadata::ptr md, emitter::ptr e, bool repair)
{
{
writeset_tree_emitter visitor(e);
ignore_writeset_tree_damage ignore;
fatal_writeset_tree_damage fatal;
writeset_tree_detail::damage_visitor &dv = repair ?
static_cast<writeset_tree_detail::damage_visitor &>(ignore) :
static_cast<writeset_tree_detail::damage_visitor &>(fatal);
walk_writeset_tree(md->tm_, *md->writeset_tree_, visitor, dv);
}
e->begin_era_array();
{
map<uint32_t, uint32_t> exceptions;
era_array_emitter visitor(e, exceptions);
ignore_era_array_damage ignore;
fatal_era_array_damage fatal;
era_array_detail::damage_visitor &dv = repair ?
static_cast<era_array_detail::damage_visitor &>(ignore) :
static_cast<era_array_detail::damage_visitor &>(fatal);
walk_era_array(*md->era_array_, visitor, dv);
}
e->end_era_array();
}
void dump_logical(metadata::ptr md, emitter::ptr e, bool repair)
{
// This will potentially use a lot of memory, but I don't
// see a way around it.
map<uint32_t, uint32_t> exceptions;
{
writeset_tree_collator visitor(exceptions);
ignore_writeset_tree_damage ignore;
fatal_writeset_tree_damage fatal;
writeset_tree_detail::damage_visitor &dv = repair ?
static_cast<writeset_tree_detail::damage_visitor &>(ignore) :
static_cast<writeset_tree_detail::damage_visitor &>(fatal);
walk_writeset_tree(md->tm_, *md->writeset_tree_, visitor, dv);
}
e->begin_era_array();
{
era_array_emitter visitor(e, exceptions);
ignore_era_array_damage ignore;
fatal_era_array_damage fatal;
era_array_detail::damage_visitor &dv = repair ?
static_cast<era_array_detail::damage_visitor &>(ignore) :
static_cast<era_array_detail::damage_visitor &>(fatal);
walk_era_array(*md->era_array_, visitor, dv);
}
e->end_era_array();
}
}
//----------------------------------------------------------------
void
era::metadata_dump(metadata::ptr md, emitter::ptr e,
bool repair, bool logical)
{
superblock const &sb = md->sb_;
e->begin_superblock(to_string(sb.uuid), sb.data_block_size,
sb.nr_blocks,
sb.current_era);
{
if (logical)
dump_logical(md, e, repair);
else
dump(md, e, repair);
}
e->end_superblock();
}
//----------------------------------------------------------------

16
era/metadata_dump.h Normal file
View File

@ -0,0 +1,16 @@
#ifndef ERA_METADATA_DUMP_H
#define ERA_METADATA_DUMP_H
#include "era/metadata.h"
#include "era/emitter.h"
//----------------------------------------------------------------
namespace era {
void metadata_dump(metadata::ptr md, emitter::ptr out,
bool repair, bool logical);
}
//----------------------------------------------------------------
#endif

116
era/restore_emitter.cc Normal file
View File

@ -0,0 +1,116 @@
#include "era/restore_emitter.h"
#include "era/superblock.h"
using namespace era;
using namespace persistent_data;
//----------------------------------------------------------------
namespace {
class restorer : public emitter {
public:
restorer(metadata &md)
: md_(md),
in_superblock_(false),
in_writeset_(false),
in_era_array_(false) {
}
virtual void begin_superblock(std::string const &uuid,
uint32_t data_block_size,
pd::block_address nr_blocks,
uint32_t current_era) {
superblock &sb = md_.sb_;
memcpy(sb.uuid, reinterpret_cast<__u8 const *>(uuid.c_str()),
min<size_t>(sizeof(sb.uuid), uuid.length()));
sb.data_block_size = data_block_size;
sb.nr_blocks = nr_blocks;
sb.current_era = current_era;
nr_blocks = nr_blocks;
md_.era_array_->grow(nr_blocks, 0);
in_superblock_ = true;
}
virtual void end_superblock() {
if (!in_superblock_)
throw runtime_error("xml missing superblock");
md_.commit();
}
virtual void begin_writeset(uint32_t era, uint32_t nr_bits) {
if (!in_superblock_)
throw runtime_error("missing superblock");
if (in_writeset_)
throw runtime_error("attempt to begin writeset when already in one");
in_writeset_ = true;
era_ = era;
bits_.reset(new bitset(*md_.tm_));
bits_->grow(nr_bits, false);
}
virtual void writeset_bit(uint32_t bit, bool value) {
bits_->set(bit, value);
}
virtual void end_writeset() {
in_writeset_ = false;
bits_->flush();
era_detail e;
e.nr_bits = bits_->get_nr_bits();
e.writeset_root = bits_->get_root();
uint64_t key[1] = {era_};
md_.writeset_tree_->insert(key, e);
}
virtual void begin_era_array() {
if (!in_superblock_)
throw runtime_error("missing superblock");
in_era_array_ = true;
}
virtual void era(pd::block_address block, uint32_t era) {
if (!in_era_array_)
throw runtime_error("missing era array");
md_.era_array_->set(block, era);
}
virtual void end_era_array() {
in_era_array_ = false;
}
private:
metadata &md_;
bool in_superblock_;
bool in_writeset_;
uint32_t era_;
pd::bitset::ptr bits_;
bool in_era_array_;
uint32_t nr_blocks_;
};
}
//----------------------------------------------------------------
emitter::ptr
era::create_restore_emitter(metadata &md)
{
return emitter::ptr(new restorer(md));
}
//----------------------------------------------------------------

15
era/restore_emitter.h Normal file
View File

@ -0,0 +1,15 @@
#ifndef ERA_RESTORE_EMITTER_H
#define ERA_RESTORE_EMITTER_H
#include "era/emitter.h"
#include "era/metadata.h"
//----------------------------------------------------------------
namespace era {
emitter::ptr create_restore_emitter(metadata &md);
}
//----------------------------------------------------------------
#endif

337
era/superblock.cc Normal file
View File

@ -0,0 +1,337 @@
#include "era/superblock.h"
#include "persistent-data/checksum.h"
#include "persistent-data/errors.h"
using namespace base;
using namespace era;
using namespace superblock_damage;
using namespace persistent_data;
//----------------------------------------------------------------
namespace {
using namespace base;
size_t const SPACE_MAP_ROOT_SIZE = 128;
size_t const UUID_LEN = 16;
struct superblock_disk {
le32 csum;
le32 flags;
le64 blocknr;
__u8 uuid[UUID_LEN];
le64 magic;
le32 version;
__u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
le32 data_block_size;
le32 metadata_block_size;
le32 nr_blocks;
le32 current_era;
era_detail_disk current_detail;
le64 writeset_tree_root;
le64 era_array_root;
le64 metadata_snap;
} __attribute__ ((packed));
struct superblock_traits {
typedef superblock_disk disk_type;
typedef superblock value_type;
static void unpack(disk_type const &disk, value_type &value);
static void pack(value_type const &value, disk_type &disk);
};
uint32_t const SUPERBLOCK_MAGIC = 2126579579;
uint32_t const VERSION_BEGIN = 1;
uint32_t const VERSION_END = 2;
}
//----------------------------------------------------------------
superblock_flags::superblock_flags()
: unhandled_flags_(0)
{
}
superblock_flags::superblock_flags(uint32_t bits)
{
if (bits & (1 << CLEAN_SHUTDOWN_BIT)) {
flags_.insert(CLEAN_SHUTDOWN);
bits &= ~(1 << CLEAN_SHUTDOWN_BIT);
}
unhandled_flags_ = bits;
}
void
superblock_flags::set_flag(flag f)
{
flags_.insert(f);
}
void
superblock_flags::clear_flag(flag f)
{
flags_.erase(f);
}
bool
superblock_flags::get_flag(flag f) const
{
return flags_.find(f) != flags_.end();
}
uint32_t
superblock_flags::encode() const
{
uint32_t r = 0;
if (get_flag(CLEAN_SHUTDOWN))
r = r | (1 << CLEAN_SHUTDOWN_BIT);
return r;
}
uint32_t
superblock_flags::get_unhandled_flags() const
{
return unhandled_flags_;
}
//----------------------------------------------------------------
superblock::superblock()
: csum(0),
blocknr(0),
flags(),
magic(SUPERBLOCK_MAGIC),
version(VERSION_END - 1),
data_block_size(0),
metadata_block_size(8),
nr_blocks(0),
current_era(0),
writeset_tree_root(0),
era_array_root(0)
{
memset(uuid, 0, sizeof(uuid));
memset(metadata_space_map_root, 0, sizeof(metadata_space_map_root));
}
//----------------------------------------------------------------
void
superblock_traits::unpack(disk_type const &disk, value_type &value)
{
//value.flags = to_cpu<uint32_t>(disk.flags);
value.blocknr = to_cpu<uint64_t>(disk.blocknr);
value.magic = to_cpu<uint64_t>(disk.magic);
value.version = to_cpu<uint32_t>(disk.version);
memcpy(value.metadata_space_map_root, disk.metadata_space_map_root,
sizeof(value.metadata_space_map_root));
value.data_block_size = to_cpu<uint32_t>(disk.data_block_size);
value.metadata_block_size = to_cpu<uint32_t>(disk.metadata_block_size);
value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
value.current_era = to_cpu<uint32_t>(disk.current_era);
era_detail_traits::unpack(disk.current_detail, value.current_detail);
value.writeset_tree_root = to_cpu<uint64_t>(disk.writeset_tree_root);
value.era_array_root = to_cpu<uint64_t>(disk.era_array_root);
block_address ms = to_cpu<uint64_t>(disk.metadata_snap);
value.metadata_snap = (ms == SUPERBLOCK_LOCATION) ?
boost::optional<block_address>() :
boost::optional<block_address>(ms);
}
void
superblock_traits::pack(value_type const &value, disk_type &disk)
{
//disk.flags = to_disk<uint32_t>(value.flags);
disk.blocknr = to_disk<le64>(value.blocknr);
disk.magic = to_disk<le64>(value.magic);
disk.version = to_disk<le32>(value.version);
memcpy(disk.metadata_space_map_root, value.metadata_space_map_root,
sizeof(disk.metadata_space_map_root));
disk.data_block_size = to_disk<le32>(value.data_block_size);
disk.metadata_block_size = to_disk<le32>(value.metadata_block_size);
disk.nr_blocks = to_disk<le32>(value.nr_blocks);
disk.current_era = to_disk<le32>(value.current_era);
era_detail_traits::pack(value.current_detail, disk.current_detail);
disk.writeset_tree_root = to_disk<le64>(value.writeset_tree_root);
disk.era_array_root = to_disk<le64>(value.era_array_root);
disk.metadata_snap = value.metadata_snap ?
to_disk<le64>(*value.metadata_snap) :
to_disk<le64>(SUPERBLOCK_LOCATION);
}
//--------------------------------
superblock_corrupt::superblock_corrupt(std::string const &desc)
: damage(desc)
{
}
void
superblock_corrupt::visit(damage_visitor &v) const
{
v.visit(*this);
}
superblock_invalid::superblock_invalid(std::string const &desc)
: damage(desc)
{
}
void
superblock_invalid::visit(damage_visitor &v) const
{
v.visit(*this);
}
//----------------------------------------------------------------
namespace era_validator {
using namespace persistent_data;
uint32_t const VERSION = 1;
unsigned const SECTOR_TO_BLOCK_SHIFT = 3;
uint32_t const SUPERBLOCK_CSUM_SEED = 146538381;
// FIXME: turn into a template, we have 3 similar classes now
struct sb_validator : public bcache::validator {
virtual void check(void const *raw, block_address location) const {
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);
sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
if (sum.get_sum() != to_cpu<uint32_t>(sbd->csum))
throw checksum_error("bad checksum in superblock");
}
virtual void prepare(void *raw, block_address location) const {
superblock_disk *sbd = reinterpret_cast<superblock_disk *>(raw);
crc32c sum(SUPERBLOCK_CSUM_SEED);
sum.append(&sbd->flags, MD_BLOCK_SIZE - sizeof(uint32_t));
sbd->csum = to_disk<base::le32>(sum.get_sum());
}
};
bcache::validator::ptr mk_v() {
return bcache::validator::ptr(new sb_validator);
}
}
//----------------------------------------------------------------
superblock
era::read_superblock(block_manager<>::ptr bm, block_address location)
{
superblock sb;
block_manager<>::read_ref r = bm->read_lock(location, era_validator::mk_v());
superblock_disk const *sbd = reinterpret_cast<superblock_disk const *>(r.data());
superblock_traits::unpack(*sbd, sb);
return sb;
}
void
era::write_superblock(block_manager<>::ptr bm, superblock const &sb, block_address location)
{
block_manager<>::write_ref w = bm->superblock_zero(location, era_validator::mk_v());
superblock_traits::pack(sb, *reinterpret_cast<superblock_disk *>(w.data()));
}
void
era::check_superblock(superblock const &sb,
block_address nr_metadata_blocks,
damage_visitor &visitor)
{
if (sb.flags.get_unhandled_flags()) {
ostringstream msg;
msg << "invalid flags: " << sb.flags.get_unhandled_flags();
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.blocknr >= nr_metadata_blocks) {
ostringstream msg;
msg << "blocknr out of bounds: " << sb.blocknr << " >= " << nr_metadata_blocks;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.magic != SUPERBLOCK_MAGIC) {
ostringstream msg;
msg << "magic in incorrect: " << sb.magic;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.version >= VERSION_END) {
ostringstream msg;
msg << "version incorrect: " << sb.version;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.version < VERSION_BEGIN) {
ostringstream msg;
msg << "version incorrect: " << sb.version;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.metadata_block_size != 8) {
ostringstream msg;
msg << "metadata block size incorrect: " << sb.metadata_block_size;
visitor.visit(superblock_invalid(msg.str()));
}
if (sb.writeset_tree_root == SUPERBLOCK_LOCATION) {
string msg("writeset tree root points back to the superblock");
visitor.visit(superblock_invalid(msg));
}
if (sb.era_array_root == SUPERBLOCK_LOCATION) {
string msg("era array root points back to the superblock");
visitor.visit(superblock_invalid(msg));
}
if (sb.writeset_tree_root == sb.era_array_root) {
ostringstream msg;
msg << "writeset tree root and era array both point to the same block: "
<< sb.era_array_root;
visitor.visit(superblock_invalid(msg.str()));
}
}
void
era::check_superblock(persistent_data::block_manager<>::ptr bm,
block_address nr_metadata_blocks,
damage_visitor &visitor)
{
superblock sb;
try {
sb = read_superblock(bm, SUPERBLOCK_LOCATION);
} catch (std::exception const &e) {
// FIXME: what if it fails due to a zero length file? Not
// really a corruption, so much as an io error. Should we
// separate these?
visitor.visit(superblock_corrupt(e.what()));
}
check_superblock(sb, nr_metadata_blocks, visitor);
}
//----------------------------------------------------------------

135
era/superblock.h Normal file
View File

@ -0,0 +1,135 @@
#ifndef ERA_SUPERBLOCK_H
#define ERA_SUPERBLOCK_H
#include "persistent-data/block.h"
#include "era/era_detail.h"
#include <boost/optional.hpp>
#include <set>
//----------------------------------------------------------------
namespace era {
typedef unsigned char __u8;
class superblock_flags {
public:
enum flag {
CLEAN_SHUTDOWN
};
enum flag_bits {
CLEAN_SHUTDOWN_BIT = 0
};
superblock_flags();
superblock_flags(uint32_t bits);
void set_flag(flag f);
void clear_flag(flag f);
bool get_flag(flag f) const;
uint32_t encode() const;
uint32_t get_unhandled_flags() const;
private:
uint32_t unhandled_flags_;
std::set<flag> flags_;
};
unsigned const SPACE_MAP_ROOT_SIZE = 128;
uint64_t const SUPERBLOCK_LOCATION = 0;
struct superblock {
superblock();
uint32_t csum;
uint64_t blocknr;
superblock_flags flags;
__u8 uuid[16]; // FIXME: do we really need this?
uint64_t magic;
uint32_t version;
__u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
uint32_t data_block_size;
uint32_t metadata_block_size;
uint32_t nr_blocks;
uint32_t current_era;
era_detail current_detail;
// A btree of undigested era_details
uint64_t writeset_tree_root;
// Big array holding the digested era/block info.
uint64_t era_array_root;
boost::optional<persistent_data::block_address> metadata_snap;
};
//--------------------------------
namespace superblock_damage {
class damage_visitor;
class damage {
public:
damage(std::string const &desc)
: desc_(desc) {
}
virtual ~damage() {}
virtual void visit(damage_visitor &v) const = 0;
std::string const &get_desc() const {
return desc_;
}
private:
std::string desc_;
};
struct superblock_corrupt : public damage {
superblock_corrupt(std::string const &desc);
void visit(damage_visitor &v) const;
};
struct superblock_invalid : public damage {
superblock_invalid(std::string const &desc);
void visit(damage_visitor &v) const;
};
class damage_visitor {
public:
virtual ~damage_visitor() {}
void visit(damage const &d);
virtual void visit(superblock_corrupt const &d) = 0;
virtual void visit(superblock_invalid const &d) = 0;
};
}
//--------------------------------
superblock read_superblock(persistent_data::block_manager<>::ptr bm,
persistent_data::block_address location = SUPERBLOCK_LOCATION);
void write_superblock(persistent_data::block_manager<>::ptr bm,
superblock const &sb,
persistent_data::block_address location = SUPERBLOCK_LOCATION);
void check_superblock(superblock const &sb,
persistent_data::block_address nr_metadata_blocks,
superblock_damage::damage_visitor &visitor);
void check_superblock(persistent_data::block_manager<>::ptr bm,
persistent_data::block_address nr_metadata_blocks,
superblock_damage::damage_visitor &visitor);
}
//----------------------------------------------------------------
#endif

134
era/writeset_tree.cc Normal file
View File

@ -0,0 +1,134 @@
#include "era/writeset_tree.h"
#include "persistent-data/data-structures/btree_damage_visitor.h"
#include "persistent-data/data-structures/bitset.h"
using namespace era;
using namespace writeset_tree_detail;
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
missing_eras::missing_eras(string const &desc,
run<uint32_t> const &eras)
: damage(desc),
eras_(eras)
{
}
void
missing_eras::visit(damage_visitor &v) const {
v.visit(*this);
}
damaged_writeset::damaged_writeset(string const &desc,
uint32_t era,
run<uint32_t> missing_bits)
: damage(desc),
era_(era),
missing_bits_(missing_bits)
{
}
void
damaged_writeset::visit(damage_visitor &v) const
{
v.visit(*this);
}
//----------------------------------------------------------------
namespace {
class ll_writeset_visitor : public bitset_detail::bitset_visitor {
public:
typedef persistent_data::transaction_manager::ptr tm_ptr;
ll_writeset_visitor(tm_ptr tm,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv)
: tm_(tm),
era_(0),
writeset_v_(writeset_v),
dv_(dv) {
}
void visit(btree_path const &path, era_detail const &era) {
era_ = path[0];
persistent_data::bitset bs(*tm_, era.writeset_root, era.nr_bits);
writeset_v_.writeset_begin(era_, era.nr_bits);
bs.walk_bitset(*this);
writeset_v_.writeset_end();
}
void visit(uint32_t index, bool value) {
writeset_v_.bit(index, value);
}
void visit(bitset_detail::missing_bits const &d) {
dv_.visit(writeset_tree_detail::damaged_writeset("missing bits", era_, d.keys_));
}
private:
tm_ptr tm_;
uint64_t era_;
writeset_tree_detail::writeset_visitor &writeset_v_;
writeset_tree_detail::damage_visitor &dv_;
};
class ll_damage_visitor {
public:
ll_damage_visitor(damage_visitor &v)
: v_(v) {
}
virtual void visit(btree_path const &path,
btree_detail::damage const &d) {
v_.visit(missing_eras(d.desc_, to_uint32(d.lost_keys_)));
}
private:
template <typename T>
run<uint32_t> to_uint32(run<T> const &r) {
return run<uint32_t>(boost::optional<uint32_t>(r.begin_),
boost::optional<uint32_t>(r.end_));
}
damage_visitor &v_;
};
}
void
era::walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv)
{
ll_writeset_visitor ll_bv(tm, writeset_v, dv);
ll_damage_visitor ll_dv(dv);
btree_visit_values(tree, ll_bv, ll_dv);
}
namespace {
class noop_writeset_visitor : public writeset_tree_detail::writeset_visitor {
public:
void writeset_begin(uint32_t era, uint32_t nr_bits) {
}
void bit(uint32_t index, bool value) {
}
void writeset_end() {
}
};
};
void
era::check_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::damage_visitor &dv)
{
noop_writeset_visitor bv;
walk_writeset_tree(tm, tree, bv, dv);
}
//----------------------------------------------------------------

87
era/writeset_tree.h Normal file
View File

@ -0,0 +1,87 @@
#ifndef ERA_WRITESET_TREE_H
#define ERA_WRITESET_TREE_H
#include "era/era_detail.h"
#include "persistent-data/data-structures/btree.h"
//----------------------------------------------------------------
namespace era {
namespace writeset_tree_detail {
class damage_visitor;
class damage {
public:
damage(std::string const &desc)
: desc_(desc) {
}
virtual ~damage() {}
virtual void visit(damage_visitor &v) const = 0;
std::string const &get_desc() const {
return desc_;
}
private:
std::string desc_;
};
struct missing_eras : public damage {
missing_eras(std::string const &desc, run<uint32_t> const &eras);
virtual void visit(damage_visitor &v) const;
run<uint32_t> eras_;
};
struct damaged_writeset : public damage {
damaged_writeset(std::string const &desc,
uint32_t era,
run<uint32_t> missing_bits);
virtual void visit(damage_visitor &v) const;
uint32_t era_;
run<uint32_t> missing_bits_;
};
class damage_visitor {
public:
typedef boost::shared_ptr<damage_visitor> ptr;
virtual ~damage_visitor() {}
void visit(damage const &d) {
d.visit(*this);
}
virtual void visit(missing_eras const &d) = 0;
virtual void visit(damaged_writeset const &d) = 0;
};
class writeset_visitor {
public:
typedef boost::shared_ptr<writeset_visitor> ptr;
virtual ~writeset_visitor() {}
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) = 0;
virtual void bit(uint32_t index, bool value) = 0;
virtual void writeset_end() = 0;
};
}
typedef persistent_data::btree<1, era_detail_traits> writeset_tree;
void walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::writeset_visitor &writeset_v,
writeset_tree_detail::damage_visitor &dv);
void check_writeset_tree(persistent_data::transaction_manager::ptr tm,
writeset_tree const &tree,
writeset_tree_detail::damage_visitor &dv);
}
//----------------------------------------------------------------
#endif

176
era/xml_format.cc Normal file
View File

@ -0,0 +1,176 @@
#include "era/xml_format.h"
#include "base/indented_stream.h"
#include "base/xml_utils.h"
using namespace boost;
using namespace era;
using namespace persistent_data;
using namespace std;
using namespace xml_utils;
//----------------------------------------------------------------
namespace {
class xml_emitter : public emitter {
public:
xml_emitter(ostream &out)
: out_(out) {
}
void begin_superblock(std::string const &uuid,
uint32_t block_size,
pd::block_address nr_blocks,
uint32_t current_era) {
out_.indent();
out_ << "<superblock uuid=\"" << uuid << "\""
<< " block_size=\"" << block_size << "\""
<< " nr_blocks=\"" << nr_blocks << "\""
<< " current_era=\"" << current_era << "\">";
out_ << endl;
out_.inc();
}
void end_superblock() {
out_.dec();
out_.indent();
out_ << "</superblock>" << endl;
}
void begin_writeset(uint32_t era, uint32_t nr_bits) {
out_.indent();
out_ << "<writeset era=\"" << era << "\""
<< " nr_bits=\"" << nr_bits << "\">" << endl;
out_.inc();
}
void writeset_bit(uint32_t bit, bool value) {
out_.indent();
// FIXME: collect all the bits, then uuencode
out_ << "<bit block=\"" << bit << "\" value=\"" << truth_value(value) << "\"/>" << endl;
}
void end_writeset() {
out_.dec();
out_.indent();
out_ << "</writeset>" << endl;
}
void begin_era_array() {
out_.indent();
out_ << "<era_array>" << endl;
out_.inc();
}
void era(pd::block_address block, uint32_t era) {
out_.indent();
out_ << "<era block=\"" << block
<< "\" era=\"" << era << "\"/>" << endl;
}
void end_era_array() {
out_.dec();
out_.indent();
out_ << "</era_array>" << endl;
}
char const *truth_value(bool v) const {
return v ? "true" : "false";
}
private:
indented_stream out_;
};
//--------------------------------
// Parser
//--------------------------------
void parse_bit(attributes const &a, emitter *e) {
bool value;
string txt = get_attr<string>(a, "value");
if (txt == "true")
value = true;
else if (txt == "false")
value = false;
else
throw runtime_error("invalid boolean");
e->writeset_bit(get_attr<uint32_t>(a, "block"), value);
}
void start_tag(void *data, char const *el, char const **attr) {
emitter *e = static_cast<emitter *>(data);
attributes a;
build_attributes(a, attr);
if (!strcmp(el, "superblock"))
e->begin_superblock(get_attr<string>(a, "uuid"),
get_attr<uint32_t>(a, "block_size"),
get_attr<pd::block_address>(a, "nr_blocks"),
get_attr<uint32_t>(a, "current_era"));
else if (!strcmp(el, "writeset"))
e->begin_writeset(get_attr<uint32_t>(a, "era"),
get_attr<uint32_t>(a, "nr_bits"));
else if (!strcmp(el, "bit"))
parse_bit(a, e);
else if (!strcmp(el, "era_array"))
e->begin_era_array();
else if (!strcmp(el, "era"))
e->era(get_attr<pd::block_address>(a, "block"),
get_attr<uint32_t>(a, "era"));
else
throw runtime_error("unknown tag type");
}
void end_tag(void *data, const char *el) {
emitter *e = static_cast<emitter *>(data);
if (!strcmp(el, "superblock"))
e->end_superblock();
else if (!strcmp(el, "writeset"))
e->end_writeset();
else if (!strcmp(el, "era_array"))
e->end_era_array();
else if (!strcmp(el, "era"))
/* do nothing */
;
else if (!strcmp(el, "bit"))
/* do nothing */
;
else
throw runtime_error("unknown tag type");
}
}
//----------------------------------------------------------------
emitter::ptr
era::create_xml_emitter(std::ostream &out)
{
return emitter::ptr(new xml_emitter(out));
}
void
era::parse_xml(std::string const &backup_file, emitter::ptr e, bool quiet)
{
xml_parser p;
XML_SetUserData(p.get_parser(), e.get());
XML_SetElementHandler(p.get_parser(), start_tag, end_tag);
p.parse(backup_file, quiet);
}
//----------------------------------------------------------------

18
era/xml_format.h Normal file
View File

@ -0,0 +1,18 @@
#ifndef ERA_XML_FORMAT_H
#define ERA_XML_FORMAT_H
#include "base/progress_monitor.h"
#include "era/emitter.h"
#include <iosfwd>
//----------------------------------------------------------------
namespace era {
emitter::ptr create_xml_emitter(std::ostream &out);
void parse_xml(std::string const &backup_file, emitter::ptr e, bool quiet);
}
//----------------------------------------------------------------
#endif

7
features/cache_check.feature
View File

@ -13,19 +13,19 @@ Feature: cache_check
When I run `cache_check --help`
Then it should pass
And usage to stdout
And cache_usage to stdout
Scenario: print help
When I run `cache_check -h`
Then it should pass
And usage to stdout
And cache_usage to stdout
Scenario: Metadata file must be specified
When I run `cache_check`
Then it should fail
And usage to stderr
And cache_usage to stderr
And the stderr should contain:
"""
@ -52,6 +52,7 @@ Feature: cache_check
foo: Not a block device or regular file
"""
# This test will fail if you're running as root
Scenario: Metadata file exists, but can't be opened
Given input without read permissions
When I run `cache_check input`

6
features/cache_dump.feature
View File

@ -44,7 +44,7 @@ Feature: cache_dump
Scenario: dump/restore is a noop
Given valid cache metadata
When I cache_dump
And I cache_restore
And I cache_dump
When I cache dump
And I cache restore
And I cache dump
Then cache dumps 1 and 2 should be identical

2
features/cache_metadata_size.feature
View File

@ -63,7 +63,7 @@ Feature: cache_metadata_size
When I run cache_metadata_size with --block-size 64
Then it should fail with:
"""
Please specify either --device-size and --block-size, or --nr-blocks.
If you specify --block-size you must also give --device-size.
"""
Scenario: Contradictory info causes fail

27
features/cache_restore.feature
View File

@ -1,4 +1,4 @@
Feature: thin_restore
Feature: cache_restore
Scenario: print version (-V flag)
When I run cache_restore with -V
Then it should pass with version
@ -18,6 +18,7 @@ Feature: thin_restore
{-h|--help}
{-i|--input} <input xml file>
{-o|--output} <output device or file>
{-q|--quiet}
{-V|--version}
{--debug-override-metadata-version} <integer>
@ -36,6 +37,7 @@ Feature: thin_restore
{-h|--help}
{-i|--input} <input xml file>
{-o|--output} <output device or file>
{-q|--quiet}
{-V|--version}
{--debug-override-metadata-version} <integer>
@ -80,3 +82,26 @@ Feature: thin_restore
And an empty dev file
When I run cache_restore with -i metadata.xml -o metadata.bin --omit-clean-shutdown
Then it should pass
Scenario: --quiet is accepted
Given valid cache metadata
When I run cache_restore with -i metadata.xml -o metadata.bin --quiet
Then it should pass
And the output should contain exactly:
"""
"""
Scenario: -q is accepted
Given valid cache metadata
When I run cache_restore with -i metadata.xml -o metadata.bin -q
Then it should pass
And the output should contain exactly:
"""
"""
Scenario: dump/restore is a noop
Given valid cache metadata
When I cache dump
And I cache restore
And I cache dump
Then dumps 1 and 2 should be identical

83
features/era_check.feature Normal file
View File

@ -0,0 +1,83 @@
Feature: era_check
Scenario: print version (-V flag)
When I run `era_check -V`
Then it should pass with version
Scenario: print version (--version flag)
When I run `era_check --version`
Then it should pass with version
Scenario: print help
When I run `era_check --help`
Then it should pass
And era_usage to stdout
Scenario: print help
When I run `era_check -h`
Then it should pass
And era_usage to stdout
Scenario: Metadata file must be specified
When I run `era_check`
Then it should fail
And era_usage to stderr
And the stderr should contain:
"""
No input file provided.
"""
Scenario: Metadata file doesn't exist
When I run `era_check /arbitrary/filename`
Then it should fail
And the stderr should contain:
"""
/arbitrary/filename: No such file or directory
"""
Scenario: Metadata file cannot be a directory
Given a directory called foo
When I run `era_check foo`
Then it should fail
And the stderr should contain:
"""
foo: Not a block device or regular file
"""
# This test will fail if you're running as root
Scenario: Metadata file exists, but can't be opened
Given input without read permissions
When I run `era_check input`
Then it should fail
And the stderr should contain:
"""
Permission denied
"""
Scenario: Metadata file full of zeroes
Given input file
And block 1 is zeroed
When I run `era_check input`
Then it should fail
Scenario: --quiet is observed
Given input file
And block 1 is zeroed
When I run `era_check --quiet input`
Then it should fail
And it should give no output
Scenario: -q is observed
Given input file
And block 1 is zeroed
When I run `era_check -q input`
Then it should fail
And it should give no output

95
features/era_restore.feature Normal file
View File

@ -0,0 +1,95 @@
Feature: era_restore
Scenario: print version (-V flag)
When I run era_restore with -V
Then it should pass with version
Scenario: print version (--version flag)
When I run era_restore with --version
Then it should pass with version
Scenario: print help (-h)
When I run era_restore with -h
Then it should pass
And the output should contain exactly:
"""
Usage: era_restore [options]
Options:
{-h|--help}
{-i|--input} <input xml file>
{-o|--output} <output device or file>
{-q|--quiet}
{-V|--version}
"""
Scenario: print help (--help)
When I run era_restore with -h
Then it should pass
And the output should contain exactly:
"""
Usage: era_restore [options]
Options:
{-h|--help}
{-i|--input} <input xml file>
{-o|--output} <output device or file>
{-q|--quiet}
{-V|--version}
"""
Scenario: missing input file
Given the dev file metadata.bin
When I run era_restore with -o metadata.bin
Then it should fail with:
"""
No input file provided.
"""
Scenario: input file not found
Given the dev file metadata.bin
When I run era_restore with -i foo.xml -o metadata.bin
Then it should fail
Scenario: missing output file
When I run era_restore with -i metadata.xml
Then it should fail with:
"""
No output file provided.
"""
Scenario: successfully restores a valid xml file
Given a small era xml file
And an empty dev file
When I run era_restore with -i metadata.xml -o metadata.bin
Then it should pass
And the metadata should be valid
Scenario: --quiet is accepted
Given valid era metadata
When I run era_restore with -i metadata.xml -o metadata.bin --quiet
Then it should pass
And the output should contain exactly:
"""
"""
Scenario: -q is accepted
Given valid era metadata
When I run era_restore with -i metadata.xml -o metadata.bin -q
Then it should pass
And the output should contain exactly:
"""
"""
Scenario: dump/restore is a noop
Given valid era metadata
When I era dump
And I era restore
And I era dump
Then dumps 1 and 2 should be identical
Scenario: dump matches original metadata
Given valid era metadata
When I era dump
Then dumps 0 and 1 should be identical

16
features/step_definitions/cache_steps.rb
View File

@ -34,7 +34,7 @@ Then /^it should fail$/ do
assert_success(false)
end
USAGE =<<EOF
CACHE_USAGE =<<EOF
Usage: cache_check [options] {device|file}
Options:
{-q|--quiet}
@ -45,12 +45,12 @@ Options:
{--skip-hints}
EOF
Then /^usage to stdout$/ do
assert_partial_output(USAGE, all_stdout)
Then /^cache_usage to stdout$/ do
assert_partial_output(CACHE_USAGE, all_stdout)
end
Then /^usage to stderr$/ do
assert_partial_output(USAGE, all_stderr)
Then /^cache_usage to stderr$/ do
assert_partial_output(CACHE_USAGE, all_stderr)
end
When(/^I run cache_check with (.*?)$/) do |opts|
@ -80,9 +80,9 @@ end
Given(/^valid cache metadata$/) do
in_current_dir do
system("cache_xml create --nr-cache-blocks uniform[1000..5000] --nr-mappings uniform[500..1000] > #{xml_file}")
system("dd if=/dev/zero of=#{dev_file} bs=4k count=1024 > /dev/null")
end
run_simple("dd if=/dev/zero of=#{dev_file} bs=4k count=1024")
run_simple("cache_restore -i #{xml_file} -o #{dev_file}")
end
@ -100,10 +100,10 @@ Given(/^an empty dev file$/) do
run_simple("dd if=/dev/zero of=#{dev_file} bs=4k count=1024")
end
When(/^I cache_dump$/) do
When(/^I cache dump$/) do
run_simple("cache_dump #{dev_file} -o #{new_dump_file}", true)
end
When(/^I cache_restore$/) do
When(/^I cache restore$/) do
run_simple("cache_restore -i #{dump_files[-1]} -o #{dev_file}", true)
end

47
features/step_definitions/era_steps.rb Normal file
View File

@ -0,0 +1,47 @@
ERA_USAGE =<<EOF
Usage: era_check [options] {device|file}
Options:
{-q|--quiet}
{-h|--help}
{-V|--version}
{--super-block-only}
EOF
Then /^era_usage to stdout$/ do
assert_partial_output(ERA_USAGE, all_stdout)
end
Then /^era_usage to stderr$/ do
assert_partial_output(ERA_USAGE, all_stderr)
end
When(/^I run era_restore with (.*?)$/) do |opts|
run_simple("era_restore #{opts}", false)
end
Given(/^a small era xml file$/) do
in_current_dir do
system("era_xml create --nr-blocks 100 --nr-writesets 2 --current-era 1000 > #{xml_file}")
end
end
Then(/^the metadata should be valid$/) do
run_simple("era_check #{dev_file}", true)
end
Given(/^valid era metadata$/) do
in_current_dir do
system("era_xml create --nr-blocks 100 --nr-writesets 2 --current-era 1000 > #{xml_file}")
system("dd if=/dev/zero of=#{dev_file} bs=4k count=1024 > /dev/null")
end
run_simple("era_restore -i #{xml_file} -o #{dev_file}")
end
When(/^I era dump$/) do
run_simple("era_dump #{dev_file} -o #{new_dump_file}", true)
end
When(/^I era restore$/) do
run_simple("era_restore -i #{dump_files[-1]} -o #{dev_file}", true)
end

6
features/step_definitions/thin_steps.rb
View File

@ -1,9 +1,9 @@
Given(/^valid metadata$/) do
Given(/^valid thin metadata$/) do
in_current_dir do
system("thinp_xml create --nr-thins uniform[4..9] --nr-mappings uniform[1000..10000] > #{xml_file}")
system("dd if=/dev/zero of=#{dev_file} bs=4k count=1024 > /dev/null")
end
run_simple("dd if=/dev/zero of=#{dev_file} bs=4k count=1024")
run_simple("thin_restore -i #{xml_file} -o #{dev_file}")
end
@ -58,7 +58,7 @@ Then(/^dumps ([0-9]+) and ([0-9]+) should be identical$/) do |d1, d2|
run_simple("diff -ub #{dump_files[d1.to_i]} #{dump_files[d2.to_i]}", true)
end
Given(/^small metadata$/) do
Given(/^small thin metadata$/) do
in_current_dir do
system("thinp_xml create --nr-thins 2 --nr-mappings 1 > #{xml_file}")
end

2
features/support/aruba.rb
View File

@ -1,3 +1,3 @@
require 'aruba/cucumber'
ENV['PATH'] = "#{Dir::pwd}:#{ENV['PATH']}"
ENV['PATH'] = "#{Dir::pwd}/bin:#{ENV['PATH']}"

21
features/thin_check.feature
View File

@ -17,9 +17,10 @@ Feature: thin_check
{-q|--quiet}
{-h|--help}
{-V|--version}
{--super-block-only}
{--skip-mappings}
{--clear-needs-check-flag}
{--ignore-non-fatal-errors}
{--skip-mappings}
{--super-block-only}
"""
Scenario: print help
@ -32,9 +33,10 @@ Feature: thin_check
{-q|--quiet}
{-h|--help}
{-V|--version}
{--super-block-only}
{--skip-mappings}
{--clear-needs-check-flag}
{--ignore-non-fatal-errors}
{--skip-mappings}
{--super-block-only}
"""
Scenario: Unrecognised option should cause failure
@ -42,7 +44,7 @@ Feature: thin_check
Then it should fail
Scenario: --super-block-only check passes on valid metadata
Given valid metadata
Given valid thin metadata
When I run thin_check with --super-block-only
Then it should pass
@ -57,12 +59,12 @@ Feature: thin_check
"""
Scenario: --skip-mappings check passes on valid metadata
Given valid metadata
Given valid thin metadata
When I run thin_check with --skip-mappings
Then it should pass
Scenario: --ignore-non-fatal-errors check passes on valid metadata
Given valid metadata
Given valid thin metadata
When I run thin_check with --ignore-non-fatal-errors
Then it should pass
@ -77,3 +79,8 @@ Feature: thin_check
When I run thin_check with --quiet
Then it should fail
And it should give no output
Scenario: Accepts --clear-needs-check-flag
Given valid thin metadata
When I run thin_check with --clear-needs-check-flag
Then it should pass

56
features/thin_delta.feature Normal file
View File

@ -0,0 +1,56 @@
Feature: thin_delta
Scenario: print version (-V flag)
When I run `thin_delta -V`
Then it should pass with version
Scenario: print version (--version flag)
When I run `thin_delta --version`
Then it should pass with version
Scenario: print help
When I run `thin_delta --help`
Then it should pass with:
"""
Usage: thin_delta [options] --snap1 <snap> --snap2 <snap> <device or file>
Options:
{--verbose}
{-h|--help}
{-V|--version}
"""
Scenario: print help
When I run `thin_delta -h`
Then it should pass with:
"""
Usage: thin_delta [options] --snap1 <snap> --snap2 <snap> <device or file>
Options:
{--verbose}
{-h|--help}
{-V|--version}
"""
Scenario: Unrecognised option should cause failure
When I run `thin_delta --unleash-the-hedeghogs`
Then it should fail
Scenario: --snap1 must be specified
When I run `thin_delta --snap2 45 foo`
Then it should fail with:
"""
--snap1 not specified.
"""
Scenario: --snap2 must be specified
When I run `thin_delta --snap1 45 foo`
Then it should fail with:
"""
--snap2 not specified.
"""
Scenario: device must be specified
When I run `thin_delta --snap1 45 --snap2 50`
Then it should fail with:
"""
No input device provided.
"""

24
features/thin_restore.feature
View File

@ -17,6 +17,7 @@ Feature: thin_restore
{-h|--help}
{-i|--input} <input xml file>
{-o|--output} <output device or file>
{-q|--quiet}
{-V|--version}
"""
@ -30,6 +31,7 @@ Feature: thin_restore
{-h|--help}
{-i|--input} <input xml file>
{-o|--output} <output device or file>
{-q|--quiet}
{-V|--version}
"""
@ -53,19 +55,35 @@ Feature: thin_restore
No output file provided.
"""
Scenario: --quiet is accepted
Given valid thin metadata
When I run thin_restore with -i metadata.xml -o metadata.bin --quiet
Then it should pass
And the output should contain exactly:
"""
"""
Scenario: -q is accepted
Given valid thin metadata
When I run thin_restore with -i metadata.xml -o metadata.bin -q
Then it should pass
And the output should contain exactly:
"""
"""
Scenario: dump/restore is a noop
Given valid metadata
Given valid thin metadata
When I dump
And I restore
And I dump
Then dumps 1 and 2 should be identical
Scenario: dump matches original metadata
Given valid metadata
Given valid thin metadata
When I dump
Then dumps 0 and 1 should be identical
Scenario: dump matches original metadata (small)
Given small metadata
Given small thin metadata
When I dump
Then dumps 0 and 1 should be identical

22
features/thin_rmap.feature
View File

@ -42,56 +42,56 @@ Feature: thin_rmap
@announce
Scenario: Valid region format should pass
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region 23..7890
Then it should pass
Scenario: Invalid region format should fail (comma instean of dots)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region 23,7890
Then it should fail
Scenario: Invalid region format should fail (second number a word)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region 23..six
Then it should fail
Scenario: Invalid region format should fail (first number a word)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region four..7890
Then it should fail
Scenario: Invalid region format should fail (end is lower than begin)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region 89..88
Then it should fail
Scenario: Invalid region format should fail (end is equal to begin)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region 89..89
Then it should fail
Scenario: Invalid region format should fail (no begin)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region ..89
Then it should fail
Scenario: Invalid region format should fail (no end)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region 89..
Then it should fail
Scenario: Invalid region format should fail (no region at all)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region
Then it should fail
Scenario: Invalid region format should fail (three dots)
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region 89...99
Then it should fail
Scenario: Multiple regions should pass
Given valid metadata
Given valid thin metadata
When I run thin_rmap with --region 1..23 --region 45..78
Then it should pass

42
main.cc Normal file
View File

@ -0,0 +1,42 @@
#include <iostream>
#include "base/application.h"
#include "caching/commands.h"
#include "era/commands.h"
#include "thin-provisioning/commands.h"
//----------------------------------------------------------------
int main(int argc, char **argv)
{
using namespace base;
application app;
app.add_cmd(caching::cache_check_cmd);
app.add_cmd(caching::cache_dump_cmd);
app.add_cmd(caching::cache_metadata_size_cmd);
app.add_cmd(caching::cache_restore_cmd);
app.add_cmd(caching::cache_repair_cmd);
app.add_cmd(era::era_check_cmd);
app.add_cmd(era::era_dump_cmd);
app.add_cmd(era::era_invalidate_cmd);
app.add_cmd(era::era_restore_cmd);
app.add_cmd(thin_provisioning::thin_check_cmd);
app.add_cmd(thin_provisioning::thin_delta_cmd);
app.add_cmd(thin_provisioning::thin_dump_cmd);
app.add_cmd(thin_provisioning::thin_metadata_size_cmd);
app.add_cmd(thin_provisioning::thin_restore_cmd);
app.add_cmd(thin_provisioning::thin_repair_cmd);
app.add_cmd(thin_provisioning::thin_rmap_cmd);
// FIXME: convert thin_metadata_size to c++
//app.add_cmd(thin_provisioning::thin_metadata_size_cmd);
return app.run(argc, argv);
}
//----------------------------------------------------------------

57
man8/era_check.8 Normal file
View File

@ -0,0 +1,57 @@
.TH ERA_CHECK 8 "Thin Provisioning Tools" "Red Hat, Inc." \" -*- nroff -*-
.SH NAME
era_check \- validate era metadata on device or file
.SH SYNOPSIS
.B era_check
.RB [ options ]
.I {device|file}
.SH DESCRIPTION
.B era_check
checks era metadata created by
the device-mapper era target on a
.I device
or
.I file.
.SH OPTIONS
.IP "\fB\-q, \-\-quiet\fP"
Suppress output messages, return only exit code.
.IP "\fB\-h, \-\-help\fP"
Print help and exit.
.IP "\fB\-V, \-\-version\fP"
Output version information and exit.
.IP "\fB\-\-super\-block\-only\fP"
Only check the superblock is present.
.B era_check
will return a non-zero exit code if it finds a fatal
error. If any errors are discovered use
.B era_repair
to correct.
.SH EXAMPLE
Analyse thin provisioning metadata on logical volume
/dev/vg/metadata:
.sp
.B era_check /dev/vg/metadata
The device may not be actively used by the target
when running.
.SH DIAGNOSTICS
.B era_check
returns an exit code of 0 for success or 1 for error.
.SH SEE ALSO
.B era_dump(8)
.B era_repair(8)
.B era_restore(8)
.B era_invalidate(8)
.SH AUTHOR
Joe Thornber <ejt@redhat.com>

56
man8/era_dump.8 Normal file
View File

@ -0,0 +1,56 @@
.TH ERA_DUMP 8 "Thin Provisioning Tools" "Red Hat, Inc." \" -*- nroff -*-
.SH NAME
era_dump \- dump era metadata from device or file to standard output
.SH SYNOPSIS
.B era_dump
.RB [options]
.I {device|file}
.SH DESCRIPTION
.B era_dump
dumps binary era metadata created by the device-mapper
era target on a
.I device
or
.I file
to standard output for
analysis or postprocessing in XML format.
XML formated metadata can be fed into era_restore (see
.BR era_restore(8) )
in order to put it back onto a metadata
.I device
(to process by the device-mapper target) or
.I file.
.IP "\fB\-r, \-\-repair\fP".
Repair the metadata whilst dumping it.
.IP "\fB\-h, \-\-help\fP".
Print help and exit.
.IP "\fB\-V, \-\-version\fP".
Output version information and exit.
.IP "\fB\-\-logical\fP".
Fold any unprocessed write sets into the final era array. You
probably want to do this if you're intending to process the results as
it simplifies the XML.
.SH EXAMPLES
Dumps era metadata on logical volume /dev/vg/metadata
to standard output in XML format:
.sp
.B era_dump /dev/vg/metadata
.SH DIAGNOSTICS
.B era_dump
returns an exit code of 0 for success or 1 for error.
.SH SEE ALSO
.B era_check(8)
.B era_repair(8)
.B era_restore(8)
.B era_invalidate(8)
.SH AUTHOR
Joe Thornber <ejt@redhat.com>

46
man8/era_invalidate.8 Normal file
View File

@ -0,0 +1,46 @@
.TH ERA_INVALIDATE 8 "Thin Provisioning Tools" "Red Hat, Inc." \" -*- nroff -*-
.SH NAME
era_invalidate \- Provide a list of blocks that have changed since a particular era.
.SH SYNOPSIS
.B era_invalidate
.RB [ options ]
.I {device|file}
.SH DESCRIPTION
.B era_invalidate
Examines era metadata and lists blocks that may have changed since a given era.
.SH OPTIONS
.IP "\fB\-h, \-\-help\fP"
Print help and exit.
.IP "\fB\-V, \-\-version\fP"
Output version information and exit.
.IP "\fB\-o <output file>\fP"
Write output to a file rather than
.B stdout
.
.SH EXAMPLE
List the blocks that may have been written since the beginning of era
13 on the metadata device /dev/vg/metadata.
.sp
.B era_invalidate --written-since 13 /dev/vg/metadata
The device may not be actively used by the target
when running.
.SH DIAGNOSTICS
.B era_invalidate
returns an exit code of 0 for success or 1 for error (eg, metadata corruption).
.SH SEE ALSO
.B era_check(8),
.B era_dump(8),
.B era_repair(8),
.B era_restore(8)
.SH AUTHOR
Joe Thornber <ejt@redhat.com>

9
man8/thin_check.8
View File

@ -1,6 +1,6 @@
.TH THIN_CHECK 8 "Thin Provisioning Tools" "Red Hat, Inc." \" -*- nroff -*-
.SH NAME
thin_check \- repair thin provisioning metadata on device or file
thin_check \- validate thin provisioning metadata on device or file
.SH SYNOPSIS
.B thin_check
@ -35,19 +35,18 @@ metadata.
.IP "\fB\-\-ignore\-non\-fatal\-errors\fP"
.B thin_check
will only return a non-zero exit code if it finds a fatal
error. An example of a on fatal error is an incorrect data block
error. An example of a non fatal error is an incorrect data block
reference count causing a block to be considered allocated when it in
fact isn't. Ignoring errors for a long time is not advised, you
really should be using thin_repair to fix them.
.SH EXAMPLE
Analyses and repairs thin provisioning metadata on logical volume
Analyses thin provisioning metadata on logical volume
/dev/vg/metadata:
.sp
.B thin_check /dev/vg/metadata
The device may not be actively used by the target
when running.
The device must not be actively used by the target when running.
.SH DIAGNOSTICS
.B thin_check

47
man8/thin_delta.8 Normal file
View File

@ -0,0 +1,47 @@
.TH THIN_DELTA 8 "Thin Provisioning Tools" "Red Hat, Inc." \" -*- nroff -*-
.SH NAME
thin_delta \- Print the differences in the mappings between two thin devices.
.SH SYNOPSIS
.B thin_delta
.RB [ options ]
.I {device|file}
.SH DESCRIPTION
.B thin_delta
allows you to compare the mappings in two thin volumes (snapshots allow common blocks between thin volumes).
.
.SH OPTIONS
.IP "\fB\-\-thin1, \-\-snap1\fP"
The numeric identifier for the first thin volume to diff.
.IP "\fB\-\-thin1, \-\-snap1\fP"
The numeric identifier for the second thin volume to diff.
.IP "\fB\-m, \-\-metadata\-snap\fP [block#]"
If you want to get information out of a live pool then you will need
to take a metadata snapshot and use this switch. In order for the
information to be meaningful you need to ensure the thin volumes
you're examining are not changing (eg, do not activate those thins).
.IP "\fB\-\-verbose"
Provide extra information on the mappings.
.IP "\fB\-h, \-\-help\fP"
Print help and exit.
.IP "\fB\-V, \-\-version\fP"
Output version information and exit.
.SH SEE ALSO
.B thin_dump(8)
.B thin_repair(8)
.B thin_restore(8)
.B thin_rmap(8)
.B thin_trim(8)
.B thin_metadata_size(8)
.SH AUTHOR
Joe Thornber <ejt@redhat.com>

2
man8/thin_dump.8
View File

@ -47,7 +47,7 @@ Output version information and exit.
Dumps the thin provisioning metadata on logical volume /dev/vg/metadata
to standard output in human readable format:
.sp
.B thin_dump -f human_redable /dev/vg/metadata
.B thin_dump -f human_readable /dev/vg/metadata
Dumps the thin provisioning metadata on logical volume /dev/vg/metadata
to standard output in XML format:

24
man8/thin_metadata_size.8
View File

@ -15,13 +15,13 @@ Because thin provisioning pools are holding widely variable contents,
this tool is needed to provide sensible initial default size.
.IP "\fB\-b, \-\-block-size\fP \fIBLOCKSIZE[bskKmMgGtTpPeEzZyY]\fP"
Block size of thin provisioned devices in units of bytes,sectors,kilobytes,kibibytes,... respectively.
Block size of thin provisioned devices in units of bytes, sectors, kibibytes, kilobytes, ... respectively.
Default is in sectors without a block size unit specifier.
Size/number option arguments can be followed by unit specifiers in short one character
and long form (eg. -b1m or -b1megabytes).
and long form (eg. -b1m or -b1mebibytes).
.IP "\fB\-s, \-\-pool-size\fP \fIPOOLSIZE[bskKmMgGtTpPeEzZyY]\fP"
Thin provisioning pool size in units of bytes,sectors,kilobytes,kibibytes,... respectively.
Thin provisioning pool size in units of bytes, sectors, kibibytes, kilobytes, ... respectively.
Default is in sectors without a pool size unit specifier.
.IP "\fB\-m, \-\-max-thins\fP \fI#[bskKmMgGtTpPeEzZyY]\fP"
@ -30,7 +30,7 @@ Unit identifier supported to allow for convenient entry of large quantities, eg.
Default is absolute quantity without a number unit specifier.
.IP "\fB\-u, \-\-unit\fP \fI{bskKmMgGtTpPeEzZyY}\fP"
Output unit specifier in units of bytes,sectors,kilobytes,kibibytes,... respectively.
Output unit specifier in units of bytes, sectors, kibibytes, kilobytes, ... respectively.
Default is in sectors without an output unit specifier.
.IP "\fB\-n, \-\-numeric-only [short|long]\fP"
@ -43,24 +43,24 @@ Print help and exit.
Output version information and exit.
.SH EXAMPLES
Calculates the thin provisioning metadata device size for block size 64 kilobytes,
pool size 1 terabytes and maximum number of thin provisioned devices and snapshots of 1000
Calculates the thin provisioning metadata device size for block size 64 kibibytes,
pool size 1 tebibytes and maximum number of thin provisioned devices and snapshots of 1000
in units of sectors with long output:
.sp
.B thin_metadata_size -b64k -s1t -m1000
Or (using the long options instead) for block size 1 gigabyte, pool size 1 petabytes and maximum number of thin provisioned devices
and snapshots of 1 million with numeric only output in units of gigabytes:
Or (using the long options instead) for block size 1 gibibyte, pool size 1 petabyte and maximum number of thin provisioned devices
and snapshots of 1 million with numeric-only output in units of gigabytes:
.sp
.B thin_metadata_size --block-size=1g --pool-size=1p --max-thins=1M --unit=g --numeric-only
.B thin_metadata_size --block-size=1g --pool-size=1P --max-thins=1M --unit=G --numeric-only
Same as before (1g,1p,1M,numeric-only) but with unit specifier character appended:
Same as before (1g, 1P, 1M, numeric-only) but with unit specifier character appended:
.sp
.B thin_metadata_size --block-size=1giga --pool-size=1petabytes --max-thins=1mebi --unit=g --numeric-only=short
.B thin_metadata_size --block-size=1gibi --pool-size=1petabytes --max-thins=1mega --unit=G --numeric-only=short
Or with unit specifier string appended:
.sp
.B thin_metadata_size --block-size=1giga --pool-size=1petabytes --max-thins=1mebi --unit=g -nlong
.B thin_metadata_size --block-size=1gibi --pool-size=1petabytes --max-thins=1mega --unit=G -nlong
.SH DIAGNOSTICS
.B thin_metadata_size

3
man8/thin_restore.8
View File

@ -24,6 +24,9 @@ If restored to a metadata
.I device
, the metadata can be processed by the device-mapper target.
.IP "\fB\-q, \-\-quiet\fP"
Suppress output messages, return only exit code.
.IP "\fB\-i, \-\-input\fP \fI{device|file}\fP"
Input file or device with metadata.

2
man8/thin_rmap.8
View File

@ -29,7 +29,7 @@ Output version information and exit.
output reverse map for pool blocks 5..45 (denotes blocks 5 to 44 inclusive, but not block 45)
.sp
.B thin_rmap -r 5..45 /dev/vg/pool
.B thin_rmap --region 5..45 /dev/vg/pool
.SH DIAGNOSTICS
.B thin_rmap

Some files were not shown because too many files have changed in this diff Show More