emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into 2020-10-09-thin-restore-rewrite

Browse Source
This commit is contained in:
Joe Thornber 2021-03-23 11:22:03 +00:00
commit 12c1c6e1f5
73 changed files with 2577 additions and 552 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Cargo.lock generated
View File

@ -671,6 +671,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "safemem"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
[[package]]
name = "shared_child"
version = "0.3.4"
@ -798,11 +804,13 @@ dependencies = [
"quickcheck",
"quickcheck_macros",
"rand",
"safemem",
"tempfile",
"termion",
"thiserror",
"threadpool",
"tui",
"typenum",
]
[[package]]
@ -856,6 +864,12 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "typenum"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
[[package]]
name = "unicode-segmentation"
version = "1.6.0"

2
Cargo.toml
View File

@ -27,11 +27,13 @@ num-derive = "0.3"
num-traits = "0.2"
quick-xml = "0.18"
rand = "0.7"
safemem = "0.3.3"
tempfile = "3.1"
threadpool = "1.8"
thiserror = "1.0"
tui = "0.10"
termion = "1.5"
typenum = "1.12.0"
[dev-dependencies]
json = "0.12"

94
Makefile.in
View File

@ -22,6 +22,7 @@ PROGRAMS=\
bin/pdata_tools
ifeq ("@TESTING@", "yes")
PROGRAMS += bin/pdata_tools_dev
TESTLIBS=\
lib/libft.so
endif
@ -31,7 +32,7 @@ all: $(PROGRAMS) $(TESTLIBS)
include contrib/Makefile
SOURCE=\
COMMON_SOURCE=\
base/output_file_requirements.cc \
base/application.cc \
base/base64.cc \
@ -50,13 +51,6 @@ SOURCE=\
block-cache/copier.cc \
block-cache/io_engine.cc \
block-cache/mem_pool.cc \
caching/cache_check.cc \
caching/cache_dump.cc \
caching/cache_metadata_size.cc \
caching/cache_repair.cc \
caching/cache_restore.cc \
caching/cache_writeback.cc \
caching/commands.cc \
caching/hint_array.cc \
caching/mapping_array.cc \
caching/metadata.cc \
@ -64,13 +58,8 @@ SOURCE=\
caching/restore_emitter.cc \
caching/superblock.cc \
caching/xml_format.cc \
era/commands.cc \
era/era_array.cc \
era/era_check.cc \
era/era_detail.cc \
era/era_dump.cc \
era/era_invalidate.cc \
era/era_restore.cc \
era/metadata.cc \
era/metadata_dump.cc \
era/restore_emitter.cc \
@ -95,7 +84,6 @@ SOURCE=\
persistent-data/space_map.cc \
persistent-data/transaction_manager.cc \
persistent-data/validators.cc \
thin-provisioning/commands.cc \
thin-provisioning/cache_stream.cc \
thin-provisioning/chunk_stream.cc \
thin-provisioning/device_tree.cc \
@ -111,37 +99,64 @@ SOURCE=\
thin-provisioning/restore_emitter.cc \
thin-provisioning/rmap_visitor.cc \
thin-provisioning/superblock.cc \
thin-provisioning/xml_format.cc
TOOLS_SOURCE=\
caching/commands.cc \
caching/cache_check.cc \
caching/cache_dump.cc \
caching/cache_metadata_size.cc \
caching/cache_repair.cc \
caching/cache_restore.cc \
caching/cache_writeback.cc \
era/commands.cc \
era/era_check.cc \
era/era_dump.cc \
era/era_invalidate.cc \
era/era_restore.cc \
thin-provisioning/commands.cc \
thin-provisioning/thin_check.cc \
thin-provisioning/thin_delta.cc \
thin-provisioning/thin_dump.cc \
thin-provisioning/thin_ls.cc \
thin-provisioning/thin_metadata_size.cc \
thin-provisioning/thin_pool.cc \
thin-provisioning/thin_repair.cc \
thin-provisioning/thin_restore.cc \
thin-provisioning/thin_rmap.cc \
thin-provisioning/thin_trim.cc \
thin-provisioning/xml_format.cc
thin-provisioning/thin_trim.cc
DEVTOOLS_SOURCE=\
caching/cache_debug.cc \
caching/devel_commands.cc \
dbg-lib/bitset_block_dumper.cc \
dbg-lib/command_interpreter.cc \
dbg-lib/commands.cc \
dbg-lib/index_block_dumper.cc \
dbg-lib/output_formatter.cc \
dbg-lib/simple_show_traits.cc \
dbg-lib/sm_show_traits.cc \
era/devel_commands.cc \
era/era_debug.cc \
thin-provisioning/damage_generator.cc \
thin-provisioning/devel_commands.cc \
thin-provisioning/thin_debug.cc \
thin-provisioning/thin_generate_damage.cc \
thin-provisioning/thin_generate_mappings.cc \
thin-provisioning/thin_generate_metadata.cc \
thin-provisioning/thin_journal.cc \
thin-provisioning/thin_journal_check.cc \
thin-provisioning/thin_ll_dump.cc \
thin-provisioning/thin_ll_restore.cc \
thin-provisioning/thin_show_duplicates.cc \
thin-provisioning/thin_generate_damage.cc \
thin-provisioning/thin_generate_metadata.cc \
thin-provisioning/thin_generate_mappings.cc \
thin-provisioning/variable_chunk_stream.cc \
thin-provisioning/thin_patch_superblock.cc \
thin-provisioning/thin_show_metadata.cc \
thin-provisioning/thin_pool.cc \
thin-provisioning/thin_scan.cc \
thin-provisioning/thin_show_duplicates.cc \
thin-provisioning/thin_show_metadata.cc \
thin-provisioning/variable_chunk_stream.cc \
ui/ui.cc
ifeq ("@DEVTOOLS@", "yes")
SOURCE+=$(DEVTOOLS_SOURCE)
endif
SOURCE=$(COMMON_SOURCE) $(TOOLS_SOURCE)
DEV_SOURCE=$(COMMON_SOURCE) $(DEVTOOLS_SOURCE)
ifeq ("@STATIC@", "yes")
SOURCE += thin-provisioning/static_library_emitter.cc
@ -154,6 +169,7 @@ CXX:=@CXX@
AR:=@AR@
STRIP:=@STRIP@
OBJECTS:=$(subst .cc,.o,$(SOURCE))
DEV_OBJECTS:=$(subst .cc,.o,$(DEV_SOURCE))
ifeq ("@STATIC@", "yes")
EMITTERS += $(PLUGIN_LIBS)
@ -165,20 +181,13 @@ CFLAGS+=-g -Wall -O3 -fPIC
CFLAGS+=@LFS_FLAGS@
CXXFLAGS+=-g -Wall -fPIC -fno-strict-aliasing -std=c++11
ifeq ("@DEVTOOLS@", "yes")
CXXFLAGS+=-DDEV_TOOLS
endif
CXXFLAGS+=@CXXOPTIMISE_FLAG@
CXXFLAGS+=@CXXDEBUG_FLAG@
CXXFLAGS+=@CXX_STRERROR_FLAG@
CXXFLAGS+=@LFS_FLAGS@
INCLUDES+=-I$(TOP_BUILDDIR) -I$(TOP_DIR) -I$(TOP_DIR)/thin-provisioning
LIBS:=-laio -lexpat -lboost_iostreams -ldl
ifeq ("@DEVTOOLS@", "yes")
LIBS+=-lncurses
endif
DEV_LIBS:=-lncurses
ifeq ("@STATIC_CXX@", "yes")
CXXLIB+=-Wl,-Bstatic -lstdc++ -Wl,-Bdynamic -Wl,--as-needed
@ -246,10 +255,16 @@ bin/pdata_tools: $(OBJECTS) $(EMITTERS)
@mkdir -p $(dir $@)
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(CXXLIB)
bin/pdata_tools_dev: $(DEV_OBJECTS)
@echo " [LD] $@"
@mkdir -p $(dir $@)
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(DEV_LIBS) $(CXXLIB)
#----------------------------------------------------------------
DEPEND_FILES=\
$(subst .cc,.d,$(SOURCE)) \
$(subst .cc,.d,$(DEV_SOURCE)) \
$(subst .cc,.d,$(TEST_SOURCE)) \
$(subst .cc,.d,$(CXX_PROGRAM_SOURCE)) \
$(subst .c,.d,$(C_PROGRAM_SOURCE))
@ -260,7 +275,7 @@ clean:
find . -name \*.o -delete
find . -name \*.gmo -delete
find . -name \*.d -delete
$(RM) $(TEST_PROGRAMS) $(PROGRAMS) $(GMOCK_OBJECTS) lib/*.a lib/*.so
$(RM) $(PROGRAMS) lib/*.a lib/*.so
distclean: clean
$(RM) config.cache config.log config.status configure.h version.h Makefile unit-tests/Makefile
@ -331,13 +346,6 @@ install: bin/pdata_tools $(MANPAGES)
$(INSTALL_DATA) man8/era_restore.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/era_invalidate.8 $(MANPATH)/man8
$(INSTALL_DATA) man8/thin_trim.8 $(MANPATH)/man8
ifeq ("@DEVTOOLS@", "yes")
ln -s -f pdata_tools $(BINDIR)/thin_show_duplicates
ln -s -f pdata_tools $(BINDIR)/thin_ll_dump
ln -s -f pdata_tools $(BINDIR)/thin_show_duplicates
ln -s -f pdata_tools $(BINDIR)/thin_generate_metadata
ln -s -f pdata_tools $(BINDIR)/thin_scan
endif
.PHONY: install install-rust-tools rust-tools
@ -368,7 +376,7 @@ lib/libft.so: $(LIBFT_OBJECTS)
.PHONEY: functional-test unit-test
functional-test: bin/pdata_tools lib/libft.so
functional-test: $(PROGRAMS) $(TESTLIBS)
cd functional-tests && ./run-tests run
test: functional-test unit-test

2
base/application.cc
View File

@ -47,7 +47,7 @@ application::run(int argc, char **argv)
{
string cmd = get_basename(argv[0]);
if (cmd == string("pdata_tools")) {
if (cmd.find("pdata_tools") == 0) {
argc--;
argv++;

2
bin/thin_generate_damage
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

2
bin/thin_generate_mappings
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

2
bin/thin_generate_metadata
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

2
bin/thin_journal_check
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

2
bin/thin_ll_dump
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

2
bin/thin_ll_restore
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

2
bin/thin_scan
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

2
bin/thin_show_duplicates
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

2
bin/thin_show_metadata
View File

@ -1 +1 @@
pdata_tools
pdata_tools_dev

22
block-cache/copier.cc
View File

@ -10,12 +10,14 @@ using namespace std;
copier::copier(io_engine &engine,
string const &src, string const &dest,
sector_t block_size, size_t mem)
sector_t block_size, size_t mem,
sector_t src_offset, sector_t dest_offset)
: pool_(block_size * 512, mem, PAGE_SIZE),
block_size_(block_size),
engine_(engine),
src_handle_(engine_.open_file(src, io_engine::M_READ_ONLY)),
dest_handle_(engine_.open_file(dest, io_engine::M_READ_WRITE)),
src_offset_(src_offset), dest_offset_(dest_offset),
genkey_count_(0)
{
}
@ -45,8 +47,8 @@ copier::issue(copy_op const &op)
auto r = engine_.issue_io(src_handle_,
io_engine::D_READ,
to_sector(op.src_b),
to_sector(op.src_e),
to_src_sector(op.src_b),
to_src_sector(op.src_e),
data,
key);
@ -151,8 +153,8 @@ copier::wait_successful(io_engine::wait_result const &p)
j.op.read_complete = true;
if (!engine_.issue_io(dest_handle_,
io_engine::D_WRITE,
to_sector(j.op.dest_b),
to_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)),
to_dest_sector(j.op.dest_b),
to_dest_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)),
j.data,
it->first)) {
complete(j);
@ -177,9 +179,15 @@ copier::complete(copy_job const &j)
}
sector_t
copier::to_sector(block_address b) const
copier::to_src_sector(block_address b) const
{
return b * block_size_;
return src_offset_ + b * block_size_;
}
sector_t
copier::to_dest_sector(block_address b) const
{
return dest_offset_ + b * block_size_;
}
unsigned

16
block-cache/copier.h
View File

@ -61,13 +61,22 @@ namespace bcache {
public:
copier(io_engine &engine,
std::string const &src, std::string const &dest,
sector_t block_size, size_t mem);
sector_t block_size, size_t mem,
sector_t src_offset, sector_t dest_offset);
~copier();
sector_t get_block_size() const {
return block_size_;
}
sector_t get_src_offset() const {
return src_offset_;
}
sector_t get_dest_offset() const {
return dest_offset_;
}
// Blocks if out of memory.
void issue(copy_op const &op);
@ -83,7 +92,8 @@ namespace bcache {
void wait_();
void complete(copy_job const &j);
sector_t to_sector(block_address b) const;
sector_t to_src_sector(block_address b) const;
sector_t to_dest_sector(block_address b) const;
unsigned genkey();
mempool pool_;
@ -91,6 +101,8 @@ namespace bcache {
io_engine &engine_;
io_engine::handle src_handle_;
io_engine::handle dest_handle_;
sector_t src_offset_;
sector_t dest_offset_;
unsigned genkey_count_;
using job_map = std::map<unsigned, copy_job>;

231
caching/cache_debug.cc Normal file
View File

@ -0,0 +1,231 @@
// Copyright (C) 2012 Red Hat, Inc. All rights reserved.
//
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include <boost/lexical_cast.hpp>
#include <getopt.h>
#include <iostream>
#include <string>
#include "dbg-lib/array_block_dumper.h"
#include "dbg-lib/btree_node_dumper.h"
#include "dbg-lib/bitset_block_dumper.h"
#include "dbg-lib/command_interpreter.h"
#include "dbg-lib/commands.h"
#include "dbg-lib/index_block_dumper.h"
#include "dbg-lib/output_formatter.h"
#include "dbg-lib/sm_show_traits.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "caching/commands.h"
#include "caching/metadata.h"
#include "version.h"
using namespace dbg;
using namespace persistent_data;
using namespace std;
using namespace caching;
//----------------------------------------------------------------
namespace {
class help : public dbg::command {
virtual void exec(strings const &args, ostream &out) {
out << "Commands:" << endl
<< " superblock [block#]" << endl
<< " block_node <block# of array block-tree node>" << endl
<< " bitset_block <block# of bitset block>" << endl
<< " index_block <block# of metadata space map root>" << endl
<< " mapping_block <block# of mappings array block>" << endl
<< " exit" << endl;
}
};
class show_superblock : public dbg::command {
public:
explicit show_superblock(block_manager::ptr bm)
: bm_(bm) {
}
virtual void exec(strings const &args, ostream &out) {
if (args.size() > 2)
throw runtime_error("incorrect number of arguments");
block_address b = caching::SUPERBLOCK_LOCATION;
if (args.size() == 2)
b = boost::lexical_cast<block_address>(args[1]);
caching::superblock sb = read_superblock(bm_, b);
formatter::ptr f = create_xml_formatter();
ostringstream version;
field(*f, "csum", sb.csum);
field(*f, "flags", sb.flags.encode());
field(*f, "blocknr", sb.blocknr);
field(*f, "uuid", sb.uuid); // FIXME: delimit, and handle non-printable chars
field(*f, "magic", sb.magic);
field(*f, "version", sb.version);
field(*f, "policy_name", reinterpret_cast<char const*>(sb.policy_name));
version << sb.policy_version[0] << "."
<< sb.policy_version[1] << "."
<< sb.policy_version[2];
field(*f, "policy_version", version.str().c_str());
field(*f, "policy_hint_size", sb.policy_hint_size);
sm_disk_detail::sm_root_disk const *d;
sm_disk_detail::sm_root v;
{
d = reinterpret_cast<sm_disk_detail::sm_root_disk const *>(sb.metadata_space_map_root);
sm_disk_detail::sm_root_traits::unpack(*d, v);
formatter::ptr f2 = create_xml_formatter();
sm_root_show_traits::show(f2, "value", v);
f->child("metadata_space_map_root", f2);
}
field(*f, "mapping_root", sb.mapping_root);
if (sb.version >= 2)
field(*f, "dirty_root", *sb.dirty_root);
field(*f, "hint_root", sb.hint_root);
field(*f, "discard_root", sb.discard_root);
field(*f, "discard_block_size", sb.discard_block_size);
field(*f, "discard_nr_blocks", sb.discard_nr_blocks);
field(*f, "data_block_size", sb.data_block_size);
field(*f, "metadata_block_size", sb.metadata_block_size);
field(*f, "cache_blocks", sb.cache_blocks);
field(*f, "compat_flags", sb.compat_flags);
field(*f, "compat_ro_flags", sb.compat_ro_flags);
field(*f, "incompat_flags", sb.incompat_flags);
field(*f, "read_hits", sb.read_hits);
field(*f, "read_misses", sb.read_misses);
field(*f, "write_hits", sb.write_hits);
field(*f, "write_misses", sb.write_misses);
f->output(out, 0);
}
private:
block_manager::ptr bm_;
};
class mapping_show_traits : public caching::mapping_traits {
public:
typedef mapping_traits value_trait;
static void show(formatter::ptr f, string const &key, caching::mapping const &value) {
field(*f, "oblock", value.oblock_);
field(*f, "flags", value.flags_);
}
};
//--------------------------------
template <typename ShowTraits>
dbg::command::ptr
create_btree_node_handler(block_manager::ptr bm) {
return create_block_handler(bm, create_btree_node_dumper<ShowTraits>());
}
template <typename ShowTraits>
dbg::command::ptr
create_array_block_handler(block_manager::ptr bm,
typename ShowTraits::value_trait::ref_counter rc) {
return create_block_handler(bm, create_array_block_dumper<ShowTraits>(rc));
}
dbg::command::ptr
create_bitset_block_handler(block_manager::ptr bm) {
return create_block_handler(bm, create_bitset_block_dumper());
}
dbg::command::ptr
create_index_block_handler(block_manager::ptr bm) {
return create_block_handler(bm, create_index_block_dumper());
}
int debug(string const &path) {
using dbg::command;
try {
block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY);
command_interpreter::ptr interp = create_command_interpreter(cin, cout);
interp->register_command("hello", create_hello_handler());
interp->register_command("superblock", command::ptr(new show_superblock(bm)));
interp->register_command("block_node", create_btree_node_handler<uint64_show_traits>(bm));
interp->register_command("bitset_block", create_bitset_block_handler(bm));
interp->register_command("index_block", create_index_block_handler(bm));
interp->register_command("mapping_block", create_array_block_handler<mapping_show_traits>(bm,
mapping_traits::ref_counter()));
interp->register_command("help", command::ptr(new help));
interp->register_command("exit", create_exit_handler(interp));
interp->enter_main_loop();
} catch (std::exception &e) {
cerr << e.what();
return 1;
}
return 0;
}
}
//----------------------------------------------------------------
cache_debug_cmd::cache_debug_cmd()
: command("cache_debug")
{
}
void
cache_debug_cmd::usage(std::ostream &out) const
{
out << "Usage: " << get_name() << " {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-V|--version}" << endl;
}
int
cache_debug_cmd::run(int argc, char **argv)
{
int c;
const char shortopts[] = "hV";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'V':
cerr << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
}
}
if (argc == optind) {
usage(cerr);
exit(1);
}
return debug(argv[optind]);
}
//----------------------------------------------------------------

27
caching/cache_writeback.cc
View File

@ -31,6 +31,8 @@ namespace {
flags()
: cache_size(4 * 1024 * 1024),
sort_buffers(16 * 1024),
origin_dev_offset(0),
fast_dev_offset(0),
list_failed_blocks(false),
update_metadata(true) {
}
@ -52,6 +54,8 @@ namespace {
maybe_string metadata_dev;
maybe_string origin_dev;
maybe_string fast_dev;
sector_t origin_dev_offset;
sector_t fast_dev_offset;
bool list_failed_blocks;
bool update_metadata;
};
@ -303,7 +307,9 @@ namespace {
unsigned max_ios = f.cache_size / (md.sb_.data_block_size << SECTOR_SHIFT);
aio_engine engine(max_ios);
copier c(engine, *f.fast_dev, *f.origin_dev,
md.sb_.data_block_size, f.cache_size);
md.sb_.data_block_size, f.cache_size,
f.fast_dev_offset >> SECTOR_SHIFT,
f.origin_dev_offset >> SECTOR_SHIFT);
auto bar = create_progress_bar("Copying data");
copy_visitor cv(c, f.sort_buffers, clean_shutdown(md), f.list_failed_blocks,
@ -364,6 +370,8 @@ cache_writeback_cmd::usage(std::ostream &out) const
<< "\t\t--buffer-size-meg <size>\n"
<< "\t\t--list-failed-blocks\n"
<< "\t\t--no-metadata-update\n"
<< "\t\t--origin-device-offset <bytes>\n"
<< "\t\t--fast-device-offset <bytes>\n"
<< "Options:\n"
<< " {-h|--help}\n"
<< " {-V|--version}" << endl;
@ -382,6 +390,8 @@ cache_writeback_cmd::run(int argc, char **argv)
{ "buffer-size-meg", required_argument, NULL, 3 },
{ "list-failed-blocks", no_argument, NULL, 4 },
{ "no-metadata-update", no_argument, NULL, 5 },
{ "origin-device-offset", required_argument, NULL, 6 },
{ "fast-device-offset", required_argument, NULL, 7 },
{ "help", no_argument, NULL, 'h'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
@ -413,6 +423,14 @@ cache_writeback_cmd::run(int argc, char **argv)
fs.update_metadata = false;
break;
case 6:
fs.origin_dev_offset = parse_uint64(optarg, "origin dev offset");
break;
case 7:
fs.fast_dev_offset = parse_uint64(optarg, "fast dev offset");
break;
case 'h':
usage(cout);
return 0;
@ -452,6 +470,13 @@ cache_writeback_cmd::run(int argc, char **argv)
return 1;
}
if (fs.origin_dev_offset & (SECTOR_SHIFT - 1) ||
fs.fast_dev_offset & (SECTOR_SHIFT - 1)) {
cerr << "Offset must be sector-aligned\n\n";
usage(cerr);
return 1;
}
return writeback(fs);
}

12
caching/commands.h
View File

@ -70,6 +70,18 @@ namespace caching {
virtual int run(int argc, char **argv);
};
//------------------------------------------------------
class cache_debug_cmd : public base::command {
public:
cache_debug_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
//------------------------------------------------------
void register_cache_commands(base::application &app);
}

14
caching/devel_commands.cc Normal file
View File

@ -0,0 +1,14 @@
#include "caching/commands.h"
using namespace base;
using namespace caching;
//----------------------------------------------------------------
void
caching::register_cache_commands(application &app)
{
app.add_cmd(command::ptr(new cache_debug_cmd));
}
//----------------------------------------------------------------

9
configure.ac
View File

@ -150,14 +150,6 @@ AC_ARG_ENABLE(testing,
TESTING=$enableval, TESTING=no)
AC_MSG_RESULT($TESTING)
################################################################################
dnl -- Enable development tools
AC_MSG_CHECKING(whether to enable development tools)
AC_ARG_ENABLE(dev-tools,
AC_HELP_STRING(--enable-dev-tools, [enable development tools in the makefile]),
DEVTOOLS=$enableval, DEVTOOLS=no)
AC_MSG_RESULT($DEVTOOLS)
################################################################################
dnl -- Enable static libstdc++
AC_MSG_CHECKING(whether to statically link libstdc++)
@ -214,6 +206,5 @@ Makefile
contrib/Makefile
unit-tests/Makefile
version.h
src/version.rs
])
AC_OUTPUT

53
dbg-lib/array_block_dumper.h Normal file
View File

@ -0,0 +1,53 @@
#include "dbg-lib/block_dumper.h"
#include "dbg-lib/output_formatter.h"
#include "persistent-data/data-structures/array_block.h"
//----------------------------------------------------------------
namespace dbg {
using persistent_data::block_manager;
using persistent_data::array_block;
template <typename ShowTraits>
class array_block_dumper : public block_dumper {
public:
array_block_dumper(typename ShowTraits::value_trait::ref_counter rc)
: rc_(rc) {
}
virtual void show(block_manager::read_ref &rr, std::ostream &out) {
rblock b(rr, rc_);
show_array_entries(b, out);
}
private:
typedef array_block<typename ShowTraits::value_trait, block_manager::read_ref> rblock;
void show_array_entries(rblock const& b, std::ostream &out) {
formatter::ptr f = create_xml_formatter();
uint32_t nr_entries = b.nr_entries();
field(*f, "max_entries", b.max_entries());
field(*f, "nr_entries", nr_entries);
field(*f, "value_size", b.value_size());
for (unsigned i = 0; i < nr_entries; i++) {
formatter::ptr f2 = create_xml_formatter();
ShowTraits::show(f2, "value", b.get(i));
f->child(boost::lexical_cast<std::string>(i), f2);
}
f->output(out, 0);
}
typename ShowTraits::value_trait::ref_counter rc_;
};
template <typename ShowTraits>
block_dumper::ptr
create_array_block_dumper(typename ShowTraits::value_trait::ref_counter rc) {
return block_dumper::ptr(new array_block_dumper<ShowTraits>(rc));
}
}
//----------------------------------------------------------------

137
dbg-lib/bitset_block_dumper.cc Normal file
View File

@ -0,0 +1,137 @@
#include "dbg-lib/bitset_block_dumper.h"
#include "dbg-lib/output_formatter.h"
#include "persistent-data/data-structures/array_block.h"
#include "persistent-data/data-structures/simple_traits.h"
using namespace dbg;
using namespace persistent_data;
//----------------------------------------------------------------
namespace {
class bitset_block_dumper : public dbg::block_dumper {
typedef array_block<uint64_traits, block_manager::read_ref> rblock;
public:
explicit bitset_block_dumper()
: BITS_PER_ARRAY_ENTRY(64) {
}
virtual void show(block_manager::read_ref &rr, ostream &out) {
rblock b(rr, rc_);
show_bitset_entries(b, out);
}
private:
void show_bitset_entries(rblock const& b, ostream &out) {
formatter::ptr f = create_xml_formatter();
uint32_t nr_entries = b.nr_entries();
field(*f, "max_entries", b.max_entries());
field(*f, "nr_entries", nr_entries);
field(*f, "value_size", b.value_size());
uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY;
std::pair<uint32_t, uint32_t> range = next_set_bits(b, 0);
for (; range.first < end_pos; range = next_set_bits(b, range.second)) {
formatter::ptr f2 = create_xml_formatter();
field(*f2, "begin", range.first);
field(*f2, "end", range.second);
f->child("set_bits", f2);
}
f->output(out, 0);
}
// Returns the range of set bits, starts from the offset.
pair<uint32_t, uint32_t> next_set_bits(rblock const &b, uint32_t offset) {
uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY;
uint32_t begin = find_first_set(b, offset);
if (begin == end_pos) // not found
return make_pair(end_pos, end_pos);
uint32_t end = find_first_unset(b, begin + 1);
return make_pair(begin, end);
}
// Returns the position (zero-based) of the first bit set
// in the array block, starts from the offset.
// Returns the pass-the-end position if not found.
uint32_t find_first_set(rblock const &b, uint32_t offset) {
uint32_t entry = offset / BITS_PER_ARRAY_ENTRY;
uint32_t nr_entries = b.nr_entries();
if (entry >= nr_entries)
return entry * BITS_PER_ARRAY_ENTRY;
uint32_t idx = offset % BITS_PER_ARRAY_ENTRY;
uint64_t v = b.get(entry++) >> idx;
while (!v && entry < nr_entries) {
v = b.get(entry++);
idx = 0;
}
if (!v) // not found
return entry * BITS_PER_ARRAY_ENTRY;
return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + ffsll(static_cast<long long>(v)) - 1;
}
// Returns the position (zero-based) of the first zero bit
// in the array block, starts from the offset.
// Returns the pass-the-end position if not found.
// FIXME: improve efficiency
uint32_t find_first_unset(rblock const& b, uint32_t offset) {
uint32_t entry = offset / BITS_PER_ARRAY_ENTRY;
uint32_t nr_entries = b.nr_entries();
if (entry >= nr_entries)
return entry * BITS_PER_ARRAY_ENTRY;
uint32_t idx = offset % BITS_PER_ARRAY_ENTRY;
uint64_t v = b.get(entry++);
while (all_bits_set(v, idx) && entry < nr_entries) {
v = b.get(entry++);
idx = 0;
}
if (all_bits_set(v, idx)) // not found
return entry * BITS_PER_ARRAY_ENTRY;
return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + count_leading_bits(v, idx);
}
// Returns true if all the bits beyond the position are set.
bool all_bits_set(uint64_t v, uint32_t offset) {
return (v >> offset) == (numeric_limits<uint64_t>::max() >> offset);
}
// Counts the number of leading 1's in the given value, starts from the offset
// FIXME: improve efficiency
uint32_t count_leading_bits(uint64_t v, uint32_t offset) {
uint32_t count = 0;
v >>= offset;
while (v & 0x1) {
v >>= 1;
count++;
}
return count;
}
block_manager::ptr bm_;
uint64_traits::ref_counter rc_;
const uint32_t BITS_PER_ARRAY_ENTRY;
};
}
//----------------------------------------------------------------
block_dumper::ptr
dbg::create_bitset_block_dumper() {
return block_dumper::ptr(new bitset_block_dumper());
}
//----------------------------------------------------------------

14
dbg-lib/bitset_block_dumper.h Normal file
View File

@ -0,0 +1,14 @@
#ifndef DBG_BITSET_BLOCK_DUMPER
#define DBG_BITSET_BLOCK_DUMPER
#include "dbg-lib/block_dumper.h"
//----------------------------------------------------------------
namespace dbg {
block_dumper::ptr create_bitset_block_dumper();
}
//----------------------------------------------------------------
#endif

21
dbg-lib/block_dumper.h Normal file
View File

@ -0,0 +1,21 @@
#ifndef DBG_BLOCK_DUMPER_H
#define DBG_BLOCK_DUMPER_H
#include "persistent-data/block.h"
//----------------------------------------------------------------
namespace dbg {
class block_dumper {
public:
typedef std::shared_ptr<block_dumper> ptr;
// pass the read_ref by reference since the caller already held the ref-count
virtual void show(persistent_data::block_manager::read_ref &rr,
std::ostream &out) = 0;
};
}
//----------------------------------------------------------------
#endif

57
dbg-lib/btree_node_dumper.h Normal file
View File

@ -0,0 +1,57 @@
#ifndef DBG_BTREE_NODE_DUMPER
#define DBG_BTREE_NODE_DUMPER
#include "dbg-lib/block_dumper.h"
#include "dbg-lib/simple_show_traits.h"
#include "persistent-data/data-structures/btree.h"
//----------------------------------------------------------------
namespace dbg {
using persistent_data::block_manager;
using persistent_data::btree_detail::node_ref;
template <typename ShowTraits>
class btree_node_dumper : public block_dumper {
public:
virtual void show(block_manager::read_ref &rr, std::ostream &out) {
node_ref<uint64_traits> n = btree_detail::to_node<uint64_traits>(rr);
if (n.get_type() == INTERNAL)
btree_node_dumper<uint64_show_traits>::show_node(n, out);
else {
node_ref<typename ShowTraits::value_trait> n = btree_detail::to_node<typename ShowTraits::value_trait>(rr);
show_node(n, out);
}
}
static void show_node(node_ref<typename ShowTraits::value_trait> n, std::ostream &out) {
formatter::ptr f = create_xml_formatter();
field(*f, "csum", n.get_checksum());
field(*f, "blocknr", n.get_block_nr());
field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf");
field(*f, "nr_entries", n.get_nr_entries());
field(*f, "max_entries", n.get_max_entries());
field(*f, "value_size", n.get_value_size());
for (unsigned i = 0; i < n.get_nr_entries(); i++) {
formatter::ptr f2 = create_xml_formatter();
field(*f2, "key", n.key_at(i));
ShowTraits::show(f2, "value", n.value_at(i));
f->child(boost::lexical_cast<std::string>(i), f2);
}
f->output(out, 0);
}
};
template <typename ShowTraits>
block_dumper::ptr
create_btree_node_dumper() {
return block_dumper::ptr(new btree_node_dumper<ShowTraits>());
}
}
//----------------------------------------------------------------
#endif

88
dbg-lib/command_interpreter.cc Normal file
View File

@ -0,0 +1,88 @@
#include "dbg-lib/command_interpreter.h"
using namespace dbg;
using namespace std;
//----------------------------------------------------------------
namespace {
class command_interpreter_impl : public command_interpreter {
public:
typedef std::shared_ptr<command_interpreter> ptr;
command_interpreter_impl(std::istream &in, std::ostream &out)
: in_(in),
out_(out),
exit_(false) {
}
void register_command(std::string const &str, command::ptr cmd) {
commands_.insert(make_pair(str, cmd));
}
void enter_main_loop() {
while (!exit_)
do_once();
}
void exit_main_loop() {
exit_ = true;
}
private:
void do_once();
std::istream &in_;
std::ostream &out_;
std::map <std::string, command::ptr> commands_;
bool exit_;
};
//--------------------------------
strings read_input(std::istream &in)
{
using namespace boost::algorithm;
std::string input;
getline(in, input);
strings toks;
split(toks, input, is_any_of(" \t"), token_compress_on);
return toks;
}
}
//----------------------------------------------------------------
void command_interpreter_impl::do_once()
{
if (in_.eof())
throw runtime_error("input closed");
out_ << "> ";
strings args = read_input(in_);
std::map<std::string, command::ptr>::iterator it;
it = commands_.find(args[0]);
if (it == commands_.end())
out_ << "Unrecognised command" << endl;
else {
try {
it->second->exec(args, out_);
} catch (std::exception &e) {
cerr << e.what() << endl;
}
}
}
//----------------------------------------------------------------
command_interpreter::ptr
dbg::create_command_interpreter(std::istream &in, std::ostream &out)
{
return command_interpreter::ptr(new command_interpreter_impl(in, out));
}
//----------------------------------------------------------------

40
dbg-lib/command_interpreter.h Normal file
View File

@ -0,0 +1,40 @@
#ifndef DBG_COMMAND_INTERPRETER
#define DBG_COMMAND_INTERPRETER
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>
#include <iostream>
#include <map>
#include <string>
#include <vector>
//----------------------------------------------------------------
namespace dbg {
typedef std::vector<std::string> strings;
class command {
public:
typedef std::shared_ptr<command> ptr;
virtual ~command() {}
virtual void exec(strings const &args, std::ostream &out) = 0;
};
class command_interpreter {
public:
typedef std::shared_ptr<command_interpreter> ptr;
virtual void register_command(std::string const &str, command::ptr cmd) = 0;
virtual void enter_main_loop() = 0;
virtual void exit_main_loop() = 0;
};
command_interpreter::ptr
create_command_interpreter(std::istream &in, std::ostream &out);
}
//----------------------------------------------------------------
#endif

79
dbg-lib/commands.cc Normal file
View File

@ -0,0 +1,79 @@
#include "dbg-lib/commands.h"
#include "persistent-data/block.h"
#include <boost/lexical_cast.hpp>
#include <stdexcept>
using namespace dbg;
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
namespace {
class hello : public dbg::command {
virtual void exec(dbg::strings const &args, ostream &out) {
out << "Hello, world!" << endl;
}
};
class exit_handler : public dbg::command {
public:
exit_handler(command_interpreter::ptr interpreter)
: interpreter_(interpreter) {
}
virtual void exec(dbg::strings const &args, ostream &out) {
out << "Goodbye!" << endl;
interpreter_->exit_main_loop();
}
private:
command_interpreter::ptr interpreter_;
};
class block_handler : public dbg::command {
public:
block_handler(block_manager::ptr bm, block_dumper::ptr dumper)
: bm_(bm), dumper_(dumper) {
}
virtual void exec(dbg::strings const &args, ostream &out) {
if (args.size() != 2)
throw runtime_error("incorrect number of arguments");
block_address block = boost::lexical_cast<block_address>(args[1]);
// no checksum validation for debugging purpose
block_manager::read_ref rr = bm_->read_lock(block);
dumper_->show(rr, out);
}
private:
block_manager::ptr bm_;
block_dumper::ptr dumper_;
};
}
//----------------------------------------------------------------
command::ptr
dbg::create_hello_handler()
{
return command::ptr(new hello());
}
command::ptr
dbg::create_exit_handler(command_interpreter::ptr interp)
{
return command::ptr(new exit_handler(interp));
}
command::ptr
dbg::create_block_handler(block_manager::ptr bm, block_dumper::ptr dumper)
{
return command::ptr(new block_handler(bm, dumper));
}
//----------------------------------------------------------------

24
dbg-lib/commands.h Normal file
View File

@ -0,0 +1,24 @@
#ifndef DBG_COMMANDS_H
#define DBG_COMMANDS_H
#include "dbg-lib/command_interpreter.h"
#include "dbg-lib/block_dumper.h"
#include "persistent-data/block.h"
//----------------------------------------------------------------
namespace dbg {
dbg::command::ptr
create_hello_handler();
dbg::command::ptr
create_exit_handler(dbg::command_interpreter::ptr interp);
dbg::command::ptr
create_block_handler(persistent_data::block_manager::ptr bm,
dbg::block_dumper::ptr dumper);
}
//----------------------------------------------------------------
#endif

51
dbg-lib/index_block_dumper.cc Normal file
View File

@ -0,0 +1,51 @@
#include "dbg-lib/index_block_dumper.h"
#include "dbg-lib/output_formatter.h"
#include "dbg-lib/sm_show_traits.h"
#include "persistent-data/space-maps/disk_structures.h"
using namespace dbg;
//----------------------------------------------------------------
namespace {
using persistent_data::block_manager;
class index_block_dumper : public dbg::block_dumper {
public:
virtual void show(block_manager::read_ref &rr, std::ostream &out) {
sm_disk_detail::metadata_index const *mdi =
reinterpret_cast<sm_disk_detail::metadata_index const *>(rr.data());
show_metadata_index(mdi, sm_disk_detail::MAX_METADATA_BITMAPS, out);
}
private:
void show_metadata_index(sm_disk_detail::metadata_index const *mdi, unsigned nr_indexes, std::ostream &out) {
formatter::ptr f = create_xml_formatter();
field(*f, "csum", to_cpu<uint32_t>(mdi->csum_));
field(*f, "padding", to_cpu<uint32_t>(mdi->padding_));
field(*f, "blocknr", to_cpu<uint64_t>(mdi->blocknr_));
sm_disk_detail::index_entry ie;
for (unsigned i = 0; i < nr_indexes; i++) {
sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), ie);
if (!ie.blocknr_ && !ie.nr_free_ && !ie.none_free_before_)
continue;
formatter::ptr f2 = create_xml_formatter();
index_entry_show_traits::show(f2, "value", ie);
f->child(boost::lexical_cast<string>(i), f2);
}
f->output(out, 0);
}
};
}
//----------------------------------------------------------------
block_dumper::ptr
dbg::create_index_block_dumper() {
return block_dumper::ptr(new index_block_dumper());
}
//----------------------------------------------------------------

10
dbg-lib/index_block_dumper.h Normal file
View File

@ -0,0 +1,10 @@
#include "dbg-lib/block_dumper.h"
//----------------------------------------------------------------
namespace dbg {
block_dumper::ptr
create_index_block_dumper();
}
//----------------------------------------------------------------

86
dbg-lib/output_formatter.cc Normal file
View File

@ -0,0 +1,86 @@
#include <string>
#include "dbg-lib/output_formatter.h"
using namespace dbg;
using namespace std;
//----------------------------------------------------------------
namespace {
class abstract_formatter : public formatter {
typedef boost::optional<std::string> maybe_string;
void field(std::string const &name, std::string const &value) {
fields_.push_back(field_type(name, value));
}
void child(std::string const &name, formatter::ptr t) {
children_.push_back(field_type(name, t));
}
protected:
typedef boost::variant<std::string, ptr> value;
typedef boost::tuple<std::string, value> field_type;
std::vector<field_type> fields_;
std::vector<field_type> children_;
};
class xml_formatter : public abstract_formatter {
public:
void output(std::ostream &out, int depth = 0,
boost::optional<std::string> name = boost::none);
};
void indent(int depth, std::ostream &out) {
for (int i = 0; i < depth * 2; i++)
out << ' ';
}
}
//----------------------------------------------------------------
void xml_formatter::output(std::ostream &out,
int depth,
boost::optional<std::string> name) {
indent(depth, out);
out << "<fields";
if (name && (*name).length())
out << " id=\"" << *name << "\"";
/* output non-child fields */
std::vector<field_type>::const_iterator it;
for (it = fields_.begin(); it != fields_.end(); ++it) {
if (string const *s = boost::get<string>(&it->get<1>())) {
out << " " << it->get<0>() << "=\"" << *s << "\"";
}
}
if (children_.size() == 0) {
out << " />" << endl;
return;
}
/* output child fields */
out << ">" << endl;
for (it = children_.begin(); it != children_.end(); ++it) {
if (!boost::get<string>(&it->get<1>())) {
formatter::ptr f = boost::get<formatter::ptr>(it->get<1>());
f->output(out, depth + 1, it->get<0>());
}
}
indent(depth, out);
out << "</fields>" << endl;
}
//----------------------------------------------------------------
formatter::ptr
dbg::create_xml_formatter()
{
return formatter::ptr(new xml_formatter());
}
//----------------------------------------------------------------

37
dbg-lib/output_formatter.h Normal file
View File

@ -0,0 +1,37 @@
#ifndef DBG_OUTPUT_FORMATTER_H
#define DBG_OUTPUT_FORMATTER_H
#include <boost/lexical_cast.hpp>
#include <boost/optional.hpp>
#include <boost/tuple/tuple.hpp>
#include <boost/variant.hpp>
#include <vector>
#include <string>
//----------------------------------------------------------------
namespace dbg {
class formatter {
public:
typedef std::shared_ptr<formatter> ptr;
virtual ~formatter() {}
virtual void field(std::string const &name, std::string const &value) = 0;
virtual void child(std::string const &name, formatter::ptr t) = 0;
virtual void output(std::ostream &out, int depth = 0,
boost::optional<std::string> name = boost::none) = 0;
};
template <typename T>
void
field(formatter &t, std::string const &name, T const &value) {
t.field(name, boost::lexical_cast<std::string>(value));
}
formatter::ptr create_xml_formatter();
}
//----------------------------------------------------------------
#endif

19
dbg-lib/simple_show_traits.cc Normal file
View File

@ -0,0 +1,19 @@
#include "dbg-lib/simple_show_traits.h"
using namespace dbg;
//----------------------------------------------------------------
void
uint32_show_traits::show(formatter::ptr f, string const &key, uint32_t const &value)
{
field(*f, key, boost::lexical_cast<string>(value));
}
void
uint64_show_traits::show(formatter::ptr f, string const &key, uint64_t const &value)
{
field(*f, key, boost::lexical_cast<string>(value));
}
//----------------------------------------------------------------

27
dbg-lib/simple_show_traits.h Normal file
View File

@ -0,0 +1,27 @@
#ifndef DBG_SIMPLE_SHOW_TRAITS_H
#define DBG_SIMPLE_SHOW_TRAITS_H
#include "dbg-lib/output_formatter.h"
#include "persistent-data/data-structures/simple_traits.h"
//----------------------------------------------------------------
namespace dbg {
class uint32_show_traits {
public:
typedef persistent_data::uint32_traits value_trait;
static void show(dbg::formatter::ptr f, std::string const &key, uint32_t const &value);
};
class uint64_show_traits {
public:
typedef persistent_data::uint64_traits value_trait;
static void show(dbg::formatter::ptr f, std::string const &key, uint64_t const &value);
};
}
//----------------------------------------------------------------
#endif

27
dbg-lib/sm_show_traits.cc Normal file
View File

@ -0,0 +1,27 @@
#include "dbg-lib/sm_show_traits.h"
using namespace dbg;
using namespace std;
//----------------------------------------------------------------
void
index_entry_show_traits::show(formatter::ptr f, string const &key,
persistent_data::sm_disk_detail::index_entry const &value)
{
field(*f, "blocknr", value.blocknr_);
field(*f, "nr_free", value.nr_free_);
field(*f, "none_free_before", value.none_free_before_);
}
void
sm_root_show_traits::show(formatter::ptr f, string const &key,
persistent_data::sm_disk_detail::sm_root const &value)
{
field(*f, "nr_blocks", value.nr_blocks_);
field(*f, "nr_allocated", value.nr_allocated_);
field(*f, "bitmap_root", value.bitmap_root_);
field(*f, "ref_count_root", value.ref_count_root_);
}
//----------------------------------------------------------------

29
dbg-lib/sm_show_traits.h Normal file
View File

@ -0,0 +1,29 @@
#ifndef DBG_SM_SHOW_TRAITS_H
#define DBG_SM_SHOW_TRAITS_H
#include "dbg-lib/output_formatter.h"
#include "persistent-data/space-maps/disk_structures.h"
//----------------------------------------------------------------
namespace dbg {
class index_entry_show_traits {
public:
typedef persistent_data::sm_disk_detail::index_entry_traits value_trait;
static void show(dbg::formatter::ptr f, std::string const &key,
persistent_data::sm_disk_detail::index_entry const &value);
};
class sm_root_show_traits {
public:
typedef persistent_data::sm_disk_detail::sm_root_traits value_trait;
static void show(dbg::formatter::ptr f, std::string const &key,
persistent_data::sm_disk_detail::sm_root const &value);
};
}
//----------------------------------------------------------------
#endif

12
era/commands.h
View File

@ -34,6 +34,18 @@ namespace era {
virtual int run(int argc, char **argv);
};
//------------------------------------------------------
class era_debug_cmd : public base::command {
public:
era_debug_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
//------------------------------------------------------
void register_era_commands(base::application &app);
}

14
era/devel_commands.cc Normal file
View File

@ -0,0 +1,14 @@
#include "era/commands.h"
using namespace base;
using namespace era;
//----------------------------------------------------------------
void
era::register_era_commands(base::application &app)
{
app.add_cmd(command::ptr(new era_debug_cmd));
}
//----------------------------------------------------------------

228
era/era_debug.cc Normal file
View File

@ -0,0 +1,228 @@
// Copyright (C) 2012 Red Hat, Inc. All rights reserved.
//
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include <boost/lexical_cast.hpp>
#include <getopt.h>
#include <iostream>
#include <string>
#include "dbg-lib/array_block_dumper.h"
#include "dbg-lib/btree_node_dumper.h"
#include "dbg-lib/bitset_block_dumper.h"
#include "dbg-lib/command_interpreter.h"
#include "dbg-lib/commands.h"
#include "dbg-lib/index_block_dumper.h"
#include "dbg-lib/output_formatter.h"
#include "dbg-lib/sm_show_traits.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "era/commands.h"
#include "era/metadata.h"
#include "version.h"
using namespace dbg;
using namespace persistent_data;
using namespace std;
using namespace era;
//----------------------------------------------------------------
namespace {
class help : public dbg::command {
virtual void exec(strings const &args, ostream &out) {
out << "Commands:" << endl
<< " superblock [block#]" << endl
<< " block_node <block# of array block-tree node>" << endl
<< " bitset_block <block# of bitset block>" << endl
<< " era_block <block# of era array block>" << endl
<< " index_block <block# of metadata space map root>" << endl
<< " writeset_node <block# of writeset tree node>" << endl
<< " exit" << endl;
}
};
// for displaying the writeset tree
class writeset_show_traits : public era::era_detail_traits {
public:
typedef era_detail_traits value_trait;
static void show(formatter::ptr f, string const &key, era_detail const &value) {
field(*f, "nr_bits", value.nr_bits);
field(*f, "writeset_root", value.writeset_root);
}
};
class show_superblock : public dbg::command {
public:
explicit show_superblock(block_manager::ptr bm)
: bm_(bm) {
}
virtual void exec(strings const &args, ostream &out) {
if (args.size() > 2)
throw runtime_error("incorrect number of arguments");
block_address b = era::SUPERBLOCK_LOCATION;
if (args.size() == 2)
b = boost::lexical_cast<block_address>(args[1]);
era::superblock sb = read_superblock(bm_, b);
formatter::ptr f = create_xml_formatter();
ostringstream version;
field(*f, "csum", sb.csum);
field(*f, "flags", sb.flags.encode());
field(*f, "blocknr", sb.blocknr);
field(*f, "uuid", sb.uuid); // FIXME: delimit, and handle non-printable chars
field(*f, "magic", sb.magic);
field(*f, "version", sb.version);
sm_disk_detail::sm_root_disk const *d;
sm_disk_detail::sm_root v;
{
d = reinterpret_cast<sm_disk_detail::sm_root_disk const *>(sb.metadata_space_map_root);
sm_disk_detail::sm_root_traits::unpack(*d, v);
formatter::ptr f2 = create_xml_formatter();
sm_root_show_traits::show(f2, "value", v);
f->child("metadata_space_map_root", f2);
}
field(*f, "data_block_size", sb.data_block_size);
field(*f, "metadata_block_size", sb.metadata_block_size);
field(*f, "nr_blocks", sb.nr_blocks);
field(*f, "current_era", sb.current_era);
{
formatter::ptr f2 = create_xml_formatter();
writeset_show_traits::show(f2, "value", sb.current_detail);
f->child("current_writeset", f2);
}
field(*f, "writeset_tree_root", sb.writeset_tree_root);
field(*f, "era_array_root", sb.era_array_root);
if (sb.metadata_snap)
field(*f, "metadata_snap", *sb.metadata_snap);
f->output(out, 0);
}
private:
block_manager::ptr bm_;
};
//--------------------------------
template <typename ShowTraits>
dbg::command::ptr
create_btree_node_handler(block_manager::ptr bm) {
return create_block_handler(bm, create_btree_node_dumper<ShowTraits>());
}
template <typename ShowTraits>
dbg::command::ptr
create_array_block_handler(block_manager::ptr bm,
typename ShowTraits::value_trait::ref_counter rc) {
return create_block_handler(bm, create_array_block_dumper<ShowTraits>(rc));
}
dbg::command::ptr
create_bitset_block_handler(block_manager::ptr bm) {
return create_block_handler(bm, create_bitset_block_dumper());
}
dbg::command::ptr
create_index_block_handler(block_manager::ptr bm) {
return create_block_handler(bm, create_index_block_dumper());
}
int debug(string const &path) {
using dbg::command;
try {
block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY);
transaction_manager::ptr null_tm = open_tm(bm, era::SUPERBLOCK_LOCATION);
command_interpreter::ptr interp = create_command_interpreter(cin, cout);
interp->register_command("hello", create_hello_handler());
interp->register_command("superblock", command::ptr(new show_superblock(bm)));
interp->register_command("block_node", create_btree_node_handler<uint64_show_traits>(bm));
interp->register_command("bitset_block", create_bitset_block_handler(bm));
interp->register_command("era_block", create_array_block_handler<uint32_show_traits>(bm,
uint32_traits::ref_counter()));
interp->register_command("index_block", create_index_block_handler(bm));
interp->register_command("writeset_node", create_btree_node_handler<writeset_show_traits>(bm));
interp->register_command("help", command::ptr(new help));
interp->register_command("exit", create_exit_handler(interp));
interp->enter_main_loop();
} catch (std::exception &e) {
cerr << e.what();
return 1;
}
return 0;
}
}
//----------------------------------------------------------------
era_debug_cmd::era_debug_cmd()
: command("era_debug")
{
}
void
era_debug_cmd::usage(std::ostream &out) const
{
out << "Usage: " << get_name() << " {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-V|--version}" << endl;
}
int
era_debug_cmd::run(int argc, char **argv)
{
int c;
const char shortopts[] = "hV";
const struct option longopts[] = {
{ "help", no_argument, NULL, 'h'},
{ "version", no_argument, NULL, 'V'},
{ NULL, no_argument, NULL, 0 }
};
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(cout);
return 0;
case 'V':
cerr << THIN_PROVISIONING_TOOLS_VERSION << endl;
return 0;
}
}
if (argc == optind) {
usage(cerr);
exit(1);
}
return debug(argv[optind]);
}
//----------------------------------------------------------------

2
era/era_restore.cc
View File

@ -37,7 +37,7 @@ namespace {
bool metadata_touched = false;
try {
block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE);
file_utils::check_file_exists(*fs.output);
file_utils::check_file_exists(*fs.output, false);
metadata_touched = true;
metadata::ptr md(new metadata(bm, metadata::CREATE));
emitter::ptr restorer = create_restore_emitter(*md);

62
src/bin/cache_check.rs Normal file
View File

@ -0,0 +1,62 @@
extern crate clap;
extern crate thinp;
use clap::{App, Arg};
use std::path::Path;
use thinp::cache::check::{check, CacheCheckOptions};
//------------------------------------------
fn main() {
let parser = App::new("cache_check")
.version(thinp::version::TOOLS_VERSION)
.arg(
Arg::with_name("INPUT")
.help("Specify the input device to check")
.required(true)
.index(1),
)
.arg(
Arg::with_name("SB_ONLY")
.help("Only check the superblock.")
.long("super-block-only")
.value_name("SB_ONLY"),
)
.arg(
Arg::with_name("SKIP_MAPPINGS")
.help("Don't check the mapping array")
.long("skip-mappings")
.value_name("SKIP_MAPPINGS"),
)
.arg(
Arg::with_name("SKIP_HINTS")
.help("Don't check the hint array")
.long("skip-hints")
.value_name("SKIP_HINTS"),
)
.arg(
Arg::with_name("SKIP_DISCARDS")
.help("Don't check the discard bitset")
.long("skip-discards")
.value_name("SKIP_DISCARDS"),
);
let matches = parser.get_matches();
let input_file = Path::new(matches.value_of("INPUT").unwrap());
let opts = CacheCheckOptions {
dev: &input_file,
async_io: false,
sb_only: matches.is_present("SB_ONLY"),
skip_mappings: matches.is_present("SKIP_MAPPINGS"),
skip_hints: matches.is_present("SKIP_HINTS"),
skip_discards: matches.is_present("SKIP_DISCARDS"),
};
if let Err(reason) = check(opts) {
eprintln!("{}", reason);
std::process::exit(1);
}
}
//------------------------------------------

2
src/bin/thin_check.rs
View File

@ -115,7 +115,7 @@ fn main() {
let opts = ThinCheckOptions {
dev: &input_file,
async_io: async_io,
async_io,
sb_only: matches.is_present("SB_ONLY"),
skip_mappings: matches.is_present("SKIP_MAPPINGS"),
ignore_non_fatal: matches.is_present("IGNORE_NON_FATAL"),

60
src/bin/thin_explore.rs
View File

@ -72,7 +72,6 @@ impl Events {
let (tx, rx) = mpsc::channel();
let ignore_exit_key = Arc::new(AtomicBool::new(false));
let input_handle = {
let tx = tx.clone();
let ignore_exit_key = ignore_exit_key.clone();
thread::spawn(move || {
let stdin = io::stdin();
@ -110,6 +109,12 @@ impl Events {
}
}
impl Default for Events {
fn default() -> Self {
Self::new()
}
}
//------------------------------------
fn ls_next(ls: &mut ListState, max: usize) {
@ -158,7 +163,7 @@ impl<'a> StatefulWidget for SBWidget<'a> {
let sb = self.sb;
let flags = ["flags".to_string(), format!("{}", sb.flags)];
let block = ["block".to_string(), format!("{}", sb.block)];
let uuid = ["uuid".to_string(), format!("-")];
let uuid = ["uuid".to_string(), "-".to_string()];
let version = ["version".to_string(), format!("{}", sb.version)];
let time = ["time".to_string(), format!("{}", sb.time)];
let transaction_id = [
@ -208,11 +213,11 @@ impl<'a> StatefulWidget for SBWidget<'a> {
Widget::render(table, chunks[0], buf);
let items = vec![
ListItem::new(Span::raw(format!("Device tree"))),
ListItem::new(Span::raw(format!("Mapping tree")))
let items = vec![
ListItem::new(Span::raw("Device tree".to_string())),
ListItem::new(Span::raw("Mapping tree".to_string())),
];
let items = List::new(items)
.block(Block::default().borders(Borders::ALL).title("Entries"))
.highlight_style(
@ -315,7 +320,7 @@ impl<X: Adjacent, Y: Adjacent> Adjacent for (X, Y) {
fn adjacent_runs<V: Adjacent + Copy>(mut ns: Vec<V>) -> Vec<(V, usize)> {
let mut result = Vec::new();
if ns.len() == 0 {
if ns.is_empty() {
return result;
}
@ -330,13 +335,13 @@ fn adjacent_runs<V: Adjacent + Copy>(mut ns: Vec<V>) -> Vec<(V, usize)> {
current = v;
len += 1;
} else {
result.push((base.clone(), len));
base = v.clone();
current = v.clone();
result.push((base, len));
base = v;
current = v;
len = 1;
}
}
result.push((base.clone(), len));
result.push((base, len));
result
}
@ -344,7 +349,7 @@ fn adjacent_runs<V: Adjacent + Copy>(mut ns: Vec<V>) -> Vec<(V, usize)> {
fn mk_runs<V: Adjacent + Sized + Copy>(keys: &[u64], values: &[V]) -> Vec<((u64, V), usize)> {
let mut pairs = Vec::new();
for (k, v) in keys.iter().zip(values.iter()) {
pairs.push((k.clone(), v.clone()));
pairs.push((*k, *v));
}
adjacent_runs(pairs)
@ -498,7 +503,7 @@ impl Panel for SBPanel {
} else {
Some(PushTopLevel(self.sb.mapping_root))
}
},
}
Key::Char('h') | Key::Left => Some(PopPanel),
_ => None,
}
@ -571,14 +576,14 @@ impl Panel for DeviceDetailPanel {
fn path_action(&mut self, child: u64) -> Option<Action> {
match &self.node {
btree::Node::Internal { values, .. } => {
for i in 0..values.len() {
if values[i] == child {
for (i, v) in values.iter().enumerate() {
if *v == child {
self.state.select(Some(i));
return Some(PushDeviceDetail(child));
}
}
return None;
None
}
btree::Node::Leaf { .. } => None,
}
@ -645,14 +650,14 @@ impl Panel for TopLevelPanel {
fn path_action(&mut self, child: u64) -> Option<Action> {
match &self.node {
btree::Node::Internal { values, .. } => {
for i in 0..values.len() {
if values[i] == child {
for (i, v) in values.iter().enumerate() {
if *v == child {
self.state.select(Some(i));
return Some(PushTopLevel(child));
}
}
return None;
None
}
btree::Node::Leaf { keys, values, .. } => {
for i in 0..values.len() {
@ -662,7 +667,7 @@ impl Panel for TopLevelPanel {
}
}
return None;
None
}
}
}
@ -726,14 +731,14 @@ impl Panel for BottomLevelPanel {
fn path_action(&mut self, child: u64) -> Option<Action> {
match &self.node {
btree::Node::Internal { values, .. } => {
for i in 0..values.len() {
if values[i] == child {
for (i, v) in values.iter().enumerate() {
if *v == child {
self.state.select(Some(i));
return Some(PushBottomLevel(self.thin_id, child));
}
}
return None;
None
}
btree::Node::Leaf { .. } => None,
}
@ -789,7 +794,7 @@ fn explore(path: &Path, node_path: Option<Vec<u64>>) -> Result<()> {
}
} else {
let sb = read_superblock(&engine, 0)?;
panels.push(Box::new(SBPanel::new(sb.clone())));
panels.push(Box::new(SBPanel::new(sb)));
}
let events = Events::new();
@ -826,12 +831,11 @@ fn explore(path: &Path, node_path: Option<Vec<u64>>) -> Result<()> {
if let Event::Input(key) = events.next()? {
match key {
Key::Char('q') => break 'main,
_ => match active_panel.input(key) {
Some(action) => {
_ => {
if let Some(action) = active_panel.input(key) {
perform_action(&mut panels, &engine, action)?;
}
_ => {}
},
}
}
}
}

4
src/bin/thin_shrink.rs
View File

@ -1,3 +1,7 @@
// This work is based on the implementation by Nikhil Kshirsagar which
// can be found here:
// https://github.com/nkshirsagar/thinpool_shrink/blob/split_ranges/thin_shrink.py
extern crate clap;
extern crate thinp;

166
src/cache/check.rs vendored Normal file
View File

@ -0,0 +1,166 @@
use anyhow::anyhow;
use std::marker::PhantomData;
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::collections::*;
use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine};
use crate::cache::hint::*;
use crate::cache::mapping::*;
use crate::cache::superblock::*;
use crate::pdata::array_walker::*;
//------------------------------------------
const MAX_CONCURRENT_IO: u32 = 1024;
//------------------------------------------
struct CheckMappingVisitor {
allowed_flags: u32,
seen_oblocks: Arc<Mutex<BTreeSet<u64>>>,
}
impl CheckMappingVisitor {
fn new(metadata_version: u32) -> CheckMappingVisitor {
let mut flags: u32 = MappingFlags::Valid as u32;
if metadata_version == 1 {
flags |= MappingFlags::Dirty as u32;
}
CheckMappingVisitor {
allowed_flags: flags,
seen_oblocks: Arc::new(Mutex::new(BTreeSet::new())),
}
}
fn seen_oblock(&self, b: u64) -> bool {
let seen_oblocks = self.seen_oblocks.lock().unwrap();
return seen_oblocks.contains(&b);
}
fn record_oblock(&self, b: u64) {
let mut seen_oblocks = self.seen_oblocks.lock().unwrap();
seen_oblocks.insert(b);
}
// FIXME: is it possible to validate flags at an early phase?
// e.g., move to ctor of Mapping?
fn has_unknown_flags(&self, m: &Mapping) -> bool {
return (m.flags & self.allowed_flags) != 0;
}
}
impl ArrayBlockVisitor<Mapping> for CheckMappingVisitor {
fn visit(&self, _index: u64, m: Mapping) -> anyhow::Result<()> {
if !m.is_valid() {
return Ok(());
}
if self.seen_oblock(m.oblock) {
return Err(anyhow!("origin block already mapped"));
}
self.record_oblock(m.oblock);
if !self.has_unknown_flags(&m) {
return Err(anyhow!("unknown flags in mapping"));
}
Ok(())
}
}
//------------------------------------------
struct CheckHintVisitor<Width> {
_not_used: PhantomData<Width>,
}
impl<Width> CheckHintVisitor<Width> {
fn new() -> CheckHintVisitor<Width> {
CheckHintVisitor {
_not_used: PhantomData,
}
}
}
impl<Width: typenum::Unsigned> ArrayBlockVisitor<Hint<Width>> for CheckHintVisitor<Width> {
fn visit(&self, _index: u64, _hint: Hint<Width>) -> anyhow::Result<()> {
// TODO: check hints
Ok(())
}
}
//------------------------------------------
// TODO: ignore_non_fatal, clear_needs_check, auto_repair
pub struct CacheCheckOptions<'a> {
pub dev: &'a Path,
pub async_io: bool,
pub sb_only: bool,
pub skip_mappings: bool,
pub skip_hints: bool,
pub skip_discards: bool,
}
// TODO: thread pool, report
struct Context {
engine: Arc<dyn IoEngine + Send + Sync>,
}
fn mk_context(opts: &CacheCheckOptions) -> anyhow::Result<Context> {
let engine: Arc<dyn IoEngine + Send + Sync>;
if opts.async_io {
engine = Arc::new(AsyncIoEngine::new(
opts.dev,
MAX_CONCURRENT_IO,
false,
)?);
} else {
let nr_threads = std::cmp::max(8, num_cpus::get() * 2);
engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads, false)?);
}
Ok(Context {
engine,
})
}
pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> {
let ctx = mk_context(&opts)?;
let engine = &ctx.engine;
let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?;
if opts.sb_only {
return Ok(());
}
// TODO: factor out into check_mappings()
if !opts.skip_mappings {
let w = ArrayWalker::new(engine.clone(), false);
let c = Box::new(CheckMappingVisitor::new(sb.version));
w.walk(c, sb.mapping_root)?;
if sb.version >= 2 {
// TODO: check dirty bitset
}
}
if !opts.skip_hints && sb.hint_root != 0 {
let w = ArrayWalker::new(engine.clone(), false);
type Width = typenum::U4; // FIXME: check sb.policy_hint_size
let c = Box::new(CheckHintVisitor::<Width>::new());
w.walk(c, sb.hint_root)?;
}
if !opts.skip_discards {
// TODO: check discard bitset
}
Ok(())
}
//------------------------------------------

33
src/cache/hint.rs vendored Normal file
View File

@ -0,0 +1,33 @@
use nom::IResult;
use std::marker::PhantomData;
use std::convert::TryInto;
use crate::pdata::unpack::*;
//------------------------------------------
#[derive(Clone, Copy)]
pub struct Hint<Width> {
pub hint: [u8; 4], // FIXME: support various hint sizes
_not_used: PhantomData<Width>,
}
impl<Width: typenum::Unsigned> Unpack for Hint<Width> {
fn disk_size() -> u32 {
Width::to_u32()
}
// FIXME: support different width
fn unpack(i: &[u8]) -> IResult<&[u8], Hint<Width>> {
let size = Width::to_usize();
Ok((
&i[size..],
Hint {
hint: i[0..size].try_into().unwrap(),
_not_used: PhantomData,
},
))
}
}
//------------------------------------------

50
src/cache/mapping.rs vendored Normal file
View File

@ -0,0 +1,50 @@
use nom::IResult;
use nom::number::complete::*;
use crate::pdata::unpack::*;
//------------------------------------------
static FLAGS_MASK: u64 = (1 << 16) - 1;
//------------------------------------------
pub enum MappingFlags {
Valid = 1,
Dirty = 2,
}
#[derive(Clone, Copy)]
pub struct Mapping {
pub oblock: u64,
pub flags: u32,
}
impl Mapping {
pub fn is_valid(&self) -> bool {
return (self.flags & MappingFlags::Valid as u32) != 0;
}
}
impl Unpack for Mapping {
fn disk_size() -> u32 {
8
}
fn unpack(i: &[u8]) -> IResult<&[u8], Mapping> {
let (i, n) = le_u64(i)?;
let oblock = n >> 16;
let flags = n & FLAGS_MASK;
Ok((
i,
Mapping {
oblock,
flags: flags as u32,
},
))
}
}
//------------------------------------------

4
src/cache/mod.rs vendored
View File

@ -1 +1,5 @@
pub mod check;
pub mod hint;
pub mod mapping;
pub mod superblock;
pub mod xml;

139
src/cache/superblock.rs vendored Normal file
View File

@ -0,0 +1,139 @@
use anyhow::{anyhow, Result};
use crate::io_engine::*;
use nom::{bytes::complete::*, number::complete::*, IResult};
//------------------------------------------
pub const SUPERBLOCK_LOCATION: u64 = 0;
const POLICY_NAME_SIZE: usize = 16;
const SPACE_MAP_ROOT_SIZE: usize = 128;
//------------------------------------------
#[derive(Debug, Clone)]
pub struct SuperblockFlags {
pub needs_check: bool,
}
#[derive(Debug, Clone)]
pub struct Superblock {
pub flags: SuperblockFlags,
pub block: u64,
pub version: u32,
pub policy_name: Vec<u8>,
pub policy_version: Vec<u32>,
pub policy_hint_size: u32,
pub metadata_sm_root: Vec<u8>,
pub mapping_root: u64,
pub dirty_root: Option<u64>, // format 2 only
pub hint_root: u64,
pub discard_root: u64,
pub discard_block_size: u64,
pub discard_nr_blocks: u64,
pub data_block_size: u32,
pub metadata_block_size: u32,
pub cache_blocks: u32,
pub compat_flags: u32,
pub compat_ro_flags: u32,
pub incompat_flags: u32,
pub read_hits: u32,
pub read_misses: u32,
pub write_hits: u32,
pub write_misses: u32,
}
fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
let (i, _csum) = le_u32(data)?;
let (i, flags) = le_u32(i)?;
let (i, block) = le_u64(i)?;
let (i, _uuid) = take(16usize)(i)?;
let (i, _magic) = le_u64(i)?;
let (i, version) = le_u32(i)?;
let (i, policy_name) = take(POLICY_NAME_SIZE)(i)?;
let (i, policy_hint_size) = le_u32(i)?;
let (i, metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?;
let (i, mapping_root) = le_u64(i)?;
let (i, hint_root) = le_u64(i)?;
let (i, discard_root) = le_u64(i)?;
let (i, discard_block_size) = le_u64(i)?;
let (i, discard_nr_blocks) = le_u64(i)?;
let (i, data_block_size) = le_u32(i)?;
let (i, metadata_block_size) = le_u32(i)?;
let (i, cache_blocks) = le_u32(i)?;
let (i, compat_flags) = le_u32(i)?;
let (i, compat_ro_flags) = le_u32(i)?;
let (i, incompat_flags) = le_u32(i)?;
let (i, read_hits) = le_u32(i)?;
let (i, read_misses) = le_u32(i)?;
let (i, write_hits) = le_u32(i)?;
let (i, write_misses) = le_u32(i)?;
let (i, vsn_major) = le_u32(i)?;
let (i, vsn_minor) = le_u32(i)?;
let (i, vsn_patch) = le_u32(i)?;
let mut i = i;
let mut dirty_root = None;
if version >= 2 {
let (m, root) = le_u64(i)?;
dirty_root = Some(root);
i = &m;
}
Ok((
i,
Superblock {
flags: SuperblockFlags {
needs_check: (flags | 0x1) != 0,
},
block,
version,
policy_name: policy_name.to_vec(),
policy_version: vec![vsn_major, vsn_minor, vsn_patch],
policy_hint_size,
metadata_sm_root: metadata_sm_root.to_vec(),
mapping_root,
dirty_root,
hint_root,
discard_root,
discard_block_size,
discard_nr_blocks,
data_block_size,
metadata_block_size,
cache_blocks,
compat_flags,
compat_ro_flags,
incompat_flags,
read_hits,
read_misses,
write_hits,
write_misses,
},
))
}
pub fn read_superblock(engine: &dyn IoEngine, loc: u64) -> Result<Superblock> {
let b = engine.read(loc)?;
if let Ok((_, sb)) = unpack(&b.get_data()) {
Ok(sb)
} else {
Err(anyhow!("couldn't unpack superblock"))
}
}
//------------------------------------------

11
src/io_engine.rs
View File

@ -1,5 +1,6 @@
use io_uring::opcode::{self, types};
use io_uring::IoUring;
use safemem::write_bytes;
use std::alloc::{alloc, dealloc, Layout};
use std::fs::File;
use std::fs::OpenOptions;
@ -32,6 +33,12 @@ impl Block {
Block { loc, data: ptr }
}
pub fn zeroed(loc: u64) -> Block {
let r = Self::new(loc);
write_bytes(r.get_data(), 0);
r
}
pub fn get_data<'a>(&self) -> &'a mut [u8] {
unsafe { std::slice::from_raw_parts_mut::<'a>(self.data, BLOCK_SIZE) }
}
@ -269,10 +276,8 @@ impl AsyncIoEngine {
cqes.sort_by(|a, b| a.user_data().partial_cmp(&b.user_data()).unwrap());
let mut rs = Vec::new();
let mut i = 0;
for b in blocks {
for (i, b) in blocks.into_iter().enumerate() {
let c = &cqes[i];
i += 1;
let r = c.result();
if r < 0 {

8
src/pack/vm.rs
View File

@ -173,7 +173,7 @@ pub fn pack_literal<W: Write>(w: &mut W, bs: &[u8]) -> io::Result<()> {
use Tag::LitW;
let len = bs.len() as u64;
if len < 16 as u64 {
if len < 16 {
pack_tag(w, Tag::Lit, len as u8)?;
} else if len <= u8::MAX as u64 {
pack_tag(w, LitW, 1)?;
@ -370,6 +370,12 @@ impl VM {
}
}
impl Default for VM {
fn default() -> Self {
Self::new()
}
}
pub fn unpack<R: Read>(r: &mut R, count: usize) -> io::Result<Vec<u8>> {
let mut w = Vec::with_capacity(4096);
let mut cursor = Cursor::new(&mut w);

39
src/pdata/array.rs Normal file
View File

@ -0,0 +1,39 @@
use nom::{number::complete::*, IResult};
use std::fmt;
use crate::pdata::unpack::*;
//------------------------------------------
// TODO: build a data structure representating an array?
// FIXME: rename this struct
pub struct ArrayBlockEntry {
pub block: u64,
}
impl Unpack for ArrayBlockEntry {
fn disk_size() -> u32 {
8
}
fn unpack(i: &[u8]) -> IResult<&[u8], ArrayBlockEntry> {
let (i, n) = le_u64(i)?;
let block = n;
Ok((
i,
ArrayBlockEntry {
block,
}
))
}
}
impl fmt::Display for ArrayBlockEntry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.block)
}
}
//------------------------------------------

70
src/pdata/array_block.rs Normal file
View File

@ -0,0 +1,70 @@
use anyhow::anyhow;
use anyhow::Result;
use nom::{multi::count, number::complete::*, IResult};
use crate::pdata::unpack::Unpack;
//------------------------------------------
pub struct ArrayBlockHeader {
pub csum: u32,
pub max_entries: u32,
pub nr_entries: u32,
pub value_size: u32,
pub blocknr: u64,
}
impl Unpack for ArrayBlockHeader {
fn disk_size() -> u32 {
24
}
fn unpack(data: &[u8]) -> IResult<&[u8], ArrayBlockHeader> {
let (i, csum) = le_u32(data)?;
let (i, max_entries) = le_u32(i)?;
let (i, nr_entries) = le_u32(i)?;
let (i, value_size) = le_u32(i)?;
let (i, blocknr) = le_u64(i)?;
Ok((
i,
ArrayBlockHeader {
csum,
max_entries,
nr_entries,
value_size,
blocknr
},
))
}
}
pub struct ArrayBlock<V: Unpack> {
pub header: ArrayBlockHeader,
pub values: Vec<V>,
}
impl<V: Unpack> ArrayBlock<V> {
pub fn get_header(&self) -> &ArrayBlockHeader {
&self.header
}
}
fn convert_result<'a, V>(r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> {
r.map_err(|_| anyhow!("parse error"))
}
pub fn unpack_array_block<V: Unpack>(
data: &[u8],
) -> Result<ArrayBlock<V>> {
// TODO: collect errors
let (i, header) = ArrayBlockHeader::unpack(data).map_err(|_e| anyhow!("Couldn't parse header"))?;
let (_i, values) = convert_result(count(V::unpack, header.nr_entries as usize)(i))?;
Ok(ArrayBlock {
header,
values,
})
}
//------------------------------------------

108
src/pdata/array_walker.rs Normal file
View File

@ -0,0 +1,108 @@
use std::sync::Arc;
use crate::io_engine::*;
use crate::pdata::array::*;
use crate::pdata::array_block::*;
use crate::pdata::btree::*;
use crate::pdata::btree_walker::*;
use crate::pdata::unpack::*;
//------------------------------------------
pub struct ArrayWalker {
engine: Arc<dyn IoEngine + Send + Sync>,
ignore_non_fatal: bool
}
// FIXME: define another Result type for array visiting?
pub trait ArrayBlockVisitor<V: Unpack> {
fn visit(
&self,
index: u64,
v: V,
) -> anyhow::Result<()>;
}
struct BlockValueVisitor<V> {
engine: Arc<dyn IoEngine + Send + Sync>,
array_block_visitor: Box<dyn ArrayBlockVisitor<V>>,
}
impl<V: Unpack + Copy> BlockValueVisitor<V> {
pub fn new<Visitor: 'static + ArrayBlockVisitor<V>>(
e: Arc<dyn IoEngine + Send + Sync>,
v: Box<Visitor>,
) -> BlockValueVisitor<V> {
BlockValueVisitor {
engine: e,
array_block_visitor: v,
}
}
pub fn visit_array_block(
&self,
index: u64,
array_block: ArrayBlock<V>,
) {
let begin = index * u64::from(array_block.header.nr_entries);
for i in 0..array_block.header.nr_entries {
self.array_block_visitor.visit(begin + u64::from(i), array_block.values[i as usize]).unwrap();
}
}
}
impl<V: Unpack + Copy> NodeVisitor<ArrayBlockEntry> for BlockValueVisitor<V> {
// FIXME: return errors
fn visit(
&self,
path: &[u64],
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
values: &[ArrayBlockEntry],
) -> Result<()> {
for n in 0..keys.len() {
let index = keys[n];
let b = self.engine.read(values[n].block).map_err(|_| io_err(path))?;
let array_block = unpack_array_block::<V>(b.get_data()).map_err(|_| io_err(path))?;
self.visit_array_block(index, array_block);
}
Ok(())
}
// FIXME: stub
fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> {
Ok(())
}
// FIXME: stub
fn end_walk(&self) -> Result<()> {
Ok(())
}
}
impl ArrayWalker {
pub fn new(engine: Arc<dyn IoEngine + Send + Sync>, ignore_non_fatal: bool) -> ArrayWalker {
let r: ArrayWalker = ArrayWalker {
engine,
ignore_non_fatal,
};
r
}
// FIXME: pass the visitor by reference?
// FIXME: redefine the Result type for array visiting?
pub fn walk<BV, V: Copy>(&self, visitor: Box<BV>, root: u64) -> Result<()>
where
BV: 'static + ArrayBlockVisitor<V>,
V: Unpack,
{
let w = BTreeWalker::new(self.engine.clone(), self.ignore_non_fatal);
let mut path = Vec::new(); // FIXME: eliminate this line?
path.push(0);
let v = BlockValueVisitor::<V>::new(self.engine.clone(), visitor);
w.walk(&mut path, &v, root)
}
}
//------------------------------------------

82
src/pdata/btree.rs
View File

@ -26,6 +26,12 @@ impl KeyRange {
}
}
impl Default for KeyRange {
fn default() -> Self {
Self::new()
}
}
impl fmt::Display for KeyRange {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match (self.start, self.end) {
@ -182,22 +188,20 @@ fn test_split_range() {
}
}
fn split_one(path: &Vec<u64>, kr: &KeyRange, k: u64) -> Result<(KeyRange, KeyRange)> {
fn split_one(path: &[u64], kr: &KeyRange, k: u64) -> Result<(KeyRange, KeyRange)> {
match kr.split(k) {
None => {
return Err(node_err(
path,
&format!("couldn't split key range {} at {}", kr, k),
));
}
None => Err(node_err(
path,
&format!("couldn't split key range {} at {}", kr, k),
)),
Some(pair) => Ok(pair),
}
}
pub fn split_key_ranges(path: &Vec<u64>, kr: &KeyRange, keys: &[u64]) -> Result<Vec<KeyRange>> {
pub fn split_key_ranges(path: &[u64], kr: &KeyRange, keys: &[u64]) -> Result<Vec<KeyRange>> {
let mut krs = Vec::with_capacity(keys.len());
if keys.len() == 0 {
if keys.is_empty() {
return Err(node_err(path, "split_key_ranges: no keys present"));
}
@ -207,8 +211,8 @@ pub fn split_key_ranges(path: &Vec<u64>, kr: &KeyRange, keys: &[u64]) -> Result<
end: kr.end,
};
for i in 1..keys.len() {
let (first, rest) = split_one(path, &kr, keys[i])?;
for k in keys.iter().skip(1) {
let (first, rest) = split_one(path, &kr, *k)?;
krs.push(first);
kr = rest;
}
@ -229,7 +233,7 @@ pub fn encode_node_path(path: &[u64]) -> String {
// The first entry is normally the superblock (0), so we
// special case this.
if path.len() > 0 && path[0] == 0 {
if !path.is_empty() && path[0] == 0 {
let count = ((path.len() as u8) - 1) << 1;
cursor.write_u8(count as u8).unwrap();
vm::pack_u64s(&mut cursor, &path[1..]).unwrap();
@ -349,19 +353,19 @@ impl fmt::Display for BTreeError {
}
}
}
pub fn node_err(path: &Vec<u64>, msg: &str) -> BTreeError {
pub fn node_err(path: &[u64], msg: &str) -> BTreeError {
BTreeError::Path(
path.clone(),
path.to_vec(),
Box::new(BTreeError::NodeError(msg.to_string())),
)
}
pub fn node_err_s(path: &Vec<u64>, msg: String) -> BTreeError {
BTreeError::Path(path.clone(), Box::new(BTreeError::NodeError(msg)))
pub fn node_err_s(path: &[u64], msg: String) -> BTreeError {
BTreeError::Path(path.to_vec(), Box::new(BTreeError::NodeError(msg)))
}
pub fn io_err(path: &Vec<u64>) -> BTreeError {
BTreeError::Path(path.clone(), Box::new(BTreeError::IoError))
pub fn io_err(path: &[u64]) -> BTreeError {
BTreeError::Path(path.to_vec(), Box::new(BTreeError::IoError))
}
pub fn value_err(msg: String) -> BTreeError {
@ -424,22 +428,22 @@ impl Unpack for NodeHeader {
impl Pack for NodeHeader {
fn pack<W: WriteBytesExt>(&self, w: &mut W) -> anyhow::Result<()> {
// csum needs to be calculated right for the whole metadata block.
w.write_u32::<LittleEndian>(0)?;
// csum needs to be calculated right for the whole metadata block.
w.write_u32::<LittleEndian>(0)?;
let flags;
if self.is_leaf {
flags = LEAF_NODE;
} else {
flags = INTERNAL_NODE;
}
w.write_u32::<LittleEndian>(flags)?;
w.write_u64::<LittleEndian>(self.block)?;
w.write_u32::<LittleEndian>(self.nr_entries)?;
w.write_u32::<LittleEndian>(self.max_entries)?;
w.write_u32::<LittleEndian>(self.value_size)?;
w.write_u32::<LittleEndian>(0)?;
Ok(())
let flags;
if self.is_leaf {
flags = LEAF_NODE;
} else {
flags = INTERNAL_NODE;
}
w.write_u32::<LittleEndian>(flags)?;
w.write_u64::<LittleEndian>(self.block)?;
w.write_u32::<LittleEndian>(self.nr_entries)?;
w.write_u32::<LittleEndian>(self.max_entries)?;
w.write_u32::<LittleEndian>(self.value_size)?;
w.write_u32::<LittleEndian>(0)?;
Ok(())
}
}
@ -473,7 +477,7 @@ impl<V: Unpack> Node<V> {
Leaf { header, .. } => header,
}
}
pub fn get_keys(&self) -> &[u64] {
use Node::*;
match self {
@ -487,16 +491,16 @@ impl<V: Unpack> Node<V> {
}
}
pub fn convert_result<'a, V>(path: &Vec<u64>, r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> {
pub fn convert_result<'a, V>(path: &[u64], r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> {
r.map_err(|_e| node_err(path, "parse error"))
}
pub fn convert_io_err<V>(path: &Vec<u64>, r: std::io::Result<V>) -> Result<V> {
pub fn convert_io_err<V>(path: &[u64], r: std::io::Result<V>) -> Result<V> {
r.map_err(|_| io_err(path))
}
pub fn unpack_node<V: Unpack>(
path: &Vec<u64>,
path: &[u64],
data: &[u8],
ignore_non_fatal: bool,
is_root: bool,
@ -554,7 +558,7 @@ pub fn unpack_node<V: Unpack>(
for k in &keys {
if let Some(l) = last {
if k <= l {
return Err(node_err(path, "keys out of order"));
return Err(node_err(&path, "keys out of order"));
}
}
@ -573,7 +577,7 @@ pub fn unpack_node<V: Unpack>(
values,
})
} else {
let (_i, values) = convert_result(path, count(le_u64, header.nr_entries as usize)(i))?;
let (_i, values) = convert_result(&path, count(le_u64, header.nr_entries as usize)(i))?;
Ok(Node::Internal {
header,
keys,

34
src/pdata/btree_builder.rs
View File

@ -63,7 +63,7 @@ impl RefCounter<u64> for SMRefCounter {
//------------------------------------------
/// Pack the given node ready to write to disk.
fn pack_node<W: WriteBytesExt, V: Pack + Unpack>(node: &Node<V>, w: &mut W) -> Result<()> {
pub fn pack_node<W: WriteBytesExt, V: Pack + Unpack>(node: &Node<V>, w: &mut W) -> Result<()> {
match node {
Node::Internal {
header,
@ -108,6 +108,14 @@ fn pack_node<W: WriteBytesExt, V: Pack + Unpack>(node: &Node<V>, w: &mut W) -> R
Ok(())
}
//------------------------------------------
pub fn calc_max_entries<V: Unpack>() -> usize {
let elt_size = 8 + V::disk_size() as usize;
let total = ((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize;
total / 3 * 3
}
pub struct WriteResult {
first_key: u64,
loc: u64,
@ -217,15 +225,6 @@ impl NodeIO<u64> for InternalIO {
//------------------------------------------
/// What is the maximum number of entries of a given size we can fit in
/// a btree node?
fn calc_max_entries<V: Unpack>() -> usize {
let elt_size = 8 + V::disk_size() as usize;
((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize
}
//------------------------------------------
/// This takes a sequence of values or nodes, and builds a vector of leaf nodes.
/// Care is taken to make sure that all nodes are at least half full unless there's
/// only a single node.
@ -269,7 +268,6 @@ impl<V: Pack + Unpack + Clone> NodeBuilder<V> {
nodes: Vec::new(),
}
}
/// Push a single value. This may emit a new node, hence the Result
/// return type. The value's ref count will be incremented.
pub fn push_value(&mut self, key: u64, val: V) -> Result<()> {
@ -503,3 +501,17 @@ impl<V: Unpack + Pack + Clone> Builder<V> {
}
//------------------------------------------
/*
=======
fn write_node_<V: Unpack + Pack>(w: &mut WriteBatcher, mut node: Node<V>) -> Result<(u64, u64)> {
let keys = node.get_keys();
let first_key = *keys.first().unwrap_or(&0u64);
>>>>>>> main
=======
fn write_node_<V: Unpack + Pack>(w: &mut WriteBatcher, mut node: Node<V>) -> Result<(u64, u64)> {
let keys = node.get_keys();
let first_key = *keys.first().unwrap_or(&0u64);
>>>>>>> main
*/

15
src/pdata/btree_leaf_walker.rs
View File

@ -102,8 +102,7 @@ impl<'a> LeafWalker<'a> {
.read_many(&blocks[0..])
.map_err(|_e| io_err(path))?;
let mut i = 0;
for rb in rblocks {
for (i, rb) in rblocks.into_iter().enumerate() {
match rb {
Err(_) => {
return Err(io_err(path).keys_context(&filtered_krs[i]));
@ -112,8 +111,6 @@ impl<'a> LeafWalker<'a> {
self.walk_node(depth - 1, path, visitor, &filtered_krs[i], &b, false)?;
}
}
i += 1;
}
Ok(())
@ -225,14 +222,12 @@ impl<'a> LeafWalker<'a> {
self.leaves.insert(root as usize);
visitor.visit(&kr, root)?;
Ok(())
} else if self.sm_inc(root) > 0 {
visitor.visit_again(root)
} else {
if self.sm_inc(root) > 0 {
visitor.visit_again(root)
} else {
let root = self.engine.read(root).map_err(|_| io_err(path))?;
let root = self.engine.read(root).map_err(|_| io_err(path))?;
self.walk_node(depth - 1, path, visitor, &kr, &root, true)
}
self.walk_node(depth - 1, path, visitor, &kr, &root, true)
}
}

8
src/pdata/btree_merge.rs
View File

@ -49,21 +49,21 @@ impl LeafVisitor {
impl<V: Unpack> NodeVisitor<V> for LeafVisitor {
fn visit(
&self,
path: &Vec<u64>,
path: &[u64],
_kr: &KeyRange,
_header: &NodeHeader,
keys: &[u64],
_values: &[V],
) -> btree::Result<()> {
// ignore empty nodes
if keys.len() == 0 {
if keys.is_empty() {
return Ok(());
}
let mut inner = self.inner.lock().unwrap();
// Check keys are ordered.
if inner.leaves.len() > 0 {
if !inner.leaves.is_empty() {
let last_key = inner.leaves.last().unwrap().key_high;
if keys[0] <= last_key {
return Err(BTreeError::NodeError(
@ -83,7 +83,7 @@ impl<V: Unpack> NodeVisitor<V> for LeafVisitor {
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> btree::Result<()> {
fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> {
Ok(())
}

32
src/pdata/btree_walker.rs
View File

@ -14,7 +14,7 @@ pub trait NodeVisitor<V: Unpack> {
// &self is deliberately non mut to allow the walker to use multiple threads.
fn visit(
&self,
path: &Vec<u64>,
path: &[u64],
kr: &KeyRange,
header: &NodeHeader,
keys: &[u64],
@ -24,7 +24,7 @@ pub trait NodeVisitor<V: Unpack> {
// Nodes may be shared and thus visited multiple times. The walker avoids
// doing repeated IO, but it does call this method to keep the visitor up to
// date.
fn visit_again(&self, path: &Vec<u64>, b: u64) -> Result<()>;
fn visit_again(&self, path: &[u64], b: u64) -> Result<()>;
fn end_walk(&self) -> Result<()>;
}
@ -154,8 +154,7 @@ impl BTreeWalker {
}
}
Ok(rblocks) => {
let mut i = 0;
for rb in rblocks {
for (i, rb) in rblocks.into_iter().enumerate() {
match rb {
Err(_) => {
let e = io_err(path).keys_context(&filtered_krs[i]);
@ -169,8 +168,6 @@ impl BTreeWalker {
Ok(()) => {}
},
}
i += 1;
}
}
}
@ -215,7 +212,7 @@ impl BTreeWalker {
values,
} => {
if let Err(e) = visitor.visit(path, &kr, &header, &keys, &values) {
let e = BTreeError::Path(path.clone(), Box::new(e.clone()));
let e = BTreeError::Path(path.clone(), Box::new(e));
self.set_fail(b.loc, e.clone());
return Err(e);
}
@ -251,7 +248,7 @@ impl BTreeWalker {
{
if self.sm_inc(root) > 0 {
if let Some(e) = self.failed(root) {
Err(e.clone())
Err(e)
} else {
visitor.visit_again(path, root)
}
@ -382,10 +379,9 @@ where
}
}
Ok(rblocks) => {
let mut i = 0;
let errs = Arc::new(Mutex::new(Vec::new()));
for rb in rblocks {
for (i, rb) in rblocks.into_iter().enumerate() {
match rb {
Err(_) => {
let e = io_err(path).keys_context(&filtered_krs[i]);
@ -411,8 +407,6 @@ where
});
}
}
i += 1;
}
pool.join();
@ -435,7 +429,7 @@ where
{
if w.sm_inc(root) > 0 {
if let Some(e) = w.failed(root) {
Err(e.clone())
Err(e)
} else {
visitor.visit_again(path, root)
}
@ -467,7 +461,7 @@ impl<V> ValueCollector<V> {
impl<V: Unpack + Copy> NodeVisitor<V> for ValueCollector<V> {
fn visit(
&self,
_path: &Vec<u64>,
_path: &[u64],
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
@ -475,13 +469,13 @@ impl<V: Unpack + Copy> NodeVisitor<V> for ValueCollector<V> {
) -> Result<()> {
let mut vals = self.values.lock().unwrap();
for n in 0..keys.len() {
vals.insert(keys[n], values[n].clone());
vals.insert(keys[n], values[n]);
}
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> Result<()> {
fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> {
Ok(())
}
@ -533,7 +527,7 @@ impl<V> ValuePathCollector<V> {
impl<V: Unpack + Clone> NodeVisitor<V> for ValuePathCollector<V> {
fn visit(
&self,
path: &Vec<u64>,
path: &[u64],
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
@ -541,13 +535,13 @@ impl<V: Unpack + Clone> NodeVisitor<V> for ValuePathCollector<V> {
) -> Result<()> {
let mut vals = self.values.lock().unwrap();
for n in 0..keys.len() {
vals.insert(keys[n], (path.clone(), values[n].clone()));
vals.insert(keys[n], (path.to_vec(), values[n].clone()));
}
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> Result<()> {
fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> {
Ok(())
}

3
src/pdata/mod.rs
View File

@ -1,3 +1,6 @@
pub mod array;
pub mod array_block;
pub mod array_walker;
pub mod btree;
pub mod btree_builder;
pub mod btree_merge;

1
src/report.rs
View File

@ -95,6 +95,7 @@ impl Report {
//------------------------------------------
#[allow(dead_code)]
struct PBInner {
title: String,
bar: ProgressBar,

65
src/thin/check.rs
View File

@ -2,6 +2,7 @@ use anyhow::{anyhow, Result};
use std::collections::BTreeMap;
use std::io::Cursor;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::{self, JoinHandle};
use threadpool::ThreadPool;
@ -28,7 +29,7 @@ struct BottomLevelVisitor {
impl NodeVisitor<BlockTime> for BottomLevelVisitor {
fn visit(
&self,
_path: &Vec<u64>,
_path: &[u64],
_kr: &KeyRange,
_h: &NodeHeader,
_k: &[u64],
@ -36,7 +37,7 @@ impl NodeVisitor<BlockTime> for BottomLevelVisitor {
) -> btree::Result<()> {
// FIXME: do other checks
if values.len() == 0 {
if values.is_empty() {
return Ok(());
}
@ -45,8 +46,8 @@ impl NodeVisitor<BlockTime> for BottomLevelVisitor {
let mut start = values[0].block;
let mut len = 1;
for n in 1..values.len() {
let block = values[n].block;
for b in values.iter().skip(1) {
let block = b.block;
if block == start + len {
len += 1;
} else {
@ -60,7 +61,7 @@ impl NodeVisitor<BlockTime> for BottomLevelVisitor {
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> btree::Result<()> {
fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> {
Ok(())
}
@ -84,7 +85,7 @@ impl<'a> OverflowChecker<'a> {
impl<'a> NodeVisitor<u32> for OverflowChecker<'a> {
fn visit(
&self,
_path: &Vec<u64>,
_path: &[u64],
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
@ -103,7 +104,7 @@ impl<'a> NodeVisitor<u32> for OverflowChecker<'a> {
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, _b: u64) -> btree::Result<()> {
fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> {
Ok(())
}
@ -152,13 +153,12 @@ fn check_space_map(
}
// FIXME: we should do this in batches
let blocks = engine.read_many(&mut blocks)?;
let blocks = engine.read_many(&blocks)?;
let mut leaks = 0;
let mut blocknr = 0;
let mut bitmap_leaks = Vec::new();
for n in 0..entries.len() {
let b = &blocks[n];
for b in blocks.iter().take(entries.len()) {
match b {
Err(_e) => {
return Err(anyhow!("Unable to read bitmap block"));
@ -233,12 +233,8 @@ fn repair_space_map(ctx: &Context, entries: Vec<BitmapLeak>, sm: ASpaceMap) -> R
let rblocks = engine.read_many(&blocks[0..])?;
let mut write_blocks = Vec::new();
let mut i = 0;
for rb in rblocks {
if rb.is_err() {
return Err(anyhow!("Unable to reread bitmap blocks for repair"));
} else {
let b = rb.unwrap();
for (i, rb) in rblocks.into_iter().enumerate() {
if let Ok(b) = rb {
let be = &entries[i];
let mut blocknr = be.blocknr;
let mut bitmap = unpack::<Bitmap>(b.get_data())?;
@ -262,9 +258,9 @@ fn repair_space_map(ctx: &Context, entries: Vec<BitmapLeak>, sm: ASpaceMap) -> R
checksum::write_checksum(b.get_data(), checksum::BT::BITMAP)?;
write_blocks.push(b);
} else {
return Err(anyhow!("Unable to reread bitmap blocks for repair"));
}
i += 1;
}
engine.write_many(&write_blocks[0..])?;
@ -305,20 +301,17 @@ fn spawn_progress_thread(
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
nr_allocated_metadata: u64,
report: Arc<Report>,
) -> Result<(JoinHandle<()>, Arc<Mutex<bool>>)> {
) -> Result<(JoinHandle<()>, Arc<AtomicBool>)> {
let tid;
let stop_progress = Arc::new(Mutex::new(false));
let stop_progress = Arc::new(AtomicBool::new(false));
{
let stop_progress = stop_progress.clone();
tid = thread::spawn(move || {
let interval = std::time::Duration::from_millis(250);
loop {
{
let stop_progress = stop_progress.lock().unwrap();
if *stop_progress {
break;
}
if stop_progress.load(Ordering::Relaxed) {
break;
}
let sm = sm.lock().unwrap();
@ -364,7 +357,7 @@ fn check_mapping_bottom_level(
if roots.len() > 64 {
let errs = Arc::new(Mutex::new(Vec::new()));
for (_thin_id, (path, root)) in roots {
for (path, root) in roots.values() {
let data_sm = data_sm.clone();
let root = *root;
let v = BottomLevelVisitor { data_sm };
@ -381,12 +374,12 @@ fn check_mapping_bottom_level(
}
ctx.pool.join();
let errs = Arc::try_unwrap(errs).unwrap().into_inner().unwrap();
if errs.len() > 0 {
if !errs.is_empty() {
ctx.report.fatal(&format!("{}", aggregate_error(errs)));
failed = true;
}
} else {
for (_thin_id, (path, root)) in roots {
for (path, root) in roots.values() {
let w = w.clone();
let data_sm = data_sm.clone();
let root = *root;
@ -468,7 +461,12 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
// Device details. We read this once to get the number of thin devices, and hence the
// maximum metadata ref count. Then create metadata space map, and reread to increment
// the ref counts for that metadata.
let devs = btree_to_map::<DeviceDetail>(&mut path, engine.clone(), opts.ignore_non_fatal, sb.details_root)?;
let devs = btree_to_map::<DeviceDetail>(
&mut path,
engine.clone(),
opts.ignore_non_fatal,
sb.details_root,
)?;
let nr_devs = devs.len();
let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32);
inc_superblock(&metadata_sm)?;
@ -574,21 +572,18 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
bail_out(&ctx, "metadata space map")?;
if opts.auto_repair {
if data_leaks.len() > 0 {
if !data_leaks.is_empty() {
ctx.report.info("Repairing data leaks.");
repair_space_map(&ctx, data_leaks, data_sm.clone())?;
}
if metadata_leaks.len() > 0 {
if !metadata_leaks.is_empty() {
ctx.report.info("Repairing metadata leaks.");
repair_space_map(&ctx, metadata_leaks, metadata_sm.clone())?;
}
}
{
let mut stop_progress = stop_progress.lock().unwrap();
*stop_progress = true;
}
stop_progress.store(true, Ordering::Relaxed);
tid.join().unwrap();
Ok(())

22
src/thin/dump.rs
View File

@ -38,7 +38,7 @@ impl RunBuilder {
self.run = Some(xml::Map {
thin_begin: thin_block,
data_begin: data_block,
time: time,
time,
len: 1,
});
None
@ -59,7 +59,7 @@ impl RunBuilder {
self.run.replace(Map {
thin_begin: thin_block,
data_begin: data_block,
time: time,
time,
len: 1,
})
}
@ -99,7 +99,7 @@ impl<'a> MappingVisitor<'a> {
impl<'a> NodeVisitor<BlockTime> for MappingVisitor<'a> {
fn visit(
&self,
_path: &Vec<u64>,
_path: &[u64],
_kr: &KeyRange,
_h: &NodeHeader,
keys: &[u64],
@ -118,7 +118,7 @@ impl<'a> NodeVisitor<BlockTime> for MappingVisitor<'a> {
Ok(())
}
fn visit_again(&self, _path: &Vec<u64>, b: u64) -> btree::Result<()> {
fn visit_again(&self, _path: &[u64], b: u64) -> btree::Result<()> {
let mut inner = self.inner.lock().unwrap();
inner
.md_out
@ -337,7 +337,7 @@ fn build_metadata(ctx: &Context, sb: &Superblock) -> Result<Metadata> {
let (mut shared, sm) = find_shared_nodes(ctx, &roots)?;
// Add in the roots, because they may not be shared.
for (_thin_id, (_path, root)) in &roots {
for (_path, root) in roots.values() {
shared.insert(*root);
}
@ -354,7 +354,7 @@ fn build_metadata(ctx: &Context, sb: &Superblock) -> Result<Metadata> {
let kr = KeyRange::new(); // FIXME: finish
devs.push(Device {
thin_id: thin_id as u32,
detail: detail.clone(),
detail: *detail,
map: Mapping {
kr,
entries: es.to_vec(),
@ -381,7 +381,7 @@ fn build_metadata(ctx: &Context, sb: &Superblock) -> Result<Metadata> {
//------------------------------------------
fn gather_entries(g: &mut Gatherer, es: &Vec<Entry>) {
fn gather_entries(g: &mut Gatherer, es: &[Entry]) {
g.new_seq();
for e in es {
match e {
@ -395,7 +395,7 @@ fn gather_entries(g: &mut Gatherer, es: &Vec<Entry>) {
}
}
fn entries_to_runs(runs: &BTreeMap<u64, Vec<u64>>, es: &Vec<Entry>) -> Vec<Entry> {
fn entries_to_runs(runs: &BTreeMap<u64, Vec<u64>>, es: &[Entry]) -> Vec<Entry> {
use Entry::*;
let mut result = Vec::new();
@ -542,7 +542,7 @@ fn emit_leaves(ctx: &Context, out: &mut dyn xml::MetadataVisitor, ls: &[u64]) ->
fn emit_entries<W: Write>(
ctx: &Context,
out: &mut xml::XmlWriter<W>,
entries: &Vec<Entry>,
entries: &[Entry],
) -> Result<()> {
let mut leaves = Vec::new();
@ -552,7 +552,7 @@ fn emit_entries<W: Write>(
leaves.push(*b);
}
Entry::Ref(id) => {
if leaves.len() > 0 {
if !leaves.is_empty() {
emit_leaves(&ctx, out, &leaves[0..])?;
leaves.clear();
}
@ -562,7 +562,7 @@ fn emit_entries<W: Write>(
}
}
if leaves.len() > 0 {
if !leaves.is_empty() {
emit_leaves(&ctx, out, &leaves[0..])?;
}

6
src/thin/runs.rs
View File

@ -139,6 +139,12 @@ impl Gatherer {
}
}
impl Default for Gatherer {
fn default() -> Self {
Self::new()
}
}
//------------------------------------------
#[cfg(test)]

1
src/version.rs Normal file
View File

@ -0,0 +1 @@
pub const TOOLS_VERSION: &str = include_str!("../VERSION");

2
src/version.rs.in
View File

@ -1,2 +0,0 @@
pub const TOOLS_VERSION: &str = @THIN_PROVISIONING_TOOLS_VERSION@;

6
tests/thin_delta.rs
View File

@ -21,7 +21,7 @@ fn accepts_version() -> Result<()> {
Ok(())
}
const USAGE: &str = "Usage: thin_delta [options] <device or file>\nOptions:\n {--thin1, --snap1}\n {--thin2, --snap2}\n {-m, --metadata-snap} [block#]\n {--verbose}\n {-h|--help}\n {-V|--version}";
const USAGE: &str = "Usage: thin_delta [options] <device or file>\nOptions:\n {--thin1, --snap1, --root1}\n {--thin2, --snap2, --root2}\n {-m, --metadata-snap} [block#]\n {--verbose}\n {-h|--help}\n {-V|--version}";
#[test]
fn accepts_h() -> Result<()> {
@ -49,7 +49,7 @@ fn snap1_unspecified() -> Result<()> {
let mut td = TestDir::new()?;
let md = mk_valid_md(&mut td)?;
let stderr = run_fail(thin_delta!("--snap2", "45", &md))?;
assert!(stderr.contains("--snap1 not specified"));
assert!(stderr.contains("--snap1 or --root1 not specified"));
Ok(())
}
@ -58,7 +58,7 @@ fn snap2_unspecified() -> Result<()> {
let mut td = TestDir::new()?;
let md = mk_valid_md(&mut td)?;
let stderr = run_fail(thin_delta!("--snap1", "45", &md))?;
assert!(stderr.contains("--snap2 not specified"));
assert!(stderr.contains("--snap2 or --root2 not specified"));
Ok(())
}

13
thin-provisioning/commands.cc
View File

@ -17,19 +17,6 @@ thin_provisioning::register_thin_commands(base::application &app)
app.add_cmd(command::ptr(new thin_repair_cmd()));
app.add_cmd(command::ptr(new thin_rmap_cmd()));
app.add_cmd(command::ptr(new thin_trim_cmd()));
#ifdef DEV_TOOLS
app.add_cmd(command::ptr(new thin_ll_dump_cmd()));
app.add_cmd(command::ptr(new thin_ll_restore_cmd()));
app.add_cmd(command::ptr(new thin_scan_cmd()));
app.add_cmd(command::ptr(new thin_generate_damage_cmd()));
app.add_cmd(command::ptr(new thin_generate_metadata_cmd()));
app.add_cmd(command::ptr(new thin_generate_mappings_cmd()));
app.add_cmd(command::ptr(new thin_patch_superblock_cmd()));
app.add_cmd(command::ptr(new thin_show_duplicates_cmd()));
app.add_cmd(command::ptr(new thin_show_metadata_cmd()));
app.add_cmd(command::ptr(new thin_journal_cmd()));
#endif
}
//----------------------------------------------------------------

78
thin-provisioning/commands.h
View File

@ -71,34 +71,11 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv);
};
#ifdef DEV_TOOLS
class thin_ll_dump_cmd : public base::command {
//------------------------------------------------------
class thin_debug_cmd : public base::command {
public:
thin_ll_dump_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_ll_restore_cmd : public base::command {
public:
thin_ll_restore_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_scan_cmd : public base::command {
public:
thin_scan_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_show_duplicates_cmd : public base::command {
public:
thin_show_duplicates_cmd();
thin_debug_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
@ -124,6 +101,29 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv);
};
class thin_journal_cmd : public base::command {
public:
thin_journal_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_ll_dump_cmd : public base::command {
public:
thin_ll_dump_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_ll_restore_cmd : public base::command {
public:
thin_ll_restore_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_patch_superblock_cmd : public base::command {
public:
thin_patch_superblock_cmd();
@ -131,6 +131,22 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv);
};
class thin_scan_cmd : public base::command {
public:
thin_scan_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_show_duplicates_cmd : public base::command {
public:
thin_show_duplicates_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
class thin_show_metadata_cmd : public base::command {
public:
thin_show_metadata_cmd();
@ -138,14 +154,6 @@ namespace thin_provisioning {
virtual int run(int argc, char **argv);
};
class thin_journal_cmd : public base::command {
public:
thin_journal_cmd();
virtual void usage(std::ostream &out) const;
virtual int run(int argc, char **argv);
};
#endif
void register_thin_commands(base::application &app);
}

24
thin-provisioning/devel_commands.cc Normal file
View File

@ -0,0 +1,24 @@
#include "thin-provisioning/commands.h"
using namespace base;
using namespace thin_provisioning;
//----------------------------------------------------------------
void
thin_provisioning::register_thin_commands(base::application &app)
{
app.add_cmd(command::ptr(new thin_debug_cmd()));
app.add_cmd(command::ptr(new thin_generate_damage_cmd()));
app.add_cmd(command::ptr(new thin_generate_mappings_cmd()));
app.add_cmd(command::ptr(new thin_generate_metadata_cmd()));
app.add_cmd(command::ptr(new thin_journal_cmd()));
app.add_cmd(command::ptr(new thin_ll_dump_cmd()));
app.add_cmd(command::ptr(new thin_ll_restore_cmd()));
app.add_cmd(command::ptr(new thin_patch_superblock_cmd()));
app.add_cmd(command::ptr(new thin_scan_cmd()));
app.add_cmd(command::ptr(new thin_show_duplicates_cmd()));
app.add_cmd(command::ptr(new thin_show_metadata_cmd()));
}
//----------------------------------------------------------------

390
thin-provisioning/thin_debug.cc
View File

@ -16,305 +16,183 @@
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/tuple/tuple.hpp>
#include <boost/variant.hpp>
#include <getopt.h>
#include <iostream>
#include <libgen.h>
#include <map>
#include <string>
#include <vector>
#include "persistent-data/data-structures/btree.h"
#include "base/math_utils.h"
#include "dbg-lib/array_block_dumper.h"
#include "dbg-lib/btree_node_dumper.h"
#include "dbg-lib/index_block_dumper.h"
#include "dbg-lib/command_interpreter.h"
#include "dbg-lib/commands.h"
#include "dbg-lib/output_formatter.h"
#include "dbg-lib/sm_show_traits.h"
#include "persistent-data/file_utils.h"
#include "persistent-data/space-maps/disk_structures.h"
#include "thin-provisioning/commands.h"
#include "thin-provisioning/metadata.h"
#include "thin-provisioning/metadata_checker.h"
#include "thin-provisioning/superblock.h"
#include "version.h"
using namespace boost;
using namespace dbg;
using namespace persistent_data;
using namespace std;
using namespace thin_provisioning;
namespace {
typedef vector<string> strings;
class formatter {
public:
typedef shared_ptr<formatter> ptr;
virtual ~formatter() {}
typedef optional<string> maybe_string;
void field(string const &name, string const &value) {
fields_.push_back(field_type(name, value));
}
void child(string const &name, formatter::ptr t) {
fields_.push_back(field_type(name, t));
}
virtual void output(ostream &out, int depth = 0) = 0;
protected:
typedef variant<string, ptr> value;
typedef tuple<string, value> field_type;
vector<field_type> fields_;
};
template <typename T>
void
field(formatter &t, string const &name, T const &value) {
t.field(name, lexical_cast<string>(value));
}
//--------------------------------
class xml_formatter : public formatter {
public:
virtual void output(ostream &out, int depth) {
indent(depth, out);
out << "<fields>" << endl;
vector<field_type>::const_iterator it;
for (it = fields_.begin(); it != fields_.end(); ++it) {
if (string const *s = get<string>(&it->get<1>())) {
indent(depth + 1, out);
out << "<field key=\""
<< it->get<0>()
<< "\" value=\""
<< *s
<< "\"/>"
<< endl;
} else {
formatter::ptr f = get<formatter::ptr>(it->get<1>());
f->output(out, depth + 1);
}
}
indent(depth, out);
out << "</fields>" << endl;
}
private:
void indent(int depth, ostream &out) const {
for (int i = 0; i < depth * 2; i++)
out << ' ';
}
};
//--------------------------------
class command {
public:
typedef std::shared_ptr<command> ptr;
virtual ~command() {}
virtual void exec(strings const &args, ostream &out) = 0;
};
class command_interpreter {
public:
command_interpreter(istream &in, ostream &out)
: in_(in),
out_(out) {
}
void register_command(string const &str, command::ptr cmd) {
commands_.insert(make_pair(str, cmd));
}
void enter_main_loop() {
while (true)
do_once();
}
private:
strings read_input() {
using namespace boost::algorithm;
string input;
getline(in_, input);
strings toks;
split(toks, input, is_any_of(" \t"), token_compress_on);
return toks;
}
void do_once() {
if (in_.eof())
throw runtime_error("input closed");
out_ << "> ";
strings args = read_input();
map<string, command::ptr>::iterator it;
it = commands_.find(args[0]);
if (it == commands_.end())
out_ << "Unrecognised command" << endl;
else
it->second->exec(args, out_);
}
istream &in_;
ostream &out_;
map <string, command::ptr> commands_;
};
//--------------------------------
class hello : public command {
class help : public dbg::command {
virtual void exec(strings const &args, ostream &out) {
out << "Hello, world!" << endl;
out << "Commands:" << endl
<< " superblock [block#]" << endl
<< " m1_node <block# of top-level mapping tree node>" << endl
<< " m2_node <block# of bottom-level mapping tree node>" << endl
<< " detail_node <block# of device details tree node>" << endl
<< " index_block <block# of metadata space map root>" << endl
<< " index_node <block# of data space map node>" << endl
<< " exit" << endl;
}
};
class show_superblock : public command {
class show_superblock : public dbg::command {
public:
explicit show_superblock(metadata::ptr md)
: md_(md) {
explicit show_superblock(block_manager::ptr bm)
: bm_(bm) {
}
virtual void exec(strings const &args, ostream &out) {
xml_formatter f;
superblock const &sb = md_->sb_;
field(f, "csum", sb.csum_);
field(f, "flags", sb.flags_);
field(f, "blocknr", sb.blocknr_);
field(f, "uuid", sb.uuid_); // FIXME: delimit, and handle non-printable chars
field(f, "magic", sb.magic_);
field(f, "version", sb.version_);
field(f, "time", sb.time_);
field(f, "trans id", sb.trans_id_);
field(f, "metadata snap", sb.metadata_snap_);
field(f, "data mapping root", sb.data_mapping_root_);
field(f, "device details root", sb.device_details_root_);
field(f, "data block size", sb.data_block_size_);
field(f, "metadata block size", sb.metadata_block_size_);
field(f, "metadata nr blocks", sb.metadata_nr_blocks_);
field(f, "compat flags", sb.compat_flags_);
field(f, "compat ro flags", sb.compat_ro_flags_);
field(f, "incompat flags", sb.incompat_flags_);
f.output(out, 0);
}
private:
metadata::ptr md_;
};
class device_details_show_traits : public device_details_traits {
public:
static void show(formatter &f, string const &key, device_details const &value) {
field(f, "mapped blocks", value.mapped_blocks_);
field(f, "transaction id", value.transaction_id_);
field(f, "creation time", value.creation_time_);
field(f, "snap time", value.snapshotted_time_);
}
};
class uint64_show_traits : public uint64_traits {
public:
static void show(formatter &f, string const &key, uint64_t const &value) {
field(f, key, lexical_cast<string>(value));
}
};
class block_show_traits : public block_traits {
public:
static void show(formatter &f, string const &key, block_time const &value) {
field(f, "block", value.block_);
field(f, "time", value.time_);
}
};
template <typename ValueTraits>
class show_btree_node : public command {
public:
explicit show_btree_node(metadata::ptr md)
: md_(md) {
}
virtual void exec(strings const &args, ostream &out) {
using namespace persistent_data::btree_detail;
if (args.size() != 2)
if (args.size() > 2)
throw runtime_error("incorrect number of arguments");
block_address block = lexical_cast<block_address>(args[1]);
block_manager<>::read_ref rr = md_->tm_->read_lock(block);
block_address b = superblock_detail::SUPERBLOCK_LOCATION;
if (args.size() == 2)
b = boost::lexical_cast<block_address>(args[1]);
superblock_detail::superblock sb = read_superblock(bm_, b);
node_ref<uint64_show_traits> n = btree_detail::to_node<uint64_show_traits>(rr);
if (n.get_type() == INTERNAL)
show_node<uint64_show_traits>(n, out);
else {
node_ref<ValueTraits> n = btree_detail::to_node<ValueTraits>(rr);
show_node<ValueTraits>(n, out);
formatter::ptr f = create_xml_formatter();
field(*f, "csum", sb.csum_);
field(*f, "flags", sb.flags_);
field(*f, "blocknr", sb.blocknr_);
field(*f, "uuid", sb.uuid_); // FIXME: delimit, and handle non-printable chars
field(*f, "magic", sb.magic_);
field(*f, "version", sb.version_);
field(*f, "time", sb.time_);
field(*f, "trans_id", sb.trans_id_);
field(*f, "metadata_snap", sb.metadata_snap_);
sm_disk_detail::sm_root_disk const *d;
sm_disk_detail::sm_root v;
{
d = reinterpret_cast<sm_disk_detail::sm_root_disk const *>(sb.metadata_space_map_root_);
sm_disk_detail::sm_root_traits::unpack(*d, v);
formatter::ptr f2 = create_xml_formatter();
sm_root_show_traits::show(f2, "value", v);
f->child("metadata_space_map_root", f2);
}
{
d = reinterpret_cast<sm_disk_detail::sm_root_disk const *>(sb.data_space_map_root_);
sm_disk_detail::sm_root_traits::unpack(*d, v);
formatter::ptr f2 = create_xml_formatter();
sm_root_show_traits::show(f2, "value", v);
f->child("data_space_map_root", f2);
}
field(*f, "data_mapping_root", sb.data_mapping_root_);
field(*f, "device_details_root", sb.device_details_root_);
field(*f, "data_block_size", sb.data_block_size_);
field(*f, "metadata_block_size", sb.metadata_block_size_);
field(*f, "metadata_nr_blocks", sb.metadata_nr_blocks_);
field(*f, "compat_flags", sb.compat_flags_);
field(*f, "compat_ro_flags", sb.compat_ro_flags_);
field(*f, "incompat_flags", sb.incompat_flags_);
f->output(out, 0);
}
private:
template <typename VT>
void show_node(node_ref<VT> n, ostream &out) {
xml_formatter f;
block_manager::ptr bm_;
};
field(f, "csum", n.get_checksum());
field(f, "blocknr", n.get_location());
field(f, "type", n.get_type() == INTERNAL ? "internal" : "leaf");
field(f, "nr entries", n.get_nr_entries());
field(f, "max entries", n.get_max_entries());
field(f, "value size", n.get_value_size());
class device_details_show_traits {
public:
typedef thin_provisioning::device_tree_detail::device_details_traits value_trait;
for (unsigned i = 0; i < n.get_nr_entries(); i++) {
formatter::ptr f2(new xml_formatter);
field(*f2, "key", n.key_at(i));
VT::show(*f2, "value", n.value_at(i));
f.child("child", f2);
}
f.output(out, 0);
static void show(formatter::ptr f, string const &key,
thin_provisioning::device_tree_detail::device_details const &value) {
field(*f, "mapped_blocks", value.mapped_blocks_);
field(*f, "transaction_id", value.transaction_id_);
field(*f, "creation_time", value.creation_time_);
field(*f, "snap_time", value.snapshotted_time_);
}
};
metadata::ptr md_;
class block_show_traits {
public:
typedef thin_provisioning::mapping_tree_detail::block_traits value_trait;
static void show(formatter::ptr f, string const &key,
thin_provisioning::mapping_tree_detail::block_time const &value) {
field(*f, "block", value.block_);
field(*f, "time", value.time_);
}
};
//--------------------------------
int debug(string const &path) {
template <typename ShowTraits>
dbg::command::ptr
create_btree_node_handler(block_manager::ptr bm) {
return create_block_handler(bm, create_btree_node_dumper<ShowTraits>());
}
dbg::command::ptr
create_index_block_handler(block_manager::ptr bm) {
return create_block_handler(bm, create_index_block_dumper());
}
int debug_(string const &path) {
using dbg::command;
try {
metadata::ptr md(new metadata(path, metadata::OPEN));
command_interpreter interp(cin, cout);
interp.register_command("hello", command::ptr(new hello));
interp.register_command("superblock", command::ptr(new show_superblock(md)));
interp.register_command("m1_node", command::ptr(new show_btree_node<uint64_show_traits>(md)));
interp.register_command("m2_node", command::ptr(new show_btree_node<block_show_traits>(md)));
interp.register_command("detail_node", command::ptr(new show_btree_node<device_details_show_traits>(md)));
interp.enter_main_loop();
block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY, 1);
command_interpreter::ptr interp = create_command_interpreter(cin, cout);
interp->register_command("hello", create_hello_handler());
interp->register_command("superblock", command::ptr(new show_superblock(bm)));
interp->register_command("m1_node", create_btree_node_handler<uint64_show_traits>(bm));
interp->register_command("m2_node", create_btree_node_handler<block_show_traits>(bm));
interp->register_command("detail_node", create_btree_node_handler<device_details_show_traits>(bm));
interp->register_command("index_block", create_index_block_handler(bm));
interp->register_command("index_node", create_btree_node_handler<index_entry_show_traits>(bm));
interp->register_command("help", command::ptr(new help));
interp->register_command("exit", create_exit_handler(interp));
interp->enter_main_loop();
} catch (std::exception &e) {
cerr << e.what();
cerr << e.what() << endl;
return 1;
}
return 0;
}
void usage(string const &cmd) {
cerr << "Usage: " << cmd << " {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-V|--version}" << endl;
}
}
int main(int argc, char **argv)
thin_debug_cmd::thin_debug_cmd()
: command("thin_debug")
{
}
void
thin_debug_cmd::usage(std::ostream &out) const
{
out << "Usage: " << get_name() << " {device|file}" << endl
<< "Options:" << endl
<< " {-h|--help}" << endl
<< " {-V|--version}" << endl;
}
int
thin_debug_cmd::run(int argc, char **argv)
{
int c;
const char shortopts[] = "hV";
@ -327,7 +205,7 @@ int main(int argc, char **argv)
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
switch(c) {
case 'h':
usage(basename(argv[0]));
usage(cout);
return 0;
case 'V':
@ -337,9 +215,9 @@ int main(int argc, char **argv)
}
if (argc == optind) {
usage(basename(argv[0]));
usage(cerr);
exit(1);
}
return debug(argv[optind]);
return debug_(argv[optind]);
}

75
unit-tests/copier_t.cc
View File

@ -59,6 +59,12 @@ namespace {
}
unique_ptr<copier> make_copier() {
return make_copier(BLOCK_SIZE, 0, 0);
}
unique_ptr<copier> make_copier(sector_t block_size,
sector_t src_offset,
sector_t dest_offset) {
EXPECT_CALL(engine_, open_file(src_file_, io_engine::M_READ_ONLY, io_engine::EXCLUSIVE)).
WillOnce(Return(SRC_HANDLE));
EXPECT_CALL(engine_, open_file(dest_file_, io_engine::M_READ_WRITE, io_engine::EXCLUSIVE)).
@ -69,20 +75,22 @@ namespace {
return unique_ptr<copier>(new copier(engine_, src_file_,
dest_file_,
BLOCK_SIZE, 1 * 1024 * 1024));
block_size, 1 * 1024 * 1024,
src_offset, dest_offset));
}
static optional<wait_result> make_wr(bool success, unsigned context) {
return optional<wait_result>(wait_result(success, context));
}
// issue a copy_op, and wait for its completion synchronously
void issue_successful_op(copier &c, copy_op &op, unsigned context) {
InSequence dummy;
unsigned nr_pending = c.nr_pending();
EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ,
op.src_b * BLOCK_SIZE,
op.src_e * BLOCK_SIZE, _, context)).
expected_src_sector(c, op.src_b),
expected_src_sector(c, op.src_e), _, context)).
WillOnce(Return(true));
c.issue(op);
@ -92,8 +100,9 @@ namespace {
WillOnce(Return(make_wr(true, context)));
EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::D_WRITE,
op.dest_b * BLOCK_SIZE,
(op.dest_b + (op.src_e - op.src_b)) * BLOCK_SIZE, _, context)).
expected_dest_sector(c, op.dest_b),
expected_dest_sector(c, op.dest_b + (op.src_e - op.src_b)),
_, context)).
WillOnce(Return(true));
EXPECT_CALL(engine_, wait()).
@ -108,28 +117,63 @@ namespace {
unsigned const SRC_HANDLE = 10;
unsigned const DEST_HANDLE = 11;
StrictMock<io_engine_mock> engine_;
private:
sector_t expected_src_sector(const copier &c, block_address b) {
return c.get_src_offset() + b * c.get_block_size();
}
sector_t expected_dest_sector(const copier &c, block_address b) {
return c.get_dest_offset() + b * c.get_block_size();
}
string src_file_;
string dest_file_;
StrictMock<io_engine_mock> engine_;
};
}
//----------------------------------------------------------------
TEST_F(CopierTests, empty_test)
TEST_F(CopierTests, create_default_copier)
{
auto c = make_copier();
ASSERT_EQ(c->get_block_size(), BLOCK_SIZE);
ASSERT_EQ(c->get_src_offset(), 0u);
ASSERT_EQ(c->get_dest_offset(), 0u);
}
TEST_F(CopierTests, create_copier_with_offsets)
{
sector_t src_offset = 2048;
sector_t dest_offset = 8192;
auto c = make_copier(BLOCK_SIZE, src_offset, dest_offset);
ASSERT_EQ(c->get_block_size(), BLOCK_SIZE);
ASSERT_EQ(c->get_src_offset(), src_offset);
ASSERT_EQ(c->get_dest_offset(), dest_offset);
}
TEST_F(CopierTests, successful_copy)
{
// Copy first block
// Copy first block
copy_op op1(0, 1, 0);
auto c = make_copier();
issue_successful_op(*c, op1, 0);
}
TEST_F(CopierTests, successful_copy_with_offsets)
{
copy_op op1(0, 1, 0);
auto c = make_copier(BLOCK_SIZE, 2048, 8192);
issue_successful_op(*c, op1, 0);
}
// Verify copier's error handling against unsucessful engine operations
// at different stages.
// Test the first stage (issue_read (failed) => read => issue_write => write)
TEST_F(CopierTests, unsuccessful_issue_read)
{
copy_op op1(0, 1, 0);
@ -147,6 +191,7 @@ TEST_F(CopierTests, unsuccessful_issue_read)
ASSERT_FALSE(mop->success());
}
// Test the second stage (issue_read => read (failed) => issue_write => write)
TEST_F(CopierTests, unsuccessful_read)
{
copy_op op1(0, 1, 0);
@ -167,6 +212,7 @@ TEST_F(CopierTests, unsuccessful_read)
ASSERT_FALSE(mop->success());
}
// Test the third stage (issue_read => read => issue_write (failed) => write)
TEST_F(CopierTests, unsuccessful_issue_write)
{
copy_op op1(0, 1, 0);
@ -191,6 +237,7 @@ TEST_F(CopierTests, unsuccessful_issue_write)
ASSERT_FALSE(mop->success());
}
// Test the last stage (issue_read => read => issue_write => write (failed))
TEST_F(CopierTests, unsuccessful_write)
{
// Copy first block
@ -237,6 +284,18 @@ TEST_F(CopierTests, copy_different_blocks)
}
}
TEST_F(CopierTests, copy_different_blocks_with_offsets)
{
sector_t src_offset = 2048;
sector_t dest_offset = 8192;
auto c = make_copier(BLOCK_SIZE, src_offset, dest_offset);
for (unsigned i = 0; i < 5000; i++) {
copy_op op(i, i + 1, i);
issue_successful_op(*c, op, i);
}
}
TEST_F(CopierTests, wait_can_timeout)
{
copy_op op1(0, 1, 0);