From b67b587a109ccdab49f9d5ca1ed90a5a8fcc9467 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 22 Sep 2020 12:01:17 +0100 Subject: [PATCH 01/33] [thin_shrink] Add comment pointing people at Nikhil's PoC --- src/bin/thin_shrink.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bin/thin_shrink.rs b/src/bin/thin_shrink.rs index 2e4e270..b748b30 100644 --- a/src/bin/thin_shrink.rs +++ b/src/bin/thin_shrink.rs @@ -1,3 +1,7 @@ +// This work is based on the implementation by Nikhil Kshirsagar which +// can be found here: +// https://github.com/nkshirsagar/thinpool_shrink/blob/split_ranges/thin_shrink.py + extern crate clap; extern crate thinp; From 8d59a83f7cace1001eec2fea120dfbc041af0462 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 26 Nov 2020 10:35:40 +0800 Subject: [PATCH 02/33] [functional-tests] Update thin_delta help text --- tests/thin_delta.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/thin_delta.rs b/tests/thin_delta.rs index 49d92cb..2bc840c 100644 --- a/tests/thin_delta.rs +++ b/tests/thin_delta.rs @@ -21,7 +21,7 @@ fn accepts_version() -> Result<()> { Ok(()) } -const USAGE: &str = "Usage: thin_delta [options] \nOptions:\n {--thin1, --snap1}\n {--thin2, --snap2}\n {-m, --metadata-snap} [block#]\n {--verbose}\n {-h|--help}\n {-V|--version}"; +const USAGE: &str = "Usage: thin_delta [options] \nOptions:\n {--thin1, --snap1, --root1}\n {--thin2, --snap2, --root2}\n {-m, --metadata-snap} [block#]\n {--verbose}\n {-h|--help}\n {-V|--version}"; #[test] fn accepts_h() -> Result<()> { @@ -49,7 +49,7 @@ fn snap1_unspecified() -> Result<()> { let mut td = TestDir::new()?; let md = mk_valid_md(&mut td)?; let stderr = run_fail(thin_delta!("--snap2", "45", &md))?; - assert!(stderr.contains("--snap1 not specified")); + assert!(stderr.contains("--snap1 or --root1 not specified")); Ok(()) } @@ -58,7 +58,7 @@ fn snap2_unspecified() -> Result<()> { let mut td = TestDir::new()?; let md = mk_valid_md(&mut td)?; let stderr = run_fail(thin_delta!("--snap1", "45", &md))?; - assert!(stderr.contains("--snap2 not specified")); + assert!(stderr.contains("--snap2 or --root2 not specified")); Ok(()) } From 6a2fa73924e18b13e59ff39dba910a6d69f6c15f Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Mon, 11 Jan 2021 14:44:06 +0800 Subject: [PATCH 03/33] [build] Enable building the dev-tools for functional tests - Factor out the dev-tools into a stand-alone, no-installed program - Built the dev-tools if --enable-testing is specified - Remove the --enable-dev-tools configure option - Allow suffix on the binary name - Update symlinks - Cleanup Makefile --- Makefile.in | 84 ++++++++++++++--------------- base/application.cc | 2 +- bin/thin_generate_damage | 2 +- bin/thin_generate_mappings | 2 +- bin/thin_generate_metadata | 2 +- bin/thin_journal_check | 2 +- bin/thin_ll_dump | 2 +- bin/thin_ll_restore | 2 +- bin/thin_scan | 2 +- bin/thin_show_duplicates | 2 +- bin/thin_show_metadata | 2 +- caching/devel_commands.cc | 13 +++++ configure.ac | 8 --- era/devel_commands.cc | 12 +++++ thin-provisioning/commands.cc | 13 ----- thin-provisioning/commands.h | 65 +++++++++++----------- thin-provisioning/devel_commands.cc | 23 ++++++++ 17 files changed, 132 insertions(+), 106 deletions(-) create mode 100644 caching/devel_commands.cc create mode 100644 era/devel_commands.cc create mode 100644 thin-provisioning/devel_commands.cc diff --git a/Makefile.in b/Makefile.in index 2c3c09f..ea2bdca 100644 --- a/Makefile.in +++ b/Makefile.in @@ -22,6 +22,7 @@ PROGRAMS=\ bin/pdata_tools ifeq ("@TESTING@", "yes") +PROGRAMS += bin/pdata_tools_dev TESTLIBS=\ lib/libft.so endif @@ -31,7 +32,7 @@ all: $(PROGRAMS) $(TESTLIBS) include contrib/Makefile -SOURCE=\ +COMMON_SOURCE=\ base/output_file_requirements.cc \ base/application.cc \ base/base64.cc \ @@ -50,13 +51,6 @@ SOURCE=\ block-cache/copier.cc \ block-cache/io_engine.cc \ block-cache/mem_pool.cc \ - caching/cache_check.cc \ - caching/cache_dump.cc \ - caching/cache_metadata_size.cc \ - caching/cache_repair.cc \ - caching/cache_restore.cc \ - caching/cache_writeback.cc \ - caching/commands.cc \ caching/hint_array.cc \ caching/mapping_array.cc \ caching/metadata.cc \ @@ -64,13 +58,8 @@ SOURCE=\ caching/restore_emitter.cc \ caching/superblock.cc \ caching/xml_format.cc \ - era/commands.cc \ era/era_array.cc \ - era/era_check.cc \ era/era_detail.cc \ - era/era_dump.cc \ - era/era_invalidate.cc \ - era/era_restore.cc \ era/metadata.cc \ era/metadata_dump.cc \ era/restore_emitter.cc \ @@ -95,7 +84,6 @@ SOURCE=\ persistent-data/space_map.cc \ persistent-data/transaction_manager.cc \ persistent-data/validators.cc \ - thin-provisioning/commands.cc \ thin-provisioning/cache_stream.cc \ thin-provisioning/chunk_stream.cc \ thin-provisioning/device_tree.cc \ @@ -111,37 +99,54 @@ SOURCE=\ thin-provisioning/restore_emitter.cc \ thin-provisioning/rmap_visitor.cc \ thin-provisioning/superblock.cc \ + thin-provisioning/xml_format.cc + +TOOLS_SOURCE=\ + caching/commands.cc \ + caching/cache_check.cc \ + caching/cache_dump.cc \ + caching/cache_metadata_size.cc \ + caching/cache_repair.cc \ + caching/cache_restore.cc \ + caching/cache_writeback.cc \ + era/commands.cc \ + era/era_check.cc \ + era/era_dump.cc \ + era/era_invalidate.cc \ + era/era_restore.cc \ + thin-provisioning/commands.cc \ thin-provisioning/thin_check.cc \ thin-provisioning/thin_delta.cc \ thin-provisioning/thin_dump.cc \ thin-provisioning/thin_ls.cc \ thin-provisioning/thin_metadata_size.cc \ - thin-provisioning/thin_pool.cc \ thin-provisioning/thin_repair.cc \ thin-provisioning/thin_restore.cc \ thin-provisioning/thin_rmap.cc \ - thin-provisioning/thin_trim.cc \ - thin-provisioning/xml_format.cc + thin-provisioning/thin_trim.cc DEVTOOLS_SOURCE=\ + caching/devel_commands.cc \ + era/devel_commands.cc \ thin-provisioning/damage_generator.cc \ + thin-provisioning/devel_commands.cc \ + thin-provisioning/thin_generate_damage.cc \ + thin-provisioning/thin_generate_mappings.cc \ + thin-provisioning/thin_generate_metadata.cc \ thin-provisioning/thin_journal.cc \ thin-provisioning/thin_journal_check.cc \ thin-provisioning/thin_ll_dump.cc \ thin-provisioning/thin_ll_restore.cc \ - thin-provisioning/thin_show_duplicates.cc \ - thin-provisioning/thin_generate_damage.cc \ - thin-provisioning/thin_generate_metadata.cc \ - thin-provisioning/thin_generate_mappings.cc \ - thin-provisioning/variable_chunk_stream.cc \ thin-provisioning/thin_patch_superblock.cc \ - thin-provisioning/thin_show_metadata.cc \ + thin-provisioning/thin_pool.cc \ thin-provisioning/thin_scan.cc \ + thin-provisioning/thin_show_duplicates.cc \ + thin-provisioning/thin_show_metadata.cc \ + thin-provisioning/variable_chunk_stream.cc \ ui/ui.cc -ifeq ("@DEVTOOLS@", "yes") -SOURCE+=$(DEVTOOLS_SOURCE) -endif +SOURCE=$(COMMON_SOURCE) $(TOOLS_SOURCE) +DEV_SOURCE=$(COMMON_SOURCE) $(DEVTOOLS_SOURCE) ifeq ("@STATIC@", "yes") SOURCE += thin-provisioning/static_library_emitter.cc @@ -154,6 +159,7 @@ CXX:=@CXX@ AR:=@AR@ STRIP:=@STRIP@ OBJECTS:=$(subst .cc,.o,$(SOURCE)) +DEV_OBJECTS:=$(subst .cc,.o,$(DEV_SOURCE)) ifeq ("@STATIC@", "yes") EMITTERS += $(PLUGIN_LIBS) @@ -165,20 +171,13 @@ CFLAGS+=-g -Wall -O3 -fPIC CFLAGS+=@LFS_FLAGS@ CXXFLAGS+=-g -Wall -fPIC -fno-strict-aliasing -std=c++11 -ifeq ("@DEVTOOLS@", "yes") -CXXFLAGS+=-DDEV_TOOLS -endif - CXXFLAGS+=@CXXOPTIMISE_FLAG@ CXXFLAGS+=@CXXDEBUG_FLAG@ CXXFLAGS+=@CXX_STRERROR_FLAG@ CXXFLAGS+=@LFS_FLAGS@ INCLUDES+=-I$(TOP_BUILDDIR) -I$(TOP_DIR) -I$(TOP_DIR)/thin-provisioning LIBS:=-laio -lexpat -lboost_iostreams -ldl - -ifeq ("@DEVTOOLS@", "yes") -LIBS+=-lncurses -endif +DEV_LIBS:=-lncurses ifeq ("@STATIC_CXX@", "yes") CXXLIB+=-Wl,-Bstatic -lstdc++ -Wl,-Bdynamic -Wl,--as-needed @@ -246,10 +245,16 @@ bin/pdata_tools: $(OBJECTS) $(EMITTERS) @mkdir -p $(dir $@) $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(CXXLIB) +bin/pdata_tools_dev: $(DEV_OBJECTS) + @echo " [LD] $@" + @mkdir -p $(dir $@) + $(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS) $(DEV_LIBS) $(CXXLIB) + #---------------------------------------------------------------- DEPEND_FILES=\ $(subst .cc,.d,$(SOURCE)) \ + $(subst .cc,.d,$(DEV_SOURCE)) \ $(subst .cc,.d,$(TEST_SOURCE)) \ $(subst .cc,.d,$(CXX_PROGRAM_SOURCE)) \ $(subst .c,.d,$(C_PROGRAM_SOURCE)) @@ -260,7 +265,7 @@ clean: find . -name \*.o -delete find . -name \*.gmo -delete find . -name \*.d -delete - $(RM) $(TEST_PROGRAMS) $(PROGRAMS) $(GMOCK_OBJECTS) lib/*.a lib/*.so + $(RM) $(PROGRAMS) lib/*.a lib/*.so distclean: clean $(RM) config.cache config.log config.status configure.h version.h Makefile unit-tests/Makefile @@ -331,13 +336,6 @@ install: bin/pdata_tools $(MANPAGES) $(INSTALL_DATA) man8/era_restore.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/era_invalidate.8 $(MANPATH)/man8 $(INSTALL_DATA) man8/thin_trim.8 $(MANPATH)/man8 -ifeq ("@DEVTOOLS@", "yes") - ln -s -f pdata_tools $(BINDIR)/thin_show_duplicates - ln -s -f pdata_tools $(BINDIR)/thin_ll_dump - ln -s -f pdata_tools $(BINDIR)/thin_show_duplicates - ln -s -f pdata_tools $(BINDIR)/thin_generate_metadata - ln -s -f pdata_tools $(BINDIR)/thin_scan -endif .PHONY: install install-rust-tools rust-tools @@ -368,7 +366,7 @@ lib/libft.so: $(LIBFT_OBJECTS) .PHONEY: functional-test unit-test -functional-test: bin/pdata_tools lib/libft.so +functional-test: $(PROGRAMS) $(TESTLIBS) cd functional-tests && ./run-tests run test: functional-test unit-test diff --git a/base/application.cc b/base/application.cc index 61338ff..c0089c6 100644 --- a/base/application.cc +++ b/base/application.cc @@ -47,7 +47,7 @@ application::run(int argc, char **argv) { string cmd = get_basename(argv[0]); - if (cmd == string("pdata_tools")) { + if (cmd.find("pdata_tools") == 0) { argc--; argv++; diff --git a/bin/thin_generate_damage b/bin/thin_generate_damage index 84c01e7..c51afdf 120000 --- a/bin/thin_generate_damage +++ b/bin/thin_generate_damage @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/bin/thin_generate_mappings b/bin/thin_generate_mappings index 84c01e7..c51afdf 120000 --- a/bin/thin_generate_mappings +++ b/bin/thin_generate_mappings @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/bin/thin_generate_metadata b/bin/thin_generate_metadata index 84c01e7..c51afdf 120000 --- a/bin/thin_generate_metadata +++ b/bin/thin_generate_metadata @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/bin/thin_journal_check b/bin/thin_journal_check index 84c01e7..c51afdf 120000 --- a/bin/thin_journal_check +++ b/bin/thin_journal_check @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/bin/thin_ll_dump b/bin/thin_ll_dump index 84c01e7..c51afdf 120000 --- a/bin/thin_ll_dump +++ b/bin/thin_ll_dump @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/bin/thin_ll_restore b/bin/thin_ll_restore index 84c01e7..c51afdf 120000 --- a/bin/thin_ll_restore +++ b/bin/thin_ll_restore @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/bin/thin_scan b/bin/thin_scan index 84c01e7..c51afdf 120000 --- a/bin/thin_scan +++ b/bin/thin_scan @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/bin/thin_show_duplicates b/bin/thin_show_duplicates index 84c01e7..c51afdf 120000 --- a/bin/thin_show_duplicates +++ b/bin/thin_show_duplicates @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/bin/thin_show_metadata b/bin/thin_show_metadata index 84c01e7..c51afdf 120000 --- a/bin/thin_show_metadata +++ b/bin/thin_show_metadata @@ -1 +1 @@ -pdata_tools \ No newline at end of file +pdata_tools_dev \ No newline at end of file diff --git a/caching/devel_commands.cc b/caching/devel_commands.cc new file mode 100644 index 0000000..21a1188 --- /dev/null +++ b/caching/devel_commands.cc @@ -0,0 +1,13 @@ +#include "caching/commands.h" + +using namespace base; +using namespace caching; + +//---------------------------------------------------------------- + +void +caching::register_cache_commands(application &app) +{ +} + +//---------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index 060118d..947ffdd 100644 --- a/configure.ac +++ b/configure.ac @@ -150,14 +150,6 @@ AC_ARG_ENABLE(testing, TESTING=$enableval, TESTING=no) AC_MSG_RESULT($TESTING) -################################################################################ -dnl -- Enable development tools -AC_MSG_CHECKING(whether to enable development tools) -AC_ARG_ENABLE(dev-tools, - AC_HELP_STRING(--enable-dev-tools, [enable development tools in the makefile]), - DEVTOOLS=$enableval, DEVTOOLS=no) -AC_MSG_RESULT($DEVTOOLS) - ################################################################################ dnl -- Enable static libstdc++ AC_MSG_CHECKING(whether to statically link libstdc++) diff --git a/era/devel_commands.cc b/era/devel_commands.cc new file mode 100644 index 0000000..eac6827 --- /dev/null +++ b/era/devel_commands.cc @@ -0,0 +1,12 @@ +#include "era/commands.h" + +using namespace base; + +//---------------------------------------------------------------- + +void +era::register_era_commands(base::application &app) +{ +} + +//---------------------------------------------------------------- diff --git a/thin-provisioning/commands.cc b/thin-provisioning/commands.cc index 262b02d..c25abce 100644 --- a/thin-provisioning/commands.cc +++ b/thin-provisioning/commands.cc @@ -17,19 +17,6 @@ thin_provisioning::register_thin_commands(base::application &app) app.add_cmd(command::ptr(new thin_repair_cmd())); app.add_cmd(command::ptr(new thin_rmap_cmd())); app.add_cmd(command::ptr(new thin_trim_cmd())); - -#ifdef DEV_TOOLS - app.add_cmd(command::ptr(new thin_ll_dump_cmd())); - app.add_cmd(command::ptr(new thin_ll_restore_cmd())); - app.add_cmd(command::ptr(new thin_scan_cmd())); - app.add_cmd(command::ptr(new thin_generate_damage_cmd())); - app.add_cmd(command::ptr(new thin_generate_metadata_cmd())); - app.add_cmd(command::ptr(new thin_generate_mappings_cmd())); - app.add_cmd(command::ptr(new thin_patch_superblock_cmd())); - app.add_cmd(command::ptr(new thin_show_duplicates_cmd())); - app.add_cmd(command::ptr(new thin_show_metadata_cmd())); - app.add_cmd(command::ptr(new thin_journal_cmd())); -#endif } //---------------------------------------------------------------- diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index 88e1346..0f603ae 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -71,37 +71,7 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; -#ifdef DEV_TOOLS - class thin_ll_dump_cmd : public base::command { - public: - thin_ll_dump_cmd(); - - virtual void usage(std::ostream &out) const; - virtual int run(int argc, char **argv); - }; - - class thin_ll_restore_cmd : public base::command { - public: - thin_ll_restore_cmd(); - - virtual void usage(std::ostream &out) const; - virtual int run(int argc, char **argv); - }; - - class thin_scan_cmd : public base::command { - public: - thin_scan_cmd(); - - virtual void usage(std::ostream &out) const; - virtual int run(int argc, char **argv); - }; - - class thin_show_duplicates_cmd : public base::command { - public: - thin_show_duplicates_cmd(); - virtual void usage(std::ostream &out) const; - virtual int run(int argc, char **argv); - }; + //------------------------------------------------------ class thin_generate_damage_cmd : public base::command { public: @@ -124,6 +94,22 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_ll_dump_cmd : public base::command { + public: + thin_ll_dump_cmd(); + + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + class thin_ll_restore_cmd : public base::command { + public: + thin_ll_restore_cmd(); + + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + class thin_patch_superblock_cmd : public base::command { public: thin_patch_superblock_cmd(); @@ -131,6 +117,22 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_scan_cmd : public base::command { + public: + thin_scan_cmd(); + + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + class thin_show_duplicates_cmd : public base::command { + public: + thin_show_duplicates_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + class thin_show_metadata_cmd : public base::command { public: thin_show_metadata_cmd(); @@ -144,7 +146,6 @@ namespace thin_provisioning { virtual void usage(std::ostream &out) const; virtual int run(int argc, char **argv); }; -#endif void register_thin_commands(base::application &app); } diff --git a/thin-provisioning/devel_commands.cc b/thin-provisioning/devel_commands.cc new file mode 100644 index 0000000..7a12297 --- /dev/null +++ b/thin-provisioning/devel_commands.cc @@ -0,0 +1,23 @@ +#include "thin-provisioning/commands.h" + +using namespace base; +using namespace thin_provisioning; + +//---------------------------------------------------------------- + +void +thin_provisioning::register_thin_commands(base::application &app) +{ + app.add_cmd(command::ptr(new thin_generate_damage_cmd())); + app.add_cmd(command::ptr(new thin_generate_mappings_cmd())); + app.add_cmd(command::ptr(new thin_generate_metadata_cmd())); + app.add_cmd(command::ptr(new thin_journal_cmd())); + app.add_cmd(command::ptr(new thin_ll_dump_cmd())); + app.add_cmd(command::ptr(new thin_ll_restore_cmd())); + app.add_cmd(command::ptr(new thin_patch_superblock_cmd())); + app.add_cmd(command::ptr(new thin_scan_cmd())); + app.add_cmd(command::ptr(new thin_show_duplicates_cmd())); + app.add_cmd(command::ptr(new thin_show_metadata_cmd())); +} + +//---------------------------------------------------------------- From 4c17076f0964c46c43f906c5845e5c157551ade5 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Mon, 18 Jan 2021 01:11:01 +0800 Subject: [PATCH 04/33] [cache_writeback] Support offset within the source and destination devices --- block-cache/copier.cc | 22 +++++++---- block-cache/copier.h | 16 +++++++- caching/cache_writeback.cc | 27 +++++++++++++- unit-tests/copier_t.cc | 75 ++++++++++++++++++++++++++++++++++---- 4 files changed, 122 insertions(+), 18 deletions(-) diff --git a/block-cache/copier.cc b/block-cache/copier.cc index f200878..2e0d0d2 100644 --- a/block-cache/copier.cc +++ b/block-cache/copier.cc @@ -10,12 +10,14 @@ using namespace std; copier::copier(io_engine &engine, string const &src, string const &dest, - sector_t block_size, size_t mem) + sector_t block_size, size_t mem, + sector_t src_offset, sector_t dest_offset) : pool_(block_size * 512, mem, PAGE_SIZE), block_size_(block_size), engine_(engine), src_handle_(engine_.open_file(src, io_engine::M_READ_ONLY)), dest_handle_(engine_.open_file(dest, io_engine::M_READ_WRITE)), + src_offset_(src_offset), dest_offset_(dest_offset), genkey_count_(0) { } @@ -45,8 +47,8 @@ copier::issue(copy_op const &op) auto r = engine_.issue_io(src_handle_, io_engine::D_READ, - to_sector(op.src_b), - to_sector(op.src_e), + to_src_sector(op.src_b), + to_src_sector(op.src_e), data, key); @@ -151,8 +153,8 @@ copier::wait_successful(io_engine::wait_result const &p) j.op.read_complete = true; if (!engine_.issue_io(dest_handle_, io_engine::D_WRITE, - to_sector(j.op.dest_b), - to_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)), + to_dest_sector(j.op.dest_b), + to_dest_sector(j.op.dest_b + (j.op.src_e - j.op.src_b)), j.data, it->first)) { complete(j); @@ -177,9 +179,15 @@ copier::complete(copy_job const &j) } sector_t -copier::to_sector(block_address b) const +copier::to_src_sector(block_address b) const { - return b * block_size_; + return src_offset_ + b * block_size_; +} + +sector_t +copier::to_dest_sector(block_address b) const +{ + return dest_offset_ + b * block_size_; } unsigned diff --git a/block-cache/copier.h b/block-cache/copier.h index ef8211a..a63f76b 100644 --- a/block-cache/copier.h +++ b/block-cache/copier.h @@ -61,13 +61,22 @@ namespace bcache { public: copier(io_engine &engine, std::string const &src, std::string const &dest, - sector_t block_size, size_t mem); + sector_t block_size, size_t mem, + sector_t src_offset, sector_t dest_offset); ~copier(); sector_t get_block_size() const { return block_size_; } + sector_t get_src_offset() const { + return src_offset_; + } + + sector_t get_dest_offset() const { + return dest_offset_; + } + // Blocks if out of memory. void issue(copy_op const &op); @@ -83,7 +92,8 @@ namespace bcache { void wait_(); void complete(copy_job const &j); - sector_t to_sector(block_address b) const; + sector_t to_src_sector(block_address b) const; + sector_t to_dest_sector(block_address b) const; unsigned genkey(); mempool pool_; @@ -91,6 +101,8 @@ namespace bcache { io_engine &engine_; io_engine::handle src_handle_; io_engine::handle dest_handle_; + sector_t src_offset_; + sector_t dest_offset_; unsigned genkey_count_; using job_map = std::map; diff --git a/caching/cache_writeback.cc b/caching/cache_writeback.cc index 416e0d5..24a2385 100644 --- a/caching/cache_writeback.cc +++ b/caching/cache_writeback.cc @@ -31,6 +31,8 @@ namespace { flags() : cache_size(4 * 1024 * 1024), sort_buffers(16 * 1024), + origin_dev_offset(0), + fast_dev_offset(0), list_failed_blocks(false), update_metadata(true) { } @@ -52,6 +54,8 @@ namespace { maybe_string metadata_dev; maybe_string origin_dev; maybe_string fast_dev; + sector_t origin_dev_offset; + sector_t fast_dev_offset; bool list_failed_blocks; bool update_metadata; }; @@ -303,7 +307,9 @@ namespace { unsigned max_ios = f.cache_size / (md.sb_.data_block_size << SECTOR_SHIFT); aio_engine engine(max_ios); copier c(engine, *f.fast_dev, *f.origin_dev, - md.sb_.data_block_size, f.cache_size); + md.sb_.data_block_size, f.cache_size, + f.fast_dev_offset >> SECTOR_SHIFT, + f.origin_dev_offset >> SECTOR_SHIFT); auto bar = create_progress_bar("Copying data"); copy_visitor cv(c, f.sort_buffers, clean_shutdown(md), f.list_failed_blocks, @@ -364,6 +370,8 @@ cache_writeback_cmd::usage(std::ostream &out) const << "\t\t--buffer-size-meg \n" << "\t\t--list-failed-blocks\n" << "\t\t--no-metadata-update\n" + << "\t\t--origin-device-offset \n" + << "\t\t--fast-device-offset \n" << "Options:\n" << " {-h|--help}\n" << " {-V|--version}" << endl; @@ -382,6 +390,8 @@ cache_writeback_cmd::run(int argc, char **argv) { "buffer-size-meg", required_argument, NULL, 3 }, { "list-failed-blocks", no_argument, NULL, 4 }, { "no-metadata-update", no_argument, NULL, 5 }, + { "origin-device-offset", required_argument, NULL, 6 }, + { "fast-device-offset", required_argument, NULL, 7 }, { "help", no_argument, NULL, 'h'}, { "version", no_argument, NULL, 'V'}, { NULL, no_argument, NULL, 0 } @@ -413,6 +423,14 @@ cache_writeback_cmd::run(int argc, char **argv) fs.update_metadata = false; break; + case 6: + fs.origin_dev_offset = parse_uint64(optarg, "origin dev offset"); + break; + + case 7: + fs.fast_dev_offset = parse_uint64(optarg, "fast dev offset"); + break; + case 'h': usage(cout); return 0; @@ -452,6 +470,13 @@ cache_writeback_cmd::run(int argc, char **argv) return 1; } + if (fs.origin_dev_offset & (SECTOR_SHIFT - 1) || + fs.fast_dev_offset & (SECTOR_SHIFT - 1)) { + cerr << "Offset must be sector-aligned\n\n"; + usage(cerr); + return 1; + } + return writeback(fs); } diff --git a/unit-tests/copier_t.cc b/unit-tests/copier_t.cc index d1a6687..974f886 100644 --- a/unit-tests/copier_t.cc +++ b/unit-tests/copier_t.cc @@ -59,6 +59,12 @@ namespace { } unique_ptr make_copier() { + return make_copier(BLOCK_SIZE, 0, 0); + } + + unique_ptr make_copier(sector_t block_size, + sector_t src_offset, + sector_t dest_offset) { EXPECT_CALL(engine_, open_file(src_file_, io_engine::M_READ_ONLY, io_engine::EXCLUSIVE)). WillOnce(Return(SRC_HANDLE)); EXPECT_CALL(engine_, open_file(dest_file_, io_engine::M_READ_WRITE, io_engine::EXCLUSIVE)). @@ -69,20 +75,22 @@ namespace { return unique_ptr(new copier(engine_, src_file_, dest_file_, - BLOCK_SIZE, 1 * 1024 * 1024)); + block_size, 1 * 1024 * 1024, + src_offset, dest_offset)); } static optional make_wr(bool success, unsigned context) { return optional(wait_result(success, context)); } + // issue a copy_op, and wait for its completion synchronously void issue_successful_op(copier &c, copy_op &op, unsigned context) { InSequence dummy; unsigned nr_pending = c.nr_pending(); EXPECT_CALL(engine_, issue_io(SRC_HANDLE, io_engine::D_READ, - op.src_b * BLOCK_SIZE, - op.src_e * BLOCK_SIZE, _, context)). + expected_src_sector(c, op.src_b), + expected_src_sector(c, op.src_e), _, context)). WillOnce(Return(true)); c.issue(op); @@ -92,8 +100,9 @@ namespace { WillOnce(Return(make_wr(true, context))); EXPECT_CALL(engine_, issue_io(DEST_HANDLE, io_engine::D_WRITE, - op.dest_b * BLOCK_SIZE, - (op.dest_b + (op.src_e - op.src_b)) * BLOCK_SIZE, _, context)). + expected_dest_sector(c, op.dest_b), + expected_dest_sector(c, op.dest_b + (op.src_e - op.src_b)), + _, context)). WillOnce(Return(true)); EXPECT_CALL(engine_, wait()). @@ -108,28 +117,63 @@ namespace { unsigned const SRC_HANDLE = 10; unsigned const DEST_HANDLE = 11; + StrictMock engine_; + + private: + sector_t expected_src_sector(const copier &c, block_address b) { + return c.get_src_offset() + b * c.get_block_size(); + } + + sector_t expected_dest_sector(const copier &c, block_address b) { + return c.get_dest_offset() + b * c.get_block_size(); + } + string src_file_; string dest_file_; - StrictMock engine_; }; } //---------------------------------------------------------------- -TEST_F(CopierTests, empty_test) +TEST_F(CopierTests, create_default_copier) { auto c = make_copier(); + ASSERT_EQ(c->get_block_size(), BLOCK_SIZE); + ASSERT_EQ(c->get_src_offset(), 0u); + ASSERT_EQ(c->get_dest_offset(), 0u); +} + +TEST_F(CopierTests, create_copier_with_offsets) +{ + sector_t src_offset = 2048; + sector_t dest_offset = 8192; + + auto c = make_copier(BLOCK_SIZE, src_offset, dest_offset); + ASSERT_EQ(c->get_block_size(), BLOCK_SIZE); + ASSERT_EQ(c->get_src_offset(), src_offset); + ASSERT_EQ(c->get_dest_offset(), dest_offset); } TEST_F(CopierTests, successful_copy) { - // Copy first block + // Copy first block copy_op op1(0, 1, 0); auto c = make_copier(); issue_successful_op(*c, op1, 0); } +TEST_F(CopierTests, successful_copy_with_offsets) +{ + copy_op op1(0, 1, 0); + + auto c = make_copier(BLOCK_SIZE, 2048, 8192); + issue_successful_op(*c, op1, 0); +} + +// Verify copier's error handling against unsucessful engine operations +// at different stages. +// Test the first stage (issue_read (failed) => read => issue_write => write) TEST_F(CopierTests, unsuccessful_issue_read) { copy_op op1(0, 1, 0); @@ -147,6 +191,7 @@ TEST_F(CopierTests, unsuccessful_issue_read) ASSERT_FALSE(mop->success()); } +// Test the second stage (issue_read => read (failed) => issue_write => write) TEST_F(CopierTests, unsuccessful_read) { copy_op op1(0, 1, 0); @@ -167,6 +212,7 @@ TEST_F(CopierTests, unsuccessful_read) ASSERT_FALSE(mop->success()); } +// Test the third stage (issue_read => read => issue_write (failed) => write) TEST_F(CopierTests, unsuccessful_issue_write) { copy_op op1(0, 1, 0); @@ -191,6 +237,7 @@ TEST_F(CopierTests, unsuccessful_issue_write) ASSERT_FALSE(mop->success()); } +// Test the last stage (issue_read => read => issue_write => write (failed)) TEST_F(CopierTests, unsuccessful_write) { // Copy first block @@ -237,6 +284,18 @@ TEST_F(CopierTests, copy_different_blocks) } } +TEST_F(CopierTests, copy_different_blocks_with_offsets) +{ + sector_t src_offset = 2048; + sector_t dest_offset = 8192; + + auto c = make_copier(BLOCK_SIZE, src_offset, dest_offset); + for (unsigned i = 0; i < 5000; i++) { + copy_op op(i, i + 1, i); + issue_successful_op(*c, op, i); + } +} + TEST_F(CopierTests, wait_can_timeout) { copy_op op1(0, 1, 0); From 087a4711a59c62bec090f8feae5bc90e1f5d64d9 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 9 Dec 2020 18:40:06 +0800 Subject: [PATCH 05/33] [array (rust)] Implement array basis --- src/pdata/array.rs | 39 ++++++++++++++ src/pdata/array_block.rs | 70 ++++++++++++++++++++++++ src/pdata/array_walker.rs | 108 ++++++++++++++++++++++++++++++++++++++ src/pdata/mod.rs | 3 ++ 4 files changed, 220 insertions(+) create mode 100644 src/pdata/array.rs create mode 100644 src/pdata/array_block.rs create mode 100644 src/pdata/array_walker.rs diff --git a/src/pdata/array.rs b/src/pdata/array.rs new file mode 100644 index 0000000..d8299c2 --- /dev/null +++ b/src/pdata/array.rs @@ -0,0 +1,39 @@ +use nom::{number::complete::*, IResult}; +use std::fmt; + +use crate::pdata::unpack::*; + +//------------------------------------------ + +// TODO: build a data structure representating an array? + +// FIXME: rename this struct +pub struct ArrayBlockEntry { + pub block: u64, +} + +impl Unpack for ArrayBlockEntry { + fn disk_size() -> u32 { + 8 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], ArrayBlockEntry> { + let (i, n) = le_u64(i)?; + let block = n; + + Ok(( + i, + ArrayBlockEntry { + block, + } + )) + } +} + +impl fmt::Display for ArrayBlockEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.block) + } +} + +//------------------------------------------ diff --git a/src/pdata/array_block.rs b/src/pdata/array_block.rs new file mode 100644 index 0000000..0cf7069 --- /dev/null +++ b/src/pdata/array_block.rs @@ -0,0 +1,70 @@ +use anyhow::anyhow; +use anyhow::Result; +use nom::{multi::count, number::complete::*, IResult}; + +use crate::pdata::unpack::Unpack; + +//------------------------------------------ + +pub struct ArrayBlockHeader { + pub csum: u32, + pub max_entries: u32, + pub nr_entries: u32, + pub value_size: u32, + pub blocknr: u64, +} + +impl Unpack for ArrayBlockHeader { + fn disk_size() -> u32 { + 24 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], ArrayBlockHeader> { + let (i, csum) = le_u32(data)?; + let (i, max_entries) = le_u32(i)?; + let (i, nr_entries) = le_u32(i)?; + let (i, value_size) = le_u32(i)?; + let (i, blocknr) = le_u64(i)?; + + Ok(( + i, + ArrayBlockHeader { + csum, + max_entries, + nr_entries, + value_size, + blocknr + }, + )) + } +} + +pub struct ArrayBlock { + pub header: ArrayBlockHeader, + pub values: Vec, +} + +impl ArrayBlock { + pub fn get_header(&self) -> &ArrayBlockHeader { + &self.header + } +} + +fn convert_result<'a, V>(r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> { + r.map_err(|_| anyhow!("parse error")) +} + +pub fn unpack_array_block( + data: &[u8], +) -> Result> { + // TODO: collect errors + let (i, header) = ArrayBlockHeader::unpack(data).map_err(|_e| anyhow!("Couldn't parse header"))?; + let (_i, values) = convert_result(count(V::unpack, header.nr_entries as usize)(i))?; + + Ok(ArrayBlock { + header, + values, + }) +} + +//------------------------------------------ diff --git a/src/pdata/array_walker.rs b/src/pdata/array_walker.rs new file mode 100644 index 0000000..6322c50 --- /dev/null +++ b/src/pdata/array_walker.rs @@ -0,0 +1,108 @@ +use std::sync::Arc; + +use crate::io_engine::*; +use crate::pdata::array::*; +use crate::pdata::array_block::*; +use crate::pdata::btree::*; +use crate::pdata::btree_walker::*; +use crate::pdata::unpack::*; + +//------------------------------------------ + +pub struct ArrayWalker { + engine: Arc, + ignore_non_fatal: bool +} + +// FIXME: define another Result type for array visiting? +pub trait ArrayBlockVisitor { + fn visit( + &self, + index: u64, + v: V, + ) -> anyhow::Result<()>; +} + +struct BlockValueVisitor { + engine: Arc, + array_block_visitor: Box>, +} + +impl BlockValueVisitor { + pub fn new>( + e: Arc, + v: Box, + ) -> BlockValueVisitor { + BlockValueVisitor { + engine: e, + array_block_visitor: v, + } + } + + pub fn visit_array_block( + &self, + index: u64, + array_block: ArrayBlock, + ) { + let begin = index * u64::from(array_block.header.nr_entries); + for i in 0..array_block.header.nr_entries { + self.array_block_visitor.visit(begin + u64::from(i), array_block.values[i as usize]).unwrap(); + } + } +} + +impl NodeVisitor for BlockValueVisitor { + // FIXME: return errors + fn visit( + &self, + path: &Vec, + _kr: &KeyRange, + _h: &NodeHeader, + keys: &[u64], + values: &[ArrayBlockEntry], + ) -> Result<()> { + for n in 0..keys.len() { + let index = keys[n]; + let b = self.engine.read(values[n].block).map_err(|_| io_err(path))?; + let array_block = unpack_array_block::(b.get_data()).map_err(|_| io_err(path))?; + self.visit_array_block(index, array_block); + } + Ok(()) + } + + // FIXME: stub + fn visit_again(&self, _path: &Vec, _b: u64) -> Result<()> { + Ok(()) + } + + // FIXME: stub + fn end_walk(&self) -> Result<()> { + Ok(()) + } +} + +impl ArrayWalker { + pub fn new(engine: Arc, ignore_non_fatal: bool) -> ArrayWalker { + let r: ArrayWalker = ArrayWalker { + engine, + ignore_non_fatal, + }; + r + } + + // FIXME: pass the visitor by reference? + // FIXME: redefine the Result type for array visiting? + pub fn walk(&self, visitor: Box, root: u64) -> Result<()> + where + BV: 'static + ArrayBlockVisitor, + V: Unpack, + { + let w = BTreeWalker::new(self.engine.clone(), self.ignore_non_fatal); + let mut path = Vec::new(); // FIXME: eliminate this line? + path.push(0); + let v = BlockValueVisitor::::new(self.engine.clone(), visitor); + w.walk(&mut path, &v, root) + } +} + +//------------------------------------------ diff --git a/src/pdata/mod.rs b/src/pdata/mod.rs index 26a7318..370d244 100644 --- a/src/pdata/mod.rs +++ b/src/pdata/mod.rs @@ -1,3 +1,6 @@ +pub mod array; +pub mod array_block; +pub mod array_walker; pub mod btree; pub mod btree_builder; pub mod btree_merge; From faa371c20863d909dc85844063e346ba8c3b67cf Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 15 Dec 2020 20:03:25 +0800 Subject: [PATCH 06/33] [cache (rust)] Implement cache primitives --- Cargo.lock | 7 ++ Cargo.toml | 1 + src/cache/hint.rs | 33 ++++++++++ src/cache/mapping.rs | 50 +++++++++++++++ src/cache/mod.rs | 3 + src/cache/superblock.rs | 139 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 233 insertions(+) create mode 100644 src/cache/hint.rs create mode 100644 src/cache/mapping.rs create mode 100644 src/cache/superblock.rs diff --git a/Cargo.lock b/Cargo.lock index 761163b..219c98a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -803,6 +803,7 @@ dependencies = [ "thiserror", "threadpool", "tui", + "typenum", ] [[package]] @@ -856,6 +857,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + [[package]] name = "unicode-segmentation" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index a98b956..9221f2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ threadpool = "1.8" thiserror = "1.0" tui = "0.10" termion = "1.5" +typenum = "1.12.0" [dev-dependencies] json = "0.12" diff --git a/src/cache/hint.rs b/src/cache/hint.rs new file mode 100644 index 0000000..ef0d83f --- /dev/null +++ b/src/cache/hint.rs @@ -0,0 +1,33 @@ +use nom::IResult; +use std::marker::PhantomData; +use std::convert::TryInto; + +use crate::pdata::unpack::*; + +//------------------------------------------ + +#[derive(Clone, Copy)] +pub struct Hint { + pub hint: [u8; 4], // FIXME: support various hint sizes + _not_used: PhantomData, +} + +impl Unpack for Hint { + fn disk_size() -> u32 { + Width::to_u32() + } + + // FIXME: support different width + fn unpack(i: &[u8]) -> IResult<&[u8], Hint> { + let size = Width::to_usize(); + Ok(( + &i[size..], + Hint { + hint: i[0..size].try_into().unwrap(), + _not_used: PhantomData, + }, + )) + } +} + +//------------------------------------------ diff --git a/src/cache/mapping.rs b/src/cache/mapping.rs new file mode 100644 index 0000000..c18325a --- /dev/null +++ b/src/cache/mapping.rs @@ -0,0 +1,50 @@ +use nom::IResult; +use nom::number::complete::*; + +use crate::pdata::unpack::*; + +//------------------------------------------ + +static FLAGS_MASK: u64 = (1 << 16) - 1; + +//------------------------------------------ + +pub enum MappingFlags { + Valid = 1, + Dirty = 2, +} + +#[derive(Clone, Copy)] +pub struct Mapping { + pub oblock: u64, + pub flags: u32, +} + +impl Mapping { + pub fn is_valid(&self) -> bool { + return (self.flags & MappingFlags::Valid as u32) != 0; + } +} + + +impl Unpack for Mapping { + fn disk_size() -> u32 { + 8 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], Mapping> { + let (i, n) = le_u64(i)?; + let oblock = n >> 16; + let flags = n & FLAGS_MASK; + + Ok(( + i, + Mapping { + oblock, + flags: flags as u32, + }, + )) + } +} + +//------------------------------------------ diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 2910ec6..f1cf219 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1 +1,4 @@ +pub mod hint; +pub mod mapping; +pub mod superblock; pub mod xml; diff --git a/src/cache/superblock.rs b/src/cache/superblock.rs new file mode 100644 index 0000000..e98a3e9 --- /dev/null +++ b/src/cache/superblock.rs @@ -0,0 +1,139 @@ +use anyhow::{anyhow, Result}; + +use crate::io_engine::*; +use nom::{bytes::complete::*, number::complete::*, IResult}; + +//------------------------------------------ + +pub const SUPERBLOCK_LOCATION: u64 = 0; + +const POLICY_NAME_SIZE: usize = 16; +const SPACE_MAP_ROOT_SIZE: usize = 128; + +//------------------------------------------ + +#[derive(Debug, Clone)] +pub struct SuperblockFlags { + pub needs_check: bool, +} + +#[derive(Debug, Clone)] +pub struct Superblock { + pub flags: SuperblockFlags, + pub block: u64, + pub version: u32, + + pub policy_name: Vec, + pub policy_version: Vec, + pub policy_hint_size: u32, + + pub metadata_sm_root: Vec, + pub mapping_root: u64, + pub dirty_root: Option, // format 2 only + pub hint_root: u64, + + pub discard_root: u64, + pub discard_block_size: u64, + pub discard_nr_blocks: u64, + + pub data_block_size: u32, + pub metadata_block_size: u32, + pub cache_blocks: u32, + + pub compat_flags: u32, + pub compat_ro_flags: u32, + pub incompat_flags: u32, + + pub read_hits: u32, + pub read_misses: u32, + pub write_hits: u32, + pub write_misses: u32, +} + +fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { + let (i, _csum) = le_u32(data)?; + let (i, flags) = le_u32(i)?; + let (i, block) = le_u64(i)?; + let (i, _uuid) = take(16usize)(i)?; + let (i, _magic) = le_u64(i)?; + let (i, version) = le_u32(i)?; + + let (i, policy_name) = take(POLICY_NAME_SIZE)(i)?; + let (i, policy_hint_size) = le_u32(i)?; + + let (i, metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; + let (i, mapping_root) = le_u64(i)?; + let (i, hint_root) = le_u64(i)?; + + let (i, discard_root) = le_u64(i)?; + let (i, discard_block_size) = le_u64(i)?; + let (i, discard_nr_blocks) = le_u64(i)?; + + let (i, data_block_size) = le_u32(i)?; + let (i, metadata_block_size) = le_u32(i)?; + let (i, cache_blocks) = le_u32(i)?; + + let (i, compat_flags) = le_u32(i)?; + let (i, compat_ro_flags) = le_u32(i)?; + let (i, incompat_flags) = le_u32(i)?; + + let (i, read_hits) = le_u32(i)?; + let (i, read_misses) = le_u32(i)?; + let (i, write_hits) = le_u32(i)?; + let (i, write_misses) = le_u32(i)?; + + let (i, vsn_major) = le_u32(i)?; + let (i, vsn_minor) = le_u32(i)?; + let (i, vsn_patch) = le_u32(i)?; + + let mut i = i; + let mut dirty_root = None; + if version >= 2 { + let (m, root) = le_u64(i)?; + dirty_root = Some(root); + i = &m; + } + + Ok(( + i, + Superblock { + flags: SuperblockFlags { + needs_check: (flags | 0x1) != 0, + }, + block, + version, + policy_name: policy_name.to_vec(), + policy_version: vec![vsn_major, vsn_minor, vsn_patch], + policy_hint_size, + metadata_sm_root: metadata_sm_root.to_vec(), + mapping_root, + dirty_root, + hint_root, + discard_root, + discard_block_size, + discard_nr_blocks, + data_block_size, + metadata_block_size, + cache_blocks, + compat_flags, + compat_ro_flags, + incompat_flags, + read_hits, + read_misses, + write_hits, + write_misses, + }, + )) +} + +pub fn read_superblock(engine: &dyn IoEngine, loc: u64) -> Result { + let b = engine.read(loc)?; + + if let Ok((_, sb)) = unpack(&b.get_data()) { + Ok(sb) + } else { + Err(anyhow!("couldn't unpack superblock")) + } +} + +//------------------------------------------ From 0ce026caf5d3e8f7915f4d5c59ff0026507efa0c Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 19 Feb 2016 00:35:10 +0800 Subject: [PATCH 07/33] [thin_debug] Integrate thin_debug into the main program --- Makefile.in | 1 + thin-provisioning/commands.h | 21 ++++++++---- thin-provisioning/devel_commands.cc | 1 + thin-provisioning/thin_debug.cc | 53 ++++++++++++++++++----------- 4 files changed, 50 insertions(+), 26 deletions(-) diff --git a/Makefile.in b/Makefile.in index ea2bdca..c70159d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -130,6 +130,7 @@ DEVTOOLS_SOURCE=\ era/devel_commands.cc \ thin-provisioning/damage_generator.cc \ thin-provisioning/devel_commands.cc \ + thin-provisioning/thin_debug.cc \ thin-provisioning/thin_generate_damage.cc \ thin-provisioning/thin_generate_mappings.cc \ thin-provisioning/thin_generate_metadata.cc \ diff --git a/thin-provisioning/commands.h b/thin-provisioning/commands.h index 0f603ae..2067970 100644 --- a/thin-provisioning/commands.h +++ b/thin-provisioning/commands.h @@ -73,6 +73,13 @@ namespace thin_provisioning { //------------------------------------------------------ + class thin_debug_cmd : public base::command { + public: + thin_debug_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + class thin_generate_damage_cmd : public base::command { public: thin_generate_damage_cmd(); @@ -94,6 +101,13 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; + class thin_journal_cmd : public base::command { + public: + thin_journal_cmd(); + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + class thin_ll_dump_cmd : public base::command { public: thin_ll_dump_cmd(); @@ -140,13 +154,6 @@ namespace thin_provisioning { virtual int run(int argc, char **argv); }; - class thin_journal_cmd : public base::command { - public: - thin_journal_cmd(); - virtual void usage(std::ostream &out) const; - virtual int run(int argc, char **argv); - }; - void register_thin_commands(base::application &app); } diff --git a/thin-provisioning/devel_commands.cc b/thin-provisioning/devel_commands.cc index 7a12297..e0f0ae5 100644 --- a/thin-provisioning/devel_commands.cc +++ b/thin-provisioning/devel_commands.cc @@ -8,6 +8,7 @@ using namespace thin_provisioning; void thin_provisioning::register_thin_commands(base::application &app) { + app.add_cmd(command::ptr(new thin_debug_cmd())); app.add_cmd(command::ptr(new thin_generate_damage_cmd())); app.add_cmd(command::ptr(new thin_generate_mappings_cmd())); app.add_cmd(command::ptr(new thin_generate_metadata_cmd())); diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index 48ad544..bee3548 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -29,6 +29,9 @@ #include #include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/simple_traits.h" +#include "persistent-data/file_utils.h" +#include "thin-provisioning/commands.h" #include "thin-provisioning/metadata.h" #include "thin-provisioning/metadata_checker.h" #include "version.h" @@ -43,7 +46,7 @@ namespace { class formatter { public: - typedef shared_ptr ptr; + typedef std::shared_ptr ptr; virtual ~formatter() {} @@ -60,8 +63,8 @@ namespace { virtual void output(ostream &out, int depth = 0) = 0; protected: - typedef variant value; - typedef tuple field_type; + typedef boost::variant value; + typedef boost::tuple field_type; vector fields_; }; @@ -183,7 +186,7 @@ namespace { virtual void exec(strings const &args, ostream &out) { xml_formatter f; - superblock const &sb = md_->sb_; + thin_provisioning::superblock_detail::superblock const &sb = md_->sb_; field(f, "csum", sb.csum_); field(f, "flags", sb.flags_); @@ -210,9 +213,10 @@ namespace { metadata::ptr md_; }; - class device_details_show_traits : public device_details_traits { + class device_details_show_traits : public thin_provisioning::device_tree_detail::device_details_traits { public: - static void show(formatter &f, string const &key, device_details const &value) { + static void show(formatter &f, string const &key, + thin_provisioning::device_tree_detail::device_details const &value) { field(f, "mapped blocks", value.mapped_blocks_); field(f, "transaction id", value.transaction_id_); field(f, "creation time", value.creation_time_); @@ -227,9 +231,10 @@ namespace { } }; - class block_show_traits : public block_traits { + class block_show_traits : public thin_provisioning::mapping_tree_detail::block_traits { public: - static void show(formatter &f, string const &key, block_time const &value) { + static void show(formatter &f, string const &key, + thin_provisioning::mapping_tree_detail::block_time const &value) { field(f, "block", value.block_); field(f, "time", value.time_); } @@ -249,7 +254,7 @@ namespace { throw runtime_error("incorrect number of arguments"); block_address block = lexical_cast(args[1]); - block_manager<>::read_ref rr = md_->tm_->read_lock(block); + block_manager::read_ref rr = md_->tm_->read_lock(block); node_ref n = btree_detail::to_node(rr); if (n.get_type() == INTERNAL) @@ -289,7 +294,8 @@ namespace { int debug(string const &path) { try { - metadata::ptr md(new metadata(path, metadata::OPEN)); + block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY, 1); + metadata::ptr md(new metadata(bm, false)); command_interpreter interp(cin, cout); interp.register_command("hello", command::ptr(new hello)); interp.register_command("superblock", command::ptr(new show_superblock(md))); @@ -306,15 +312,24 @@ namespace { return 0; } - void usage(string const &cmd) { - cerr << "Usage: " << cmd << " {device|file}" << endl - << "Options:" << endl - << " {-h|--help}" << endl - << " {-V|--version}" << endl; - } } -int main(int argc, char **argv) +thin_debug_cmd::thin_debug_cmd() + : command("thin_debug") +{ +} + +void +thin_debug_cmd::usage(std::ostream &out) const +{ + out << "Usage: " << get_name() << " {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-V|--version}" << endl; +} + +int +thin_debug_cmd::run(int argc, char **argv) { int c; const char shortopts[] = "hV"; @@ -327,7 +342,7 @@ int main(int argc, char **argv) while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { switch(c) { case 'h': - usage(basename(argv[0])); + usage(cout); return 0; case 'V': @@ -337,7 +352,7 @@ int main(int argc, char **argv) } if (argc == optind) { - usage(basename(argv[0])); + usage(cerr); exit(1); } From 62d09c675226d891fdf4a69016de7fb6fec3959b Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 12 Jan 2021 18:09:00 +0800 Subject: [PATCH 08/33] [thin_debug] Reduce code size by eliminating duplicated types This patch doesn't have a significant effect - only a few KBs of code is reduced. However, it's still a nice have. --- thin-provisioning/thin_debug.cc | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index bee3548..8a175e0 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -215,6 +215,8 @@ namespace { class device_details_show_traits : public thin_provisioning::device_tree_detail::device_details_traits { public: + typedef thin_provisioning::device_tree_detail::device_details_traits value_trait; + static void show(formatter &f, string const &key, thin_provisioning::device_tree_detail::device_details const &value) { field(f, "mapped blocks", value.mapped_blocks_); @@ -226,6 +228,8 @@ namespace { class uint64_show_traits : public uint64_traits { public: + typedef uint64_traits value_trait; + static void show(formatter &f, string const &key, uint64_t const &value) { field(f, key, lexical_cast(value)); } @@ -233,6 +237,8 @@ namespace { class block_show_traits : public thin_provisioning::mapping_tree_detail::block_traits { public: + typedef thin_provisioning::mapping_tree_detail::block_traits value_trait; + static void show(formatter &f, string const &key, thin_provisioning::mapping_tree_detail::block_time const &value) { field(f, "block", value.block_); @@ -240,7 +246,7 @@ namespace { } }; - template + template class show_btree_node : public command { public: explicit show_btree_node(metadata::ptr md) @@ -256,18 +262,18 @@ namespace { block_address block = lexical_cast(args[1]); block_manager::read_ref rr = md_->tm_->read_lock(block); - node_ref n = btree_detail::to_node(rr); + node_ref n = btree_detail::to_node(rr); if (n.get_type() == INTERNAL) show_node(n, out); else { - node_ref n = btree_detail::to_node(rr); - show_node(n, out); + node_ref n = btree_detail::to_node(rr); + show_node(n, out); } } private: - template - void show_node(node_ref n, ostream &out) { + template + void show_node(node_ref n, ostream &out) { xml_formatter f; field(f, "csum", n.get_checksum()); @@ -280,7 +286,7 @@ namespace { for (unsigned i = 0; i < n.get_nr_entries(); i++) { formatter::ptr f2(new xml_formatter); field(*f2, "key", n.key_at(i)); - VT::show(*f2, "value", n.value_at(i)); + ST::show(*f2, "value", n.value_at(i)); f.child("child", f2); } From 127f44c66b885d4c14de588a7e6f5cd961c9704b Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Tue, 12 Jan 2021 20:59:02 +0800 Subject: [PATCH 09/33] [thin_debug] Enhance error handling - Handle exception thrown by commands - Add help and exit commands --- thin-provisioning/thin_debug.cc | 62 ++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index 8a175e0..2340479 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -34,6 +34,7 @@ #include "thin-provisioning/commands.h" #include "thin-provisioning/metadata.h" #include "thin-provisioning/metadata_checker.h" +#include "thin-provisioning/superblock.h" #include "version.h" using namespace boost; @@ -122,9 +123,12 @@ namespace { class command_interpreter { public: + typedef std::shared_ptr ptr; + command_interpreter(istream &in, ostream &out) : in_(in), - out_(out) { + out_(out), + exit_(false) { } void register_command(string const &str, command::ptr cmd) { @@ -132,10 +136,14 @@ namespace { } void enter_main_loop() { - while (true) + while (!exit_) do_once(); } + void exit_main_loop() { + exit_ = true; + } + private: strings read_input() { using namespace boost::algorithm; @@ -160,13 +168,19 @@ namespace { it = commands_.find(args[0]); if (it == commands_.end()) out_ << "Unrecognised command" << endl; - else - it->second->exec(args, out_); + else { + try { + it->second->exec(args, out_); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + } } istream &in_; ostream &out_; map commands_; + bool exit_; }; //-------------------------------- @@ -177,6 +191,31 @@ namespace { } }; + class help : public command { + virtual void exec(strings const &args, ostream &out) { + out << "Commands:" << endl + << " superblock" << endl + << " m1_node " << endl + << " m2_node " << endl + << " detail_node " << endl + << " exit" << endl; + } + }; + + class exit_handler : public command { + public: + exit_handler(command_interpreter &interpreter) + : interpreter_(interpreter) { + } + + virtual void exec(strings const &args, ostream &out) { + out << "Goodbye!" << endl; + interpreter_.exit_main_loop(); + } + + command_interpreter &interpreter_; + }; + class show_superblock : public command { public: explicit show_superblock(metadata::ptr md) @@ -298,7 +337,7 @@ namespace { //-------------------------------- - int debug(string const &path) { + int debug(string const &path, bool ignore_metadata_sm) { try { block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY, 1); metadata::ptr md(new metadata(bm, false)); @@ -308,16 +347,17 @@ namespace { interp.register_command("m1_node", command::ptr(new show_btree_node(md))); interp.register_command("m2_node", command::ptr(new show_btree_node(md))); interp.register_command("detail_node", command::ptr(new show_btree_node(md))); + interp.register_command("help", command::ptr(new help)); + interp.register_command("exit", command::ptr(new exit_handler(interp))); interp.enter_main_loop(); } catch (std::exception &e) { - cerr << e.what(); + cerr << e.what() << endl; return 1; } return 0; } - } thin_debug_cmd::thin_debug_cmd() @@ -342,8 +382,10 @@ thin_debug_cmd::run(int argc, char **argv) const struct option longopts[] = { { "help", no_argument, NULL, 'h'}, { "version", no_argument, NULL, 'V'}, + { "ignore-metadata-sm", no_argument, NULL, 1}, { NULL, no_argument, NULL, 0 } }; + bool ignore_metadata_sm = false; while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { switch(c) { @@ -354,6 +396,10 @@ thin_debug_cmd::run(int argc, char **argv) case 'V': cerr << THIN_PROVISIONING_TOOLS_VERSION << endl; return 0; + + case 1: + ignore_metadata_sm = true; + break; } } @@ -362,5 +408,5 @@ thin_debug_cmd::run(int argc, char **argv) exit(1); } - return debug(argv[optind]); + return debug(argv[optind], ignore_metadata_sm); } From 3bfa775887d2fb613057d1c69f013f63ab59a28e Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 13 Jan 2021 11:43:14 +0800 Subject: [PATCH 10/33] [thin_debug] Show space map roots in superblock --- thin-provisioning/thin_debug.cc | 41 +++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index 2340479..fa6e457 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -31,6 +31,7 @@ #include "persistent-data/data-structures/btree.h" #include "persistent-data/data-structures/simple_traits.h" #include "persistent-data/file_utils.h" +#include "persistent-data/space-maps/disk_structures.h" #include "thin-provisioning/commands.h" #include "thin-provisioning/metadata.h" #include "thin-provisioning/metadata_checker.h" @@ -61,7 +62,7 @@ namespace { fields_.push_back(field_type(name, t)); } - virtual void output(ostream &out, int depth = 0) = 0; + virtual void output(ostream &out, int depth = 0, boost::optional name = boost::none) = 0; protected: typedef boost::variant value; @@ -80,9 +81,12 @@ namespace { class xml_formatter : public formatter { public: - virtual void output(ostream &out, int depth) { + virtual void output(ostream &out, int depth, boost::optional name = boost::none) { indent(depth, out); - out << "" << endl; + if (name && (*name).length()) + out << "" << endl; + else + out << "" << endl; vector::const_iterator it; for (it = fields_.begin(); it != fields_.end(); ++it) { if (string const *s = get(&it->get<1>())) { @@ -96,7 +100,7 @@ namespace { } else { formatter::ptr f = get(it->get<1>()); - f->output(out, depth + 1); + f->output(out, depth + 1, it->get<0>()); } } @@ -216,6 +220,17 @@ namespace { command_interpreter &interpreter_; }; + class sm_root_show_traits : public persistent_data::sm_disk_detail::sm_root_traits { + public: + static void show(formatter &f, string const &key, + persistent_data::sm_disk_detail::sm_root const &value) { + field(f, "nr blocks", value.nr_blocks_); + field(f, "nr allocated", value.nr_allocated_); + field(f, "bitmap root", value.bitmap_root_); + field(f, "ref count root", value.ref_count_root_); + } + }; + class show_superblock : public command { public: explicit show_superblock(metadata::ptr md) @@ -236,6 +251,24 @@ namespace { field(f, "time", sb.time_); field(f, "trans id", sb.trans_id_); field(f, "metadata snap", sb.metadata_snap_); + + sm_disk_detail::sm_root_disk const *d; + sm_disk_detail::sm_root v; + { + d = reinterpret_cast(sb.metadata_space_map_root_); + sm_disk_detail::sm_root_traits::unpack(*d, v); + formatter::ptr f2(new xml_formatter); + sm_root_show_traits::show(*f2, "value", v); + f.child("metadata space map root", f2); + } + { + d = reinterpret_cast(sb.data_space_map_root_); + sm_disk_detail::sm_root_traits::unpack(*d, v); + formatter::ptr f2(new xml_formatter); + sm_root_show_traits::show(*f2, "value", v); + f.child("data space map root", f2); + } + field(f, "data mapping root", sb.data_mapping_root_); field(f, "device details root", sb.device_details_root_); field(f, "data block size", sb.data_block_size_); From 6dc50866436fb451e542f40b6645f38b37cc2929 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 13 Jan 2021 11:56:18 +0800 Subject: [PATCH 11/33] [thin_debug] Add commands to show space maps --- thin-provisioning/thin_debug.cc | 63 +++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index fa6e457..fc725e8 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -28,6 +28,7 @@ #include #include +#include "base/math_utils.h" #include "persistent-data/data-structures/btree.h" #include "persistent-data/data-structures/simple_traits.h" #include "persistent-data/file_utils.h" @@ -202,6 +203,8 @@ namespace { << " m1_node " << endl << " m2_node " << endl << " detail_node " << endl + << " index_block " << endl + << " index_node " << endl << " exit" << endl; } }; @@ -318,6 +321,18 @@ namespace { } }; + class index_entry_show_traits : public persistent_data::sm_disk_detail::index_entry_traits { + public: + typedef persistent_data::sm_disk_detail::index_entry_traits value_trait; + + static void show(formatter &f, string const &key, + persistent_data::sm_disk_detail::index_entry const &value) { + field(f, "blocknr", value.blocknr_); + field(f, "nr free", value.nr_free_); + field(f, "none free before", value.none_free_before_); + } + }; + template class show_btree_node : public command { public: @@ -368,6 +383,52 @@ namespace { metadata::ptr md_; }; + class show_index_block : public command { + public: + explicit show_index_block(metadata::ptr md) + : md_(md) { + } + + virtual void exec(strings const &args, ostream &out) { + if (args.size() != 2) + throw runtime_error("incorrect number of arguments"); + + // manually load metadata_index, without using index_validator() + block_address block = lexical_cast(args[1]); + block_manager::read_ref rr = md_->tm_->read_lock(block); + + sm_disk_detail::sm_root_disk const *d = + reinterpret_cast(md_->sb_.metadata_space_map_root_); + sm_disk_detail::sm_root v; + sm_disk_detail::sm_root_traits::unpack(*d, v); + block_address nr_indexes = base::div_up(v.nr_blocks_, ENTRIES_PER_BLOCK); + + sm_disk_detail::metadata_index const *mdi = + reinterpret_cast(rr.data()); + show_metadata_index(mdi, nr_indexes, out); + } + + private: + void show_metadata_index(sm_disk_detail::metadata_index const *mdi, block_address nr_indexes, ostream &out) { + xml_formatter f; + field(f, "csum", to_cpu(mdi->csum_)); + field(f, "padding", to_cpu(mdi->padding_)); + field(f, "blocknr", to_cpu(mdi->blocknr_)); + + sm_disk_detail::index_entry ie; + for (block_address i = 0; i < nr_indexes; i++) { + sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), ie); + formatter::ptr f2(new xml_formatter); + index_entry_show_traits::show(*f2, "value", ie); + f.child(lexical_cast(i), f2); + } + f.output(out, 0); + } + + unsigned const ENTRIES_PER_BLOCK = (MD_BLOCK_SIZE - sizeof(persistent_data::sm_disk_detail::bitmap_header)) * 4; + metadata::ptr md_; + }; + //-------------------------------- int debug(string const &path, bool ignore_metadata_sm) { @@ -380,6 +441,8 @@ namespace { interp.register_command("m1_node", command::ptr(new show_btree_node(md))); interp.register_command("m2_node", command::ptr(new show_btree_node(md))); interp.register_command("detail_node", command::ptr(new show_btree_node(md))); + interp.register_command("index_block", command::ptr(new show_index_block(md))); + interp.register_command("index_node", command::ptr(new show_btree_node(md))); interp.register_command("help", command::ptr(new help)); interp.register_command("exit", command::ptr(new exit_handler(interp))); interp.enter_main_loop(); From ab3b2cbda2bc4ce76246427758fd9c59dc6a95f2 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 13 Jan 2021 12:02:20 +0800 Subject: [PATCH 12/33] [thin_debug] Refine the outputs - Show the blocknr from node header - Display the index of node entries --- thin-provisioning/thin_debug.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index fc725e8..96fca15 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -364,7 +364,7 @@ namespace { xml_formatter f; field(f, "csum", n.get_checksum()); - field(f, "blocknr", n.get_location()); + field(f, "blocknr", n.get_block_nr()); field(f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); field(f, "nr entries", n.get_nr_entries()); field(f, "max entries", n.get_max_entries()); @@ -374,7 +374,7 @@ namespace { formatter::ptr f2(new xml_formatter); field(*f2, "key", n.key_at(i)); ST::show(*f2, "value", n.value_at(i)); - f.child("child", f2); + f.child(lexical_cast(i), f2); } f.output(out, 0); From 0816430ba0550effaa89c7d79f2ef3759d70f1c3 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 13 Jan 2021 14:10:49 +0800 Subject: [PATCH 13/33] [thin_debug] Remove the using boost directive to avoid namespace pollution boost::uint32_t might conflict with the toolchain defined uint32_t. which causes template argument deduction failed. --- thin-provisioning/thin_debug.cc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index 96fca15..f3332b8 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -39,7 +39,6 @@ #include "thin-provisioning/superblock.h" #include "version.h" -using namespace boost; using namespace persistent_data; using namespace std; using namespace thin_provisioning; @@ -53,7 +52,7 @@ namespace { virtual ~formatter() {} - typedef optional maybe_string; + typedef boost::optional maybe_string; void field(string const &name, string const &value) { fields_.push_back(field_type(name, value)); @@ -75,7 +74,7 @@ namespace { template void field(formatter &t, string const &name, T const &value) { - t.field(name, lexical_cast(value)); + t.field(name, boost::lexical_cast(value)); } //-------------------------------- @@ -90,7 +89,7 @@ namespace { out << "" << endl; vector::const_iterator it; for (it = fields_.begin(); it != fields_.end(); ++it) { - if (string const *s = get(&it->get<1>())) { + if (string const *s = boost::get(&it->get<1>())) { indent(depth + 1, out); out << "get<0>() @@ -100,7 +99,7 @@ namespace { << endl; } else { - formatter::ptr f = get(it->get<1>()); + formatter::ptr f = boost::get(it->get<1>()); f->output(out, depth + 1, it->get<0>()); } } @@ -306,7 +305,7 @@ namespace { typedef uint64_traits value_trait; static void show(formatter &f, string const &key, uint64_t const &value) { - field(f, key, lexical_cast(value)); + field(f, key, boost::lexical_cast(value)); } }; @@ -346,7 +345,7 @@ namespace { if (args.size() != 2) throw runtime_error("incorrect number of arguments"); - block_address block = lexical_cast(args[1]); + block_address block = boost::lexical_cast(args[1]); block_manager::read_ref rr = md_->tm_->read_lock(block); node_ref n = btree_detail::to_node(rr); @@ -374,7 +373,7 @@ namespace { formatter::ptr f2(new xml_formatter); field(*f2, "key", n.key_at(i)); ST::show(*f2, "value", n.value_at(i)); - f.child(lexical_cast(i), f2); + f.child(boost::lexical_cast(i), f2); } f.output(out, 0); @@ -394,7 +393,7 @@ namespace { throw runtime_error("incorrect number of arguments"); // manually load metadata_index, without using index_validator() - block_address block = lexical_cast(args[1]); + block_address block = boost::lexical_cast(args[1]); block_manager::read_ref rr = md_->tm_->read_lock(block); sm_disk_detail::sm_root_disk const *d = @@ -420,7 +419,7 @@ namespace { sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), ie); formatter::ptr f2(new xml_formatter); index_entry_show_traits::show(*f2, "value", ie); - f.child(lexical_cast(i), f2); + f.child(boost::lexical_cast(i), f2); } f.output(out, 0); } From e046bbf5c49206fe0c04cc0cf3c82d13e21f2b30 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 13 Jan 2021 14:30:37 +0800 Subject: [PATCH 14/33] [thin_debug] Simplify the output format Turn nested fields into attributes. Also replace spaces in field names by underscores. --- thin-provisioning/thin_debug.cc | 81 ++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index f3332b8..ea3541c 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -59,7 +59,7 @@ namespace { } void child(string const &name, formatter::ptr t) { - fields_.push_back(field_type(name, t)); + children_.push_back(field_type(name, t)); } virtual void output(ostream &out, int depth = 0, boost::optional name = boost::none) = 0; @@ -69,6 +69,7 @@ namespace { typedef boost::tuple field_type; vector fields_; + vector children_; }; template @@ -83,22 +84,27 @@ namespace { public: virtual void output(ostream &out, int depth, boost::optional name = boost::none) { indent(depth, out); + out << "" << endl; - else - out << "" << endl; + out << " id=\"" << *name << "\""; + + /* output non-child fields */ vector::const_iterator it; for (it = fields_.begin(); it != fields_.end(); ++it) { if (string const *s = boost::get(&it->get<1>())) { - indent(depth + 1, out); - out << "get<0>() - << "\" value=\"" - << *s - << "\"/>" - << endl; + out << " " << it->get<0>() << "=\"" << *s << "\""; + } + } - } else { + if (children_.size() == 0) { + out << " />" << endl; + return; + } + + /* output child fields */ + out << ">" << endl; + for (it = children_.begin(); it != children_.end(); ++it) { + if (!boost::get(&it->get<1>())) { formatter::ptr f = boost::get(it->get<1>()); f->output(out, depth + 1, it->get<0>()); } @@ -108,6 +114,7 @@ namespace { out << "" << endl; } + private: void indent(int depth, ostream &out) const { for (int i = 0; i < depth * 2; i++) @@ -226,10 +233,10 @@ namespace { public: static void show(formatter &f, string const &key, persistent_data::sm_disk_detail::sm_root const &value) { - field(f, "nr blocks", value.nr_blocks_); - field(f, "nr allocated", value.nr_allocated_); - field(f, "bitmap root", value.bitmap_root_); - field(f, "ref count root", value.ref_count_root_); + field(f, "nr_blocks", value.nr_blocks_); + field(f, "nr_allocated", value.nr_allocated_); + field(f, "bitmap_root", value.bitmap_root_); + field(f, "ref_count_root", value.ref_count_root_); } }; @@ -251,8 +258,8 @@ namespace { field(f, "magic", sb.magic_); field(f, "version", sb.version_); field(f, "time", sb.time_); - field(f, "trans id", sb.trans_id_); - field(f, "metadata snap", sb.metadata_snap_); + field(f, "trans_id", sb.trans_id_); + field(f, "metadata_snap", sb.metadata_snap_); sm_disk_detail::sm_root_disk const *d; sm_disk_detail::sm_root v; @@ -261,24 +268,24 @@ namespace { sm_disk_detail::sm_root_traits::unpack(*d, v); formatter::ptr f2(new xml_formatter); sm_root_show_traits::show(*f2, "value", v); - f.child("metadata space map root", f2); + f.child("metadata_space_map_root", f2); } { d = reinterpret_cast(sb.data_space_map_root_); sm_disk_detail::sm_root_traits::unpack(*d, v); formatter::ptr f2(new xml_formatter); sm_root_show_traits::show(*f2, "value", v); - f.child("data space map root", f2); + f.child("data_space_map_root", f2); } - field(f, "data mapping root", sb.data_mapping_root_); - field(f, "device details root", sb.device_details_root_); - field(f, "data block size", sb.data_block_size_); - field(f, "metadata block size", sb.metadata_block_size_); - field(f, "metadata nr blocks", sb.metadata_nr_blocks_); - field(f, "compat flags", sb.compat_flags_); - field(f, "compat ro flags", sb.compat_ro_flags_); - field(f, "incompat flags", sb.incompat_flags_); + field(f, "data_mapping_root", sb.data_mapping_root_); + field(f, "device_details_root", sb.device_details_root_); + field(f, "data_block_size", sb.data_block_size_); + field(f, "metadata_block_size", sb.metadata_block_size_); + field(f, "metadata_nr_blocks", sb.metadata_nr_blocks_); + field(f, "compat_flags", sb.compat_flags_); + field(f, "compat_ro_flags", sb.compat_ro_flags_); + field(f, "incompat_flags", sb.incompat_flags_); f.output(out, 0); } @@ -293,10 +300,10 @@ namespace { static void show(formatter &f, string const &key, thin_provisioning::device_tree_detail::device_details const &value) { - field(f, "mapped blocks", value.mapped_blocks_); - field(f, "transaction id", value.transaction_id_); - field(f, "creation time", value.creation_time_); - field(f, "snap time", value.snapshotted_time_); + field(f, "mapped_blocks", value.mapped_blocks_); + field(f, "transaction_id", value.transaction_id_); + field(f, "creation_time", value.creation_time_); + field(f, "snap_time", value.snapshotted_time_); } }; @@ -327,8 +334,8 @@ namespace { static void show(formatter &f, string const &key, persistent_data::sm_disk_detail::index_entry const &value) { field(f, "blocknr", value.blocknr_); - field(f, "nr free", value.nr_free_); - field(f, "none free before", value.none_free_before_); + field(f, "nr_free", value.nr_free_); + field(f, "none_free_before", value.none_free_before_); } }; @@ -365,9 +372,9 @@ namespace { field(f, "csum", n.get_checksum()); field(f, "blocknr", n.get_block_nr()); field(f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); - field(f, "nr entries", n.get_nr_entries()); - field(f, "max entries", n.get_max_entries()); - field(f, "value size", n.get_value_size()); + field(f, "nr_entries", n.get_nr_entries()); + field(f, "max_entries", n.get_max_entries()); + field(f, "value_size", n.get_value_size()); for (unsigned i = 0; i < n.get_nr_entries(); i++) { formatter::ptr f2(new xml_formatter); From b9b04dc872fadcf89eaee6211c698188bf8d30ba Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 21 Jan 2021 16:05:28 +0800 Subject: [PATCH 15/33] [thin_debug] Factor out reusable componments --- Makefile.in | 2 + base/command_interpreter.cc | 44 ++++++++ base/command_interpreter.h | 60 +++++++++++ base/output_formatter.cc | 53 ++++++++++ base/output_formatter.h | 59 +++++++++++ thin-provisioning/thin_debug.cc | 181 +++----------------------------- 6 files changed, 231 insertions(+), 168 deletions(-) create mode 100644 base/command_interpreter.cc create mode 100644 base/command_interpreter.h create mode 100644 base/output_formatter.cc create mode 100644 base/output_formatter.h diff --git a/Makefile.in b/Makefile.in index c70159d..ae7ebcd 100644 --- a/Makefile.in +++ b/Makefile.in @@ -126,6 +126,8 @@ TOOLS_SOURCE=\ thin-provisioning/thin_trim.cc DEVTOOLS_SOURCE=\ + base/command_interpreter.cc \ + base/output_formatter.cc \ caching/devel_commands.cc \ era/devel_commands.cc \ thin-provisioning/damage_generator.cc \ diff --git a/base/command_interpreter.cc b/base/command_interpreter.cc new file mode 100644 index 0000000..e2e4e82 --- /dev/null +++ b/base/command_interpreter.cc @@ -0,0 +1,44 @@ +#include "base/command_interpreter.h" + +using namespace dbg; +using namespace std; + +//---------------------------------------------------------------- + +namespace { + strings read_input(std::istream &in) { + using namespace boost::algorithm; + + std::string input; + getline(in, input); + + strings toks; + split(toks, input, is_any_of(" \t"), token_compress_on); + + return toks; + } +} + +//---------------------------------------------------------------- + +void command_interpreter::do_once() { + if (in_.eof()) + throw runtime_error("input closed"); + + out_ << "> "; + strings args = read_input(in_); + + std::map::iterator it; + it = commands_.find(args[0]); + if (it == commands_.end()) + out_ << "Unrecognised command" << endl; + else { + try { + it->second->exec(args, out_); + } catch (std::exception &e) { + cerr << e.what() << endl; + } + } +} + +//---------------------------------------------------------------- diff --git a/base/command_interpreter.h b/base/command_interpreter.h new file mode 100644 index 0000000..0b4abca --- /dev/null +++ b/base/command_interpreter.h @@ -0,0 +1,60 @@ +#ifndef DBG_COMMAND_INTERPRETER +#define DBG_COMMAND_INTERPRETER + +#include +#include + +#include +#include +#include +#include + +//---------------------------------------------------------------- + +namespace dbg { + typedef std::vector strings; + + class command { + public: + typedef std::shared_ptr ptr; + + virtual ~command() {} + virtual void exec(strings const &args, std::ostream &out) = 0; + }; + + class command_interpreter { + public: + typedef std::shared_ptr ptr; + + command_interpreter(std::istream &in, std::ostream &out) + : in_(in), + out_(out), + exit_(false) { + } + + void register_command(std::string const &str, command::ptr cmd) { + commands_.insert(make_pair(str, cmd)); + } + + void enter_main_loop() { + while (!exit_) + do_once(); + } + + void exit_main_loop() { + exit_ = true; + } + + private: + void do_once(); + + std::istream &in_; + std::ostream &out_; + std::map commands_; + bool exit_; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/base/output_formatter.cc b/base/output_formatter.cc new file mode 100644 index 0000000..54ef578 --- /dev/null +++ b/base/output_formatter.cc @@ -0,0 +1,53 @@ +#include + +#include "base/output_formatter.h" + +using namespace dbg; +using namespace std; + +//---------------------------------------------------------------- + +namespace { + void indent(int depth, std::ostream &out) { + for (int i = 0; i < depth * 2; i++) + out << ' '; + } +} + +//---------------------------------------------------------------- + +void xml_formatter::output(std::ostream &out, + int depth, + boost::optional name) { + indent(depth, out); + out << "::const_iterator it; + for (it = fields_.begin(); it != fields_.end(); ++it) { + if (string const *s = boost::get(&it->get<1>())) { + out << " " << it->get<0>() << "=\"" << *s << "\""; + } + } + + if (children_.size() == 0) { + out << " />" << endl; + return; + } + + /* output child fields */ + out << ">" << endl; + for (it = children_.begin(); it != children_.end(); ++it) { + if (!boost::get(&it->get<1>())) { + formatter::ptr f = boost::get(it->get<1>()); + f->output(out, depth + 1, it->get<0>()); + } + } + + indent(depth, out); + out << "" << endl; +} + +//---------------------------------------------------------------- diff --git a/base/output_formatter.h b/base/output_formatter.h new file mode 100644 index 0000000..2299f9d --- /dev/null +++ b/base/output_formatter.h @@ -0,0 +1,59 @@ +#ifndef DBG_OUTPUT_FORMATTER_H +#define DBG_OUTPUT_FORMATTER_H + +#include +#include +#include +#include + +#include +#include + +//---------------------------------------------------------------- + +namespace dbg { + class formatter { + public: + typedef std::shared_ptr ptr; + + virtual ~formatter() {} + + typedef boost::optional maybe_string; + + void field(std::string const &name, std::string const &value) { + fields_.push_back(field_type(name, value)); + } + + void child(std::string const &name, formatter::ptr t) { + children_.push_back(field_type(name, t)); + } + + virtual void output(std::ostream &out, int depth = 0, + boost::optional name = boost::none) = 0; + + protected: + typedef boost::variant value; + typedef boost::tuple field_type; + + std::vector fields_; + std::vector children_; + }; + + template + void + field(formatter &t, std::string const &name, T const &value) { + t.field(name, boost::lexical_cast(value)); + } + + //-------------------------------- + + class xml_formatter : public formatter { + public: + virtual void output(std::ostream &out, int depth = 0, + boost::optional name = boost::none); + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index ea3541c..a48b79f 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -16,193 +16,36 @@ // with thin-provisioning-tools. If not, see // . -#include -#include #include -#include -#include #include #include -#include -#include #include -#include +#include "base/command_interpreter.h" #include "base/math_utils.h" +#include "base/output_formatter.h" #include "persistent-data/data-structures/btree.h" #include "persistent-data/data-structures/simple_traits.h" #include "persistent-data/file_utils.h" #include "persistent-data/space-maps/disk_structures.h" #include "thin-provisioning/commands.h" #include "thin-provisioning/metadata.h" -#include "thin-provisioning/metadata_checker.h" #include "thin-provisioning/superblock.h" #include "version.h" +using namespace dbg; using namespace persistent_data; using namespace std; using namespace thin_provisioning; namespace { - typedef vector strings; - - class formatter { - public: - typedef std::shared_ptr ptr; - - virtual ~formatter() {} - - typedef boost::optional maybe_string; - - void field(string const &name, string const &value) { - fields_.push_back(field_type(name, value)); - } - - void child(string const &name, formatter::ptr t) { - children_.push_back(field_type(name, t)); - } - - virtual void output(ostream &out, int depth = 0, boost::optional name = boost::none) = 0; - - protected: - typedef boost::variant value; - typedef boost::tuple field_type; - - vector fields_; - vector children_; - }; - - template - void - field(formatter &t, string const &name, T const &value) { - t.field(name, boost::lexical_cast(value)); - } - - //-------------------------------- - - class xml_formatter : public formatter { - public: - virtual void output(ostream &out, int depth, boost::optional name = boost::none) { - indent(depth, out); - out << "::const_iterator it; - for (it = fields_.begin(); it != fields_.end(); ++it) { - if (string const *s = boost::get(&it->get<1>())) { - out << " " << it->get<0>() << "=\"" << *s << "\""; - } - } - - if (children_.size() == 0) { - out << " />" << endl; - return; - } - - /* output child fields */ - out << ">" << endl; - for (it = children_.begin(); it != children_.end(); ++it) { - if (!boost::get(&it->get<1>())) { - formatter::ptr f = boost::get(it->get<1>()); - f->output(out, depth + 1, it->get<0>()); - } - } - - indent(depth, out); - out << "" << endl; - } - - - private: - void indent(int depth, ostream &out) const { - for (int i = 0; i < depth * 2; i++) - out << ' '; - } - }; - - //-------------------------------- - - class command { - public: - typedef std::shared_ptr ptr; - - virtual ~command() {} - virtual void exec(strings const &args, ostream &out) = 0; - }; - - class command_interpreter { - public: - typedef std::shared_ptr ptr; - - command_interpreter(istream &in, ostream &out) - : in_(in), - out_(out), - exit_(false) { - } - - void register_command(string const &str, command::ptr cmd) { - commands_.insert(make_pair(str, cmd)); - } - - void enter_main_loop() { - while (!exit_) - do_once(); - } - - void exit_main_loop() { - exit_ = true; - } - - private: - strings read_input() { - using namespace boost::algorithm; - - string input; - getline(in_, input); - - strings toks; - split(toks, input, is_any_of(" \t"), token_compress_on); - - return toks; - } - - void do_once() { - if (in_.eof()) - throw runtime_error("input closed"); - - out_ << "> "; - strings args = read_input(); - - map::iterator it; - it = commands_.find(args[0]); - if (it == commands_.end()) - out_ << "Unrecognised command" << endl; - else { - try { - it->second->exec(args, out_); - } catch (std::exception &e) { - cerr << e.what() << endl; - } - } - } - - istream &in_; - ostream &out_; - map commands_; - bool exit_; - }; - - //-------------------------------- - - class hello : public command { + class hello : public dbg::command { virtual void exec(strings const &args, ostream &out) { out << "Hello, world!" << endl; } }; - class help : public command { + class help : public dbg::command { virtual void exec(strings const &args, ostream &out) { out << "Commands:" << endl << " superblock" << endl @@ -215,7 +58,7 @@ namespace { } }; - class exit_handler : public command { + class exit_handler : public dbg::command { public: exit_handler(command_interpreter &interpreter) : interpreter_(interpreter) { @@ -240,7 +83,7 @@ namespace { } }; - class show_superblock : public command { + class show_superblock : public dbg::command { public: explicit show_superblock(metadata::ptr md) : md_(md) { @@ -340,7 +183,7 @@ namespace { }; template - class show_btree_node : public command { + class show_btree_node : public dbg::command { public: explicit show_btree_node(metadata::ptr md) : md_(md) { @@ -389,7 +232,7 @@ namespace { metadata::ptr md_; }; - class show_index_block : public command { + class show_index_block : public dbg::command { public: explicit show_index_block(metadata::ptr md) : md_(md) { @@ -437,7 +280,9 @@ namespace { //-------------------------------- - int debug(string const &path, bool ignore_metadata_sm) { + int debug_(string const &path, bool ignore_metadata_sm) { + using dbg::command; + try { block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY, 1); metadata::ptr md(new metadata(bm, false)); @@ -510,5 +355,5 @@ thin_debug_cmd::run(int argc, char **argv) exit(1); } - return debug(argv[optind], ignore_metadata_sm); + return debug_(argv[optind], ignore_metadata_sm); } From a05ac553b6581fd1889f750b1e4f2c0f777174b7 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 3 Feb 2021 17:11:24 +0800 Subject: [PATCH 16/33] [dbg] Hide implementations of shared components --- base/command_interpreter.cc | 48 +++++++++- base/command_interpreter.h | 32 ++----- base/output_formatter.cc | 33 +++++++ base/output_formatter.h | 28 +----- thin-provisioning/thin_debug.cc | 154 ++++++++++++++++---------------- 5 files changed, 165 insertions(+), 130 deletions(-) diff --git a/base/command_interpreter.cc b/base/command_interpreter.cc index e2e4e82..ce7bd97 100644 --- a/base/command_interpreter.cc +++ b/base/command_interpreter.cc @@ -6,7 +6,42 @@ using namespace std; //---------------------------------------------------------------- namespace { - strings read_input(std::istream &in) { + class command_interpreter_impl : public command_interpreter { + public: + typedef std::shared_ptr ptr; + + command_interpreter_impl(std::istream &in, std::ostream &out) + : in_(in), + out_(out), + exit_(false) { + } + + void register_command(std::string const &str, command::ptr cmd) { + commands_.insert(make_pair(str, cmd)); + } + + void enter_main_loop() { + while (!exit_) + do_once(); + } + + void exit_main_loop() { + exit_ = true; + } + + private: + void do_once(); + + std::istream &in_; + std::ostream &out_; + std::map commands_; + bool exit_; + }; + + //-------------------------------- + + strings read_input(std::istream &in) + { using namespace boost::algorithm; std::string input; @@ -21,7 +56,8 @@ namespace { //---------------------------------------------------------------- -void command_interpreter::do_once() { +void command_interpreter_impl::do_once() +{ if (in_.eof()) throw runtime_error("input closed"); @@ -42,3 +78,11 @@ void command_interpreter::do_once() { } //---------------------------------------------------------------- + +command_interpreter::ptr +dbg::create_command_interpreter(std::istream &in, std::ostream &out) +{ + return command_interpreter::ptr(new command_interpreter_impl(in, out)); +} + +//---------------------------------------------------------------- diff --git a/base/command_interpreter.h b/base/command_interpreter.h index 0b4abca..b0646d9 100644 --- a/base/command_interpreter.h +++ b/base/command_interpreter.h @@ -26,33 +26,13 @@ namespace dbg { public: typedef std::shared_ptr ptr; - command_interpreter(std::istream &in, std::ostream &out) - : in_(in), - out_(out), - exit_(false) { - } - - void register_command(std::string const &str, command::ptr cmd) { - commands_.insert(make_pair(str, cmd)); - } - - void enter_main_loop() { - while (!exit_) - do_once(); - } - - void exit_main_loop() { - exit_ = true; - } - - private: - void do_once(); - - std::istream &in_; - std::ostream &out_; - std::map commands_; - bool exit_; + virtual void register_command(std::string const &str, command::ptr cmd) = 0; + virtual void enter_main_loop() = 0; + virtual void exit_main_loop() = 0; }; + + command_interpreter::ptr + create_command_interpreter(std::istream &in, std::ostream &out); } //---------------------------------------------------------------- diff --git a/base/output_formatter.cc b/base/output_formatter.cc index 54ef578..c62058f 100644 --- a/base/output_formatter.cc +++ b/base/output_formatter.cc @@ -8,6 +8,31 @@ using namespace std; //---------------------------------------------------------------- namespace { + class abstract_formatter : public formatter { + typedef boost::optional maybe_string; + + void field(std::string const &name, std::string const &value) { + fields_.push_back(field_type(name, value)); + } + + void child(std::string const &name, formatter::ptr t) { + children_.push_back(field_type(name, t)); + } + + protected: + typedef boost::variant value; + typedef boost::tuple field_type; + + std::vector fields_; + std::vector children_; + }; + + class xml_formatter : public abstract_formatter { + public: + void output(std::ostream &out, int depth = 0, + boost::optional name = boost::none); + }; + void indent(int depth, std::ostream &out) { for (int i = 0; i < depth * 2; i++) out << ' '; @@ -51,3 +76,11 @@ void xml_formatter::output(std::ostream &out, } //---------------------------------------------------------------- + +formatter::ptr +dbg::create_xml_formatter() +{ + return formatter::ptr(new xml_formatter()); +} + +//---------------------------------------------------------------- diff --git a/base/output_formatter.h b/base/output_formatter.h index 2299f9d..2cc4bc5 100644 --- a/base/output_formatter.h +++ b/base/output_formatter.h @@ -17,26 +17,10 @@ namespace dbg { typedef std::shared_ptr ptr; virtual ~formatter() {} - - typedef boost::optional maybe_string; - - void field(std::string const &name, std::string const &value) { - fields_.push_back(field_type(name, value)); - } - - void child(std::string const &name, formatter::ptr t) { - children_.push_back(field_type(name, t)); - } - + virtual void field(std::string const &name, std::string const &value) = 0; + virtual void child(std::string const &name, formatter::ptr t) = 0; virtual void output(std::ostream &out, int depth = 0, boost::optional name = boost::none) = 0; - - protected: - typedef boost::variant value; - typedef boost::tuple field_type; - - std::vector fields_; - std::vector children_; }; template @@ -45,13 +29,7 @@ namespace dbg { t.field(name, boost::lexical_cast(value)); } - //-------------------------------- - - class xml_formatter : public formatter { - public: - virtual void output(std::ostream &out, int depth = 0, - boost::optional name = boost::none); - }; + formatter::ptr create_xml_formatter(); } //---------------------------------------------------------------- diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index a48b79f..e458d7d 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -60,26 +60,26 @@ namespace { class exit_handler : public dbg::command { public: - exit_handler(command_interpreter &interpreter) + exit_handler(command_interpreter::ptr interpreter) : interpreter_(interpreter) { } virtual void exec(strings const &args, ostream &out) { out << "Goodbye!" << endl; - interpreter_.exit_main_loop(); + interpreter_->exit_main_loop(); } - command_interpreter &interpreter_; + command_interpreter::ptr interpreter_; }; class sm_root_show_traits : public persistent_data::sm_disk_detail::sm_root_traits { public: - static void show(formatter &f, string const &key, + static void show(formatter::ptr f, string const &key, persistent_data::sm_disk_detail::sm_root const &value) { - field(f, "nr_blocks", value.nr_blocks_); - field(f, "nr_allocated", value.nr_allocated_); - field(f, "bitmap_root", value.bitmap_root_); - field(f, "ref_count_root", value.ref_count_root_); + field(*f, "nr_blocks", value.nr_blocks_); + field(*f, "nr_allocated", value.nr_allocated_); + field(*f, "bitmap_root", value.bitmap_root_); + field(*f, "ref_count_root", value.ref_count_root_); } }; @@ -90,47 +90,47 @@ namespace { } virtual void exec(strings const &args, ostream &out) { - xml_formatter f; + formatter::ptr f = create_xml_formatter(); thin_provisioning::superblock_detail::superblock const &sb = md_->sb_; - field(f, "csum", sb.csum_); - field(f, "flags", sb.flags_); - field(f, "blocknr", sb.blocknr_); - field(f, "uuid", sb.uuid_); // FIXME: delimit, and handle non-printable chars - field(f, "magic", sb.magic_); - field(f, "version", sb.version_); - field(f, "time", sb.time_); - field(f, "trans_id", sb.trans_id_); - field(f, "metadata_snap", sb.metadata_snap_); + field(*f, "csum", sb.csum_); + field(*f, "flags", sb.flags_); + field(*f, "blocknr", sb.blocknr_); + field(*f, "uuid", sb.uuid_); // FIXME: delimit, and handle non-printable chars + field(*f, "magic", sb.magic_); + field(*f, "version", sb.version_); + field(*f, "time", sb.time_); + field(*f, "trans_id", sb.trans_id_); + field(*f, "metadata_snap", sb.metadata_snap_); sm_disk_detail::sm_root_disk const *d; sm_disk_detail::sm_root v; { d = reinterpret_cast(sb.metadata_space_map_root_); sm_disk_detail::sm_root_traits::unpack(*d, v); - formatter::ptr f2(new xml_formatter); - sm_root_show_traits::show(*f2, "value", v); - f.child("metadata_space_map_root", f2); + formatter::ptr f2 = create_xml_formatter(); + sm_root_show_traits::show(f2, "value", v); + f->child("metadata_space_map_root", f2); } { d = reinterpret_cast(sb.data_space_map_root_); sm_disk_detail::sm_root_traits::unpack(*d, v); - formatter::ptr f2(new xml_formatter); - sm_root_show_traits::show(*f2, "value", v); - f.child("data_space_map_root", f2); + formatter::ptr f2 = create_xml_formatter(); + sm_root_show_traits::show(f2, "value", v); + f->child("data_space_map_root", f2); } - field(f, "data_mapping_root", sb.data_mapping_root_); - field(f, "device_details_root", sb.device_details_root_); - field(f, "data_block_size", sb.data_block_size_); - field(f, "metadata_block_size", sb.metadata_block_size_); - field(f, "metadata_nr_blocks", sb.metadata_nr_blocks_); - field(f, "compat_flags", sb.compat_flags_); - field(f, "compat_ro_flags", sb.compat_ro_flags_); - field(f, "incompat_flags", sb.incompat_flags_); + field(*f, "data_mapping_root", sb.data_mapping_root_); + field(*f, "device_details_root", sb.device_details_root_); + field(*f, "data_block_size", sb.data_block_size_); + field(*f, "metadata_block_size", sb.metadata_block_size_); + field(*f, "metadata_nr_blocks", sb.metadata_nr_blocks_); + field(*f, "compat_flags", sb.compat_flags_); + field(*f, "compat_ro_flags", sb.compat_ro_flags_); + field(*f, "incompat_flags", sb.incompat_flags_); - f.output(out, 0); + f->output(out, 0); } private: @@ -141,12 +141,12 @@ namespace { public: typedef thin_provisioning::device_tree_detail::device_details_traits value_trait; - static void show(formatter &f, string const &key, + static void show(formatter::ptr f, string const &key, thin_provisioning::device_tree_detail::device_details const &value) { - field(f, "mapped_blocks", value.mapped_blocks_); - field(f, "transaction_id", value.transaction_id_); - field(f, "creation_time", value.creation_time_); - field(f, "snap_time", value.snapshotted_time_); + field(*f, "mapped_blocks", value.mapped_blocks_); + field(*f, "transaction_id", value.transaction_id_); + field(*f, "creation_time", value.creation_time_); + field(*f, "snap_time", value.snapshotted_time_); } }; @@ -154,8 +154,8 @@ namespace { public: typedef uint64_traits value_trait; - static void show(formatter &f, string const &key, uint64_t const &value) { - field(f, key, boost::lexical_cast(value)); + static void show(formatter::ptr f, string const &key, uint64_t const &value) { + field(*f, key, boost::lexical_cast(value)); } }; @@ -163,10 +163,10 @@ namespace { public: typedef thin_provisioning::mapping_tree_detail::block_traits value_trait; - static void show(formatter &f, string const &key, + static void show(formatter::ptr f, string const &key, thin_provisioning::mapping_tree_detail::block_time const &value) { - field(f, "block", value.block_); - field(f, "time", value.time_); + field(*f, "block", value.block_); + field(*f, "time", value.time_); } }; @@ -174,11 +174,11 @@ namespace { public: typedef persistent_data::sm_disk_detail::index_entry_traits value_trait; - static void show(formatter &f, string const &key, + static void show(formatter::ptr f, string const &key, persistent_data::sm_disk_detail::index_entry const &value) { - field(f, "blocknr", value.blocknr_); - field(f, "nr_free", value.nr_free_); - field(f, "none_free_before", value.none_free_before_); + field(*f, "blocknr", value.blocknr_); + field(*f, "nr_free", value.nr_free_); + field(*f, "none_free_before", value.none_free_before_); } }; @@ -210,23 +210,23 @@ namespace { private: template void show_node(node_ref n, ostream &out) { - xml_formatter f; + formatter::ptr f = create_xml_formatter(); - field(f, "csum", n.get_checksum()); - field(f, "blocknr", n.get_block_nr()); - field(f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); - field(f, "nr_entries", n.get_nr_entries()); - field(f, "max_entries", n.get_max_entries()); - field(f, "value_size", n.get_value_size()); + field(*f, "csum", n.get_checksum()); + field(*f, "blocknr", n.get_block_nr()); + field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); + field(*f, "nr_entries", n.get_nr_entries()); + field(*f, "max_entries", n.get_max_entries()); + field(*f, "value_size", n.get_value_size()); for (unsigned i = 0; i < n.get_nr_entries(); i++) { - formatter::ptr f2(new xml_formatter); + formatter::ptr f2 = create_xml_formatter(); field(*f2, "key", n.key_at(i)); - ST::show(*f2, "value", n.value_at(i)); - f.child(boost::lexical_cast(i), f2); + ST::show(f2, "value", n.value_at(i)); + f->child(boost::lexical_cast(i), f2); } - f.output(out, 0); + f->output(out, 0); } metadata::ptr md_; @@ -259,19 +259,19 @@ namespace { private: void show_metadata_index(sm_disk_detail::metadata_index const *mdi, block_address nr_indexes, ostream &out) { - xml_formatter f; - field(f, "csum", to_cpu(mdi->csum_)); - field(f, "padding", to_cpu(mdi->padding_)); - field(f, "blocknr", to_cpu(mdi->blocknr_)); + formatter::ptr f = create_xml_formatter(); + field(*f, "csum", to_cpu(mdi->csum_)); + field(*f, "padding", to_cpu(mdi->padding_)); + field(*f, "blocknr", to_cpu(mdi->blocknr_)); sm_disk_detail::index_entry ie; for (block_address i = 0; i < nr_indexes; i++) { sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), ie); - formatter::ptr f2(new xml_formatter); - index_entry_show_traits::show(*f2, "value", ie); - f.child(boost::lexical_cast(i), f2); + formatter::ptr f2 = create_xml_formatter(); + index_entry_show_traits::show(f2, "value", ie); + f->child(boost::lexical_cast(i), f2); } - f.output(out, 0); + f->output(out, 0); } unsigned const ENTRIES_PER_BLOCK = (MD_BLOCK_SIZE - sizeof(persistent_data::sm_disk_detail::bitmap_header)) * 4; @@ -286,17 +286,17 @@ namespace { try { block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY, 1); metadata::ptr md(new metadata(bm, false)); - command_interpreter interp(cin, cout); - interp.register_command("hello", command::ptr(new hello)); - interp.register_command("superblock", command::ptr(new show_superblock(md))); - interp.register_command("m1_node", command::ptr(new show_btree_node(md))); - interp.register_command("m2_node", command::ptr(new show_btree_node(md))); - interp.register_command("detail_node", command::ptr(new show_btree_node(md))); - interp.register_command("index_block", command::ptr(new show_index_block(md))); - interp.register_command("index_node", command::ptr(new show_btree_node(md))); - interp.register_command("help", command::ptr(new help)); - interp.register_command("exit", command::ptr(new exit_handler(interp))); - interp.enter_main_loop(); + command_interpreter::ptr interp = create_command_interpreter(cin, cout); + interp->register_command("hello", command::ptr(new hello)); + interp->register_command("superblock", command::ptr(new show_superblock(md))); + interp->register_command("m1_node", command::ptr(new show_btree_node(md))); + interp->register_command("m2_node", command::ptr(new show_btree_node(md))); + interp->register_command("detail_node", command::ptr(new show_btree_node(md))); + interp->register_command("index_block", command::ptr(new show_index_block(md))); + interp->register_command("index_node", command::ptr(new show_btree_node(md))); + interp->register_command("help", command::ptr(new help)); + interp->register_command("exit", command::ptr(new exit_handler(interp))); + interp->enter_main_loop(); } catch (std::exception &e) { cerr << e.what() << endl; From 08bd2f47bdba9f40b8115afb5250bd46743922c2 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 3 Dec 2020 20:50:01 +0800 Subject: [PATCH 17/33] [cache_debug] Fork thin_debug into cache_debug --- Makefile.in | 1 + caching/cache_debug.cc | 331 ++++++++++++++++++++++++++++++++++++++ caching/commands.h | 12 ++ caching/devel_commands.cc | 1 + 4 files changed, 345 insertions(+) create mode 100644 caching/cache_debug.cc diff --git a/Makefile.in b/Makefile.in index ae7ebcd..949e30a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -128,6 +128,7 @@ TOOLS_SOURCE=\ DEVTOOLS_SOURCE=\ base/command_interpreter.cc \ base/output_formatter.cc \ + caching/cache_debug.cc \ caching/devel_commands.cc \ era/devel_commands.cc \ thin-provisioning/damage_generator.cc \ diff --git a/caching/cache_debug.cc b/caching/cache_debug.cc new file mode 100644 index 0000000..1053560 --- /dev/null +++ b/caching/cache_debug.cc @@ -0,0 +1,331 @@ +// Copyright (C) 2012 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include +#include +#include +#include + +#include "base/command_interpreter.h" +#include "base/output_formatter.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/simple_traits.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "caching/commands.h" +#include "caching/metadata.h" +#include "version.h" + +using namespace dbg; +using namespace persistent_data; +using namespace std; +using namespace caching; + +//---------------------------------------------------------------- + +namespace { + class hello : public dbg::command { + virtual void exec(strings const &args, ostream &out) { + out << "Hello, world!" << endl; + } + }; + + class help : public dbg::command { + virtual void exec(strings const &args, ostream &out) { + out << "Commands:" << endl + << " superblock" << endl + << " mapping_node " << endl + << " mapping_array " << endl + << " exit" << endl; + } + }; + + class exit_handler : public dbg::command { + public: + exit_handler(command_interpreter::ptr interpreter) + : interpreter_(interpreter) { + } + + virtual void exec(strings const &args, ostream &out) { + out << "Goodbye!" << endl; + interpreter_->exit_main_loop(); + } + + command_interpreter::ptr interpreter_; + }; + + class sm_root_show_traits : public persistent_data::sm_disk_detail::sm_root_traits { + public: + static void show(formatter::ptr f, string const &key, + persistent_data::sm_disk_detail::sm_root const &value) { + field(*f, "nr_blocks", value.nr_blocks_); + field(*f, "nr_allocated", value.nr_allocated_); + field(*f, "bitmap_root", value.bitmap_root_); + field(*f, "ref_count_root", value.ref_count_root_); + } + }; + + class show_superblock : public dbg::command { + public: + explicit show_superblock(metadata::ptr md) + : md_(md) { + } + + virtual void exec(strings const &args, ostream &out) { + formatter::ptr f = create_xml_formatter(); + ostringstream version; + + superblock const &sb = md_->sb_; + + field(*f, "csum", sb.csum); + field(*f, "flags", sb.flags.encode()); + field(*f, "blocknr", sb.blocknr); + field(*f, "uuid", sb.uuid); // FIXME: delimit, and handle non-printable chars + field(*f, "magic", sb.magic); + field(*f, "version", sb.version); + field(*f, "policy_name", reinterpret_cast(sb.policy_name)); + version << sb.policy_version[0] << "." + << sb.policy_version[1] << "." + << sb.policy_version[2]; + field(*f, "policy_version", version.str().c_str()); + field(*f, "policy_hint_size", sb.policy_hint_size); + + sm_disk_detail::sm_root_disk const *d; + sm_disk_detail::sm_root v; + { + d = reinterpret_cast(sb.metadata_space_map_root); + sm_disk_detail::sm_root_traits::unpack(*d, v); + formatter::ptr f2 = create_xml_formatter(); + sm_root_show_traits::show(f2, "value", v); + f->child("metadata_space_map_root", f2); + } + + field(*f, "mapping_root", sb.mapping_root); + if (sb.version >= 2) + field(*f, "dirty_root", *sb.dirty_root); + field(*f, "hint_root", sb.hint_root); + field(*f, "discard_root", sb.discard_root); + field(*f, "discard_block_size", sb.discard_block_size); + field(*f, "discard_nr_blocks", sb.discard_nr_blocks); + field(*f, "data_block_size", sb.data_block_size); + field(*f, "metadata_block_size", sb.metadata_block_size); + field(*f, "cache_blocks", sb.cache_blocks); + field(*f, "compat_flags", sb.compat_flags); + field(*f, "compat_ro_flags", sb.compat_ro_flags); + field(*f, "incompat_flags", sb.incompat_flags); + field(*f, "read_hits", sb.read_hits); + field(*f, "read_misses", sb.read_misses); + field(*f, "write_hits", sb.write_hits); + field(*f, "write_misses", sb.write_misses); + + f->output(out, 0); + } + + private: + metadata::ptr md_; + }; + + class uint64_show_traits : public uint64_traits { + public: + static void show(formatter::ptr f, string const &key, uint64_t const &value) { + field(*f, key, boost::lexical_cast(value)); + } + }; + + // FIXME: Expose the array ValueTraits parameter or not? + // Since the block_traits isn't related to the ValueTraits. + class block_show_traits : public persistent_data::array::block_traits { + public: + static void show(formatter::ptr f, string const &key, block_address const &value) { + field(*f, "block", value); + } + }; + + class mapping_show_traits : public caching::mapping_traits { + public: + static void show(formatter::ptr f, string const &key, caching::mapping const &value) { + field(*f, "oblock", value.oblock_); + field(*f, "flags", value.flags_); + } + }; + + template + class show_btree_node : public dbg::command { + public: + explicit show_btree_node(metadata::ptr md) + : md_(md) { + } + + virtual void exec(strings const &args, ostream &out) { + using namespace persistent_data::btree_detail; + + if (args.size() != 2) + throw runtime_error("incorrect number of arguments"); + + block_address block = boost::lexical_cast(args[1]); + block_manager::read_ref rr = md_->tm_->read_lock(block); + + node_ref n = btree_detail::to_node(rr); + if (n.get_type() == INTERNAL) + show_node(n, out); + else { + node_ref n = btree_detail::to_node(rr); + show_node(n, out); + } + } + + private: + template + void show_node(node_ref n, ostream &out) { + formatter::ptr f = create_xml_formatter(); + + field(*f, "csum", n.get_checksum()); + field(*f, "blocknr", n.get_location()); + field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); + field(*f, "nr_entries", n.get_nr_entries()); + field(*f, "max_entries", n.get_max_entries()); + field(*f, "value_size", n.get_value_size()); + + for (unsigned i = 0; i < n.get_nr_entries(); i++) { + formatter::ptr f2 = create_xml_formatter(); + field(*f2, "key", n.key_at(i)); + VT::show(f2, "value", n.value_at(i)); + f->child("child", f2); + } + + f->output(out, 0); + } + + metadata::ptr md_; + }; + + template + class show_array_block : public dbg::command { + typedef array_block rblock; + public: + explicit show_array_block(metadata::ptr md) + : md_(md) { + } + + virtual void exec(strings const& args, ostream &out) { + if (args.size() != 2) + throw runtime_error("incorrect number of arguments"); + + block_address block = boost::lexical_cast(args[1]); + block_manager::read_ref rr = md_->tm_->read_lock(block); + + rblock b(rr, typename ValueTraits::ref_counter()); + show_array_entries(b, out); + } + + private: + void show_array_entries(rblock const& b, ostream &out) { + formatter::ptr f = create_xml_formatter(); + uint32_t nr_entries = b.nr_entries(); + + field(*f, "max_entries", b.max_entries()); + field(*f, "nr_entries", nr_entries); + field(*f, "value_size", b.value_size()); + + for (unsigned i = 0; i < nr_entries; i++) { + formatter::ptr f2 = create_xml_formatter(); + field(*f2, "index", i); + ValueTraits::show(f2, "value", b.get(i)); + f->child("child", f2); + } + + f->output(out, 0); + } + + metadata::ptr md_; + }; + + //-------------------------------- + + int debug(string const &path) { + using dbg::command; + + try { + block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY); + metadata::ptr md(new metadata(bm)); + command_interpreter::ptr interp = create_command_interpreter(cin, cout); + interp->register_command("hello", command::ptr(new hello)); + interp->register_command("superblock", command::ptr(new show_superblock(md))); + interp->register_command("mapping_node", command::ptr(new show_btree_node(md))); + interp->register_command("mapping_array", command::ptr(new show_array_block(md))); + interp->register_command("help", command::ptr(new help)); + interp->register_command("exit", command::ptr(new exit_handler(interp))); + interp->enter_main_loop(); + + } catch (std::exception &e) { + cerr << e.what(); + return 1; + } + + return 0; + } +} + +//---------------------------------------------------------------- + +cache_debug_cmd::cache_debug_cmd() + : command("cache_debug") +{ +} + +void +cache_debug_cmd::usage(std::ostream &out) const +{ + out << "Usage: " << get_name() << " {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-V|--version}" << endl; +} + +int +cache_debug_cmd::run(int argc, char **argv) +{ + int c; + const char shortopts[] = "hV"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'V': + cerr << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + } + } + + if (argc == optind) { + usage(cerr); + exit(1); + } + + return debug(argv[optind]); +} + +//---------------------------------------------------------------- diff --git a/caching/commands.h b/caching/commands.h index b0546ab..04deac9 100644 --- a/caching/commands.h +++ b/caching/commands.h @@ -70,6 +70,18 @@ namespace caching { virtual int run(int argc, char **argv); }; + //------------------------------------------------------ + + class cache_debug_cmd : public base::command { + public: + cache_debug_cmd(); + + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + //------------------------------------------------------ + void register_cache_commands(base::application &app); } diff --git a/caching/devel_commands.cc b/caching/devel_commands.cc index 21a1188..c63a68d 100644 --- a/caching/devel_commands.cc +++ b/caching/devel_commands.cc @@ -8,6 +8,7 @@ using namespace caching; void caching::register_cache_commands(application &app) { + app.add_cmd(command::ptr(new cache_debug_cmd)); } //---------------------------------------------------------------- From 6f30d17f04851b28f770927d894134cdc8cd2a05 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 4 Feb 2021 10:00:32 +0000 Subject: [PATCH 18/33] version.rs includes VERSION directly. So you no longer need to run ./configure to build the rust tools --- configure.ac | 1 - src/version.rs | 1 + src/version.rs.in | 2 -- 3 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 src/version.rs delete mode 100644 src/version.rs.in diff --git a/configure.ac b/configure.ac index 947ffdd..21dca90 100644 --- a/configure.ac +++ b/configure.ac @@ -206,6 +206,5 @@ Makefile contrib/Makefile unit-tests/Makefile version.h -src/version.rs ]) AC_OUTPUT diff --git a/src/version.rs b/src/version.rs new file mode 100644 index 0000000..fe73777 --- /dev/null +++ b/src/version.rs @@ -0,0 +1 @@ +pub const TOOLS_VERSION: &str = include_str!("../VERSION"); diff --git a/src/version.rs.in b/src/version.rs.in deleted file mode 100644 index afc9ef2..0000000 --- a/src/version.rs.in +++ /dev/null @@ -1,2 +0,0 @@ -pub const TOOLS_VERSION: &str = @THIN_PROVISIONING_TOOLS_VERSION@; - From c3c6d37aea581f872e768e5d1a6dbb673e173e1a Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 8 Feb 2021 10:38:21 +0000 Subject: [PATCH 19/33] Fix a lot of clippy warnings --- src/bin/thin_check.rs | 2 +- src/bin/thin_explore.rs | 60 +++++++++++++------------ src/io_engine.rs | 4 +- src/pack/vm.rs | 8 +++- src/pdata/btree.rs | 82 ++++++++++++++++++---------------- src/pdata/btree_builder.rs | 2 +- src/pdata/btree_leaf_walker.rs | 15 +++---- src/pdata/btree_merge.rs | 8 ++-- src/pdata/btree_walker.rs | 32 ++++++------- src/thin/check.rs | 65 +++++++++++++-------------- src/thin/dump.rs | 22 ++++----- src/thin/runs.rs | 6 +++ 12 files changed, 154 insertions(+), 152 deletions(-) diff --git a/src/bin/thin_check.rs b/src/bin/thin_check.rs index 8d96510..b7fbad9 100644 --- a/src/bin/thin_check.rs +++ b/src/bin/thin_check.rs @@ -115,7 +115,7 @@ fn main() { let opts = ThinCheckOptions { dev: &input_file, - async_io: async_io, + async_io, sb_only: matches.is_present("SB_ONLY"), skip_mappings: matches.is_present("SKIP_MAPPINGS"), ignore_non_fatal: matches.is_present("IGNORE_NON_FATAL"), diff --git a/src/bin/thin_explore.rs b/src/bin/thin_explore.rs index e1ba44e..356cea6 100644 --- a/src/bin/thin_explore.rs +++ b/src/bin/thin_explore.rs @@ -72,7 +72,6 @@ impl Events { let (tx, rx) = mpsc::channel(); let ignore_exit_key = Arc::new(AtomicBool::new(false)); let input_handle = { - let tx = tx.clone(); let ignore_exit_key = ignore_exit_key.clone(); thread::spawn(move || { let stdin = io::stdin(); @@ -110,6 +109,12 @@ impl Events { } } +impl Default for Events { + fn default() -> Self { + Self::new() + } +} + //------------------------------------ fn ls_next(ls: &mut ListState, max: usize) { @@ -158,7 +163,7 @@ impl<'a> StatefulWidget for SBWidget<'a> { let sb = self.sb; let flags = ["flags".to_string(), format!("{}", sb.flags)]; let block = ["block".to_string(), format!("{}", sb.block)]; - let uuid = ["uuid".to_string(), format!("-")]; + let uuid = ["uuid".to_string(), "-".to_string()]; let version = ["version".to_string(), format!("{}", sb.version)]; let time = ["time".to_string(), format!("{}", sb.time)]; let transaction_id = [ @@ -208,11 +213,11 @@ impl<'a> StatefulWidget for SBWidget<'a> { Widget::render(table, chunks[0], buf); - let items = vec![ - ListItem::new(Span::raw(format!("Device tree"))), - ListItem::new(Span::raw(format!("Mapping tree"))) + let items = vec![ + ListItem::new(Span::raw("Device tree".to_string())), + ListItem::new(Span::raw("Mapping tree".to_string())), ]; - + let items = List::new(items) .block(Block::default().borders(Borders::ALL).title("Entries")) .highlight_style( @@ -315,7 +320,7 @@ impl Adjacent for (X, Y) { fn adjacent_runs(mut ns: Vec) -> Vec<(V, usize)> { let mut result = Vec::new(); - if ns.len() == 0 { + if ns.is_empty() { return result; } @@ -330,13 +335,13 @@ fn adjacent_runs(mut ns: Vec) -> Vec<(V, usize)> { current = v; len += 1; } else { - result.push((base.clone(), len)); - base = v.clone(); - current = v.clone(); + result.push((base, len)); + base = v; + current = v; len = 1; } } - result.push((base.clone(), len)); + result.push((base, len)); result } @@ -344,7 +349,7 @@ fn adjacent_runs(mut ns: Vec) -> Vec<(V, usize)> { fn mk_runs(keys: &[u64], values: &[V]) -> Vec<((u64, V), usize)> { let mut pairs = Vec::new(); for (k, v) in keys.iter().zip(values.iter()) { - pairs.push((k.clone(), v.clone())); + pairs.push((*k, *v)); } adjacent_runs(pairs) @@ -498,7 +503,7 @@ impl Panel for SBPanel { } else { Some(PushTopLevel(self.sb.mapping_root)) } - }, + } Key::Char('h') | Key::Left => Some(PopPanel), _ => None, } @@ -571,14 +576,14 @@ impl Panel for DeviceDetailPanel { fn path_action(&mut self, child: u64) -> Option { match &self.node { btree::Node::Internal { values, .. } => { - for i in 0..values.len() { - if values[i] == child { + for (i, v) in values.iter().enumerate() { + if *v == child { self.state.select(Some(i)); return Some(PushDeviceDetail(child)); } } - return None; + None } btree::Node::Leaf { .. } => None, } @@ -645,14 +650,14 @@ impl Panel for TopLevelPanel { fn path_action(&mut self, child: u64) -> Option { match &self.node { btree::Node::Internal { values, .. } => { - for i in 0..values.len() { - if values[i] == child { + for (i, v) in values.iter().enumerate() { + if *v == child { self.state.select(Some(i)); return Some(PushTopLevel(child)); } } - return None; + None } btree::Node::Leaf { keys, values, .. } => { for i in 0..values.len() { @@ -662,7 +667,7 @@ impl Panel for TopLevelPanel { } } - return None; + None } } } @@ -726,14 +731,14 @@ impl Panel for BottomLevelPanel { fn path_action(&mut self, child: u64) -> Option { match &self.node { btree::Node::Internal { values, .. } => { - for i in 0..values.len() { - if values[i] == child { + for (i, v) in values.iter().enumerate() { + if *v == child { self.state.select(Some(i)); return Some(PushBottomLevel(self.thin_id, child)); } } - return None; + None } btree::Node::Leaf { .. } => None, } @@ -789,7 +794,7 @@ fn explore(path: &Path, node_path: Option>) -> Result<()> { } } else { let sb = read_superblock(&engine, 0)?; - panels.push(Box::new(SBPanel::new(sb.clone()))); + panels.push(Box::new(SBPanel::new(sb))); } let events = Events::new(); @@ -826,12 +831,11 @@ fn explore(path: &Path, node_path: Option>) -> Result<()> { if let Event::Input(key) = events.next()? { match key { Key::Char('q') => break 'main, - _ => match active_panel.input(key) { - Some(action) => { + _ => { + if let Some(action) = active_panel.input(key) { perform_action(&mut panels, &engine, action)?; } - _ => {} - }, + } } } } diff --git a/src/io_engine.rs b/src/io_engine.rs index baecae2..f245a03 100644 --- a/src/io_engine.rs +++ b/src/io_engine.rs @@ -269,10 +269,8 @@ impl AsyncIoEngine { cqes.sort_by(|a, b| a.user_data().partial_cmp(&b.user_data()).unwrap()); let mut rs = Vec::new(); - let mut i = 0; - for b in blocks { + for (i, b) in blocks.into_iter().enumerate() { let c = &cqes[i]; - i += 1; let r = c.result(); if r < 0 { diff --git a/src/pack/vm.rs b/src/pack/vm.rs index 8680a43..dc55e0e 100644 --- a/src/pack/vm.rs +++ b/src/pack/vm.rs @@ -173,7 +173,7 @@ pub fn pack_literal(w: &mut W, bs: &[u8]) -> io::Result<()> { use Tag::LitW; let len = bs.len() as u64; - if len < 16 as u64 { + if len < 16 { pack_tag(w, Tag::Lit, len as u8)?; } else if len <= u8::MAX as u64 { pack_tag(w, LitW, 1)?; @@ -370,6 +370,12 @@ impl VM { } } +impl Default for VM { + fn default() -> Self { + Self::new() + } +} + pub fn unpack(r: &mut R, count: usize) -> io::Result> { let mut w = Vec::with_capacity(4096); let mut cursor = Cursor::new(&mut w); diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 68cb6b3..4351456 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -26,6 +26,12 @@ impl KeyRange { } } +impl Default for KeyRange { + fn default() -> Self { + Self::new() + } +} + impl fmt::Display for KeyRange { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match (self.start, self.end) { @@ -182,22 +188,20 @@ fn test_split_range() { } } -fn split_one(path: &Vec, kr: &KeyRange, k: u64) -> Result<(KeyRange, KeyRange)> { +fn split_one(path: &[u64], kr: &KeyRange, k: u64) -> Result<(KeyRange, KeyRange)> { match kr.split(k) { - None => { - return Err(node_err( - path, - &format!("couldn't split key range {} at {}", kr, k), - )); - } + None => Err(node_err( + path, + &format!("couldn't split key range {} at {}", kr, k), + )), Some(pair) => Ok(pair), } } -pub fn split_key_ranges(path: &Vec, kr: &KeyRange, keys: &[u64]) -> Result> { +pub fn split_key_ranges(path: &[u64], kr: &KeyRange, keys: &[u64]) -> Result> { let mut krs = Vec::with_capacity(keys.len()); - if keys.len() == 0 { + if keys.is_empty() { return Err(node_err(path, "split_key_ranges: no keys present")); } @@ -207,8 +211,8 @@ pub fn split_key_ranges(path: &Vec, kr: &KeyRange, keys: &[u64]) -> Result< end: kr.end, }; - for i in 1..keys.len() { - let (first, rest) = split_one(path, &kr, keys[i])?; + for k in keys.iter().skip(1) { + let (first, rest) = split_one(path, &kr, *k)?; krs.push(first); kr = rest; } @@ -229,7 +233,7 @@ pub fn encode_node_path(path: &[u64]) -> String { // The first entry is normally the superblock (0), so we // special case this. - if path.len() > 0 && path[0] == 0 { + if !path.is_empty() && path[0] == 0 { let count = ((path.len() as u8) - 1) << 1; cursor.write_u8(count as u8).unwrap(); vm::pack_u64s(&mut cursor, &path[1..]).unwrap(); @@ -349,19 +353,19 @@ impl fmt::Display for BTreeError { } } } -pub fn node_err(path: &Vec, msg: &str) -> BTreeError { +pub fn node_err(path: &[u64], msg: &str) -> BTreeError { BTreeError::Path( - path.clone(), + path.to_vec(), Box::new(BTreeError::NodeError(msg.to_string())), ) } -pub fn node_err_s(path: &Vec, msg: String) -> BTreeError { - BTreeError::Path(path.clone(), Box::new(BTreeError::NodeError(msg))) +pub fn node_err_s(path: &[u64], msg: String) -> BTreeError { + BTreeError::Path(path.to_vec(), Box::new(BTreeError::NodeError(msg))) } -pub fn io_err(path: &Vec) -> BTreeError { - BTreeError::Path(path.clone(), Box::new(BTreeError::IoError)) +pub fn io_err(path: &[u64]) -> BTreeError { + BTreeError::Path(path.to_vec(), Box::new(BTreeError::IoError)) } pub fn value_err(msg: String) -> BTreeError { @@ -424,22 +428,22 @@ impl Unpack for NodeHeader { impl Pack for NodeHeader { fn pack(&self, w: &mut W) -> anyhow::Result<()> { - // csum needs to be calculated right for the whole metadata block. - w.write_u32::(0)?; + // csum needs to be calculated right for the whole metadata block. + w.write_u32::(0)?; - let flags; - if self.is_leaf { - flags = LEAF_NODE; - } else { - flags = INTERNAL_NODE; - } - w.write_u32::(flags)?; - w.write_u64::(self.block)?; - w.write_u32::(self.nr_entries)?; - w.write_u32::(self.max_entries)?; - w.write_u32::(self.value_size)?; - w.write_u32::(0)?; - Ok(()) + let flags; + if self.is_leaf { + flags = LEAF_NODE; + } else { + flags = INTERNAL_NODE; + } + w.write_u32::(flags)?; + w.write_u64::(self.block)?; + w.write_u32::(self.nr_entries)?; + w.write_u32::(self.max_entries)?; + w.write_u32::(self.value_size)?; + w.write_u32::(0)?; + Ok(()) } } @@ -473,7 +477,7 @@ impl Node { Leaf { header, .. } => header, } } - + pub fn get_keys(&self) -> &[u64] { use Node::*; match self { @@ -487,16 +491,16 @@ impl Node { } } -pub fn convert_result<'a, V>(path: &Vec, r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> { +pub fn convert_result<'a, V>(path: &[u64], r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> { r.map_err(|_e| node_err(path, "parse error")) } -pub fn convert_io_err(path: &Vec, r: std::io::Result) -> Result { +pub fn convert_io_err(path: &[u64], r: std::io::Result) -> Result { r.map_err(|_| io_err(path)) } pub fn unpack_node( - path: &Vec, + path: &[u64], data: &[u8], ignore_non_fatal: bool, is_root: bool, @@ -554,7 +558,7 @@ pub fn unpack_node( for k in &keys { if let Some(l) = last { if k <= l { - return Err(node_err(path, "keys out of order")); + return Err(node_err(&path, "keys out of order")); } } @@ -573,7 +577,7 @@ pub fn unpack_node( values, }) } else { - let (_i, values) = convert_result(path, count(le_u64, header.nr_entries as usize)(i))?; + let (_i, values) = convert_result(&path, count(le_u64, header.nr_entries as usize)(i))?; Ok(Node::Internal { header, keys, diff --git a/src/pdata/btree_builder.rs b/src/pdata/btree_builder.rs index 05567cd..44504ca 100644 --- a/src/pdata/btree_builder.rs +++ b/src/pdata/btree_builder.rs @@ -153,7 +153,7 @@ pub struct NodeSummary { fn write_node_(w: &mut WriteBatcher, mut node: Node) -> Result<(u64, u64)> { let keys = node.get_keys(); - let first_key = keys.first().unwrap_or(&0u64).clone(); + let first_key = *keys.first().unwrap_or(&0u64); let loc = w.alloc()?; node.set_block(loc); diff --git a/src/pdata/btree_leaf_walker.rs b/src/pdata/btree_leaf_walker.rs index dd74865..926afb0 100644 --- a/src/pdata/btree_leaf_walker.rs +++ b/src/pdata/btree_leaf_walker.rs @@ -102,8 +102,7 @@ impl<'a> LeafWalker<'a> { .read_many(&blocks[0..]) .map_err(|_e| io_err(path))?; - let mut i = 0; - for rb in rblocks { + for (i, rb) in rblocks.into_iter().enumerate() { match rb { Err(_) => { return Err(io_err(path).keys_context(&filtered_krs[i])); @@ -112,8 +111,6 @@ impl<'a> LeafWalker<'a> { self.walk_node(depth - 1, path, visitor, &filtered_krs[i], &b, false)?; } } - - i += 1; } Ok(()) @@ -225,14 +222,12 @@ impl<'a> LeafWalker<'a> { self.leaves.insert(root as usize); visitor.visit(&kr, root)?; Ok(()) + } else if self.sm_inc(root) > 0 { + visitor.visit_again(root) } else { - if self.sm_inc(root) > 0 { - visitor.visit_again(root) - } else { - let root = self.engine.read(root).map_err(|_| io_err(path))?; + let root = self.engine.read(root).map_err(|_| io_err(path))?; - self.walk_node(depth - 1, path, visitor, &kr, &root, true) - } + self.walk_node(depth - 1, path, visitor, &kr, &root, true) } } diff --git a/src/pdata/btree_merge.rs b/src/pdata/btree_merge.rs index 0eea0ab..8f3b75a 100644 --- a/src/pdata/btree_merge.rs +++ b/src/pdata/btree_merge.rs @@ -49,21 +49,21 @@ impl LeafVisitor { impl NodeVisitor for LeafVisitor { fn visit( &self, - path: &Vec, + path: &[u64], _kr: &KeyRange, _header: &NodeHeader, keys: &[u64], _values: &[V], ) -> btree::Result<()> { // ignore empty nodes - if keys.len() == 0 { + if keys.is_empty() { return Ok(()); } let mut inner = self.inner.lock().unwrap(); // Check keys are ordered. - if inner.leaves.len() > 0 { + if !inner.leaves.is_empty() { let last_key = inner.leaves.last().unwrap().key_high; if keys[0] <= last_key { return Err(BTreeError::NodeError( @@ -83,7 +83,7 @@ impl NodeVisitor for LeafVisitor { Ok(()) } - fn visit_again(&self, _path: &Vec, _b: u64) -> btree::Result<()> { + fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> { Ok(()) } diff --git a/src/pdata/btree_walker.rs b/src/pdata/btree_walker.rs index 6834ccf..3e9c2d3 100644 --- a/src/pdata/btree_walker.rs +++ b/src/pdata/btree_walker.rs @@ -14,7 +14,7 @@ pub trait NodeVisitor { // &self is deliberately non mut to allow the walker to use multiple threads. fn visit( &self, - path: &Vec, + path: &[u64], kr: &KeyRange, header: &NodeHeader, keys: &[u64], @@ -24,7 +24,7 @@ pub trait NodeVisitor { // Nodes may be shared and thus visited multiple times. The walker avoids // doing repeated IO, but it does call this method to keep the visitor up to // date. - fn visit_again(&self, path: &Vec, b: u64) -> Result<()>; + fn visit_again(&self, path: &[u64], b: u64) -> Result<()>; fn end_walk(&self) -> Result<()>; } @@ -154,8 +154,7 @@ impl BTreeWalker { } } Ok(rblocks) => { - let mut i = 0; - for rb in rblocks { + for (i, rb) in rblocks.into_iter().enumerate() { match rb { Err(_) => { let e = io_err(path).keys_context(&filtered_krs[i]); @@ -169,8 +168,6 @@ impl BTreeWalker { Ok(()) => {} }, } - - i += 1; } } } @@ -215,7 +212,7 @@ impl BTreeWalker { values, } => { if let Err(e) = visitor.visit(path, &kr, &header, &keys, &values) { - let e = BTreeError::Path(path.clone(), Box::new(e.clone())); + let e = BTreeError::Path(path.clone(), Box::new(e)); self.set_fail(b.loc, e.clone()); return Err(e); } @@ -251,7 +248,7 @@ impl BTreeWalker { { if self.sm_inc(root) > 0 { if let Some(e) = self.failed(root) { - Err(e.clone()) + Err(e) } else { visitor.visit_again(path, root) } @@ -382,10 +379,9 @@ where } } Ok(rblocks) => { - let mut i = 0; let errs = Arc::new(Mutex::new(Vec::new())); - for rb in rblocks { + for (i, rb) in rblocks.into_iter().enumerate() { match rb { Err(_) => { let e = io_err(path).keys_context(&filtered_krs[i]); @@ -411,8 +407,6 @@ where }); } } - - i += 1; } pool.join(); @@ -435,7 +429,7 @@ where { if w.sm_inc(root) > 0 { if let Some(e) = w.failed(root) { - Err(e.clone()) + Err(e) } else { visitor.visit_again(path, root) } @@ -467,7 +461,7 @@ impl ValueCollector { impl NodeVisitor for ValueCollector { fn visit( &self, - _path: &Vec, + _path: &[u64], _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], @@ -475,13 +469,13 @@ impl NodeVisitor for ValueCollector { ) -> Result<()> { let mut vals = self.values.lock().unwrap(); for n in 0..keys.len() { - vals.insert(keys[n], values[n].clone()); + vals.insert(keys[n], values[n]); } Ok(()) } - fn visit_again(&self, _path: &Vec, _b: u64) -> Result<()> { + fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> { Ok(()) } @@ -533,7 +527,7 @@ impl ValuePathCollector { impl NodeVisitor for ValuePathCollector { fn visit( &self, - path: &Vec, + path: &[u64], _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], @@ -541,13 +535,13 @@ impl NodeVisitor for ValuePathCollector { ) -> Result<()> { let mut vals = self.values.lock().unwrap(); for n in 0..keys.len() { - vals.insert(keys[n], (path.clone(), values[n].clone())); + vals.insert(keys[n], (path.to_vec(), values[n].clone())); } Ok(()) } - fn visit_again(&self, _path: &Vec, _b: u64) -> Result<()> { + fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> { Ok(()) } diff --git a/src/thin/check.rs b/src/thin/check.rs index e5f3d46..cb25257 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -2,6 +2,7 @@ use anyhow::{anyhow, Result}; use std::collections::BTreeMap; use std::io::Cursor; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; use threadpool::ThreadPool; @@ -28,7 +29,7 @@ struct BottomLevelVisitor { impl NodeVisitor for BottomLevelVisitor { fn visit( &self, - _path: &Vec, + _path: &[u64], _kr: &KeyRange, _h: &NodeHeader, _k: &[u64], @@ -36,7 +37,7 @@ impl NodeVisitor for BottomLevelVisitor { ) -> btree::Result<()> { // FIXME: do other checks - if values.len() == 0 { + if values.is_empty() { return Ok(()); } @@ -45,8 +46,8 @@ impl NodeVisitor for BottomLevelVisitor { let mut start = values[0].block; let mut len = 1; - for n in 1..values.len() { - let block = values[n].block; + for b in values.iter().skip(1) { + let block = b.block; if block == start + len { len += 1; } else { @@ -60,7 +61,7 @@ impl NodeVisitor for BottomLevelVisitor { Ok(()) } - fn visit_again(&self, _path: &Vec, _b: u64) -> btree::Result<()> { + fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> { Ok(()) } @@ -84,7 +85,7 @@ impl<'a> OverflowChecker<'a> { impl<'a> NodeVisitor for OverflowChecker<'a> { fn visit( &self, - _path: &Vec, + _path: &[u64], _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], @@ -103,7 +104,7 @@ impl<'a> NodeVisitor for OverflowChecker<'a> { Ok(()) } - fn visit_again(&self, _path: &Vec, _b: u64) -> btree::Result<()> { + fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> { Ok(()) } @@ -152,13 +153,12 @@ fn check_space_map( } // FIXME: we should do this in batches - let blocks = engine.read_many(&mut blocks)?; + let blocks = engine.read_many(&blocks)?; let mut leaks = 0; let mut blocknr = 0; let mut bitmap_leaks = Vec::new(); - for n in 0..entries.len() { - let b = &blocks[n]; + for b in blocks.iter().take(entries.len()) { match b { Err(_e) => { return Err(anyhow!("Unable to read bitmap block")); @@ -233,12 +233,8 @@ fn repair_space_map(ctx: &Context, entries: Vec, sm: ASpaceMap) -> R let rblocks = engine.read_many(&blocks[0..])?; let mut write_blocks = Vec::new(); - let mut i = 0; - for rb in rblocks { - if rb.is_err() { - return Err(anyhow!("Unable to reread bitmap blocks for repair")); - } else { - let b = rb.unwrap(); + for (i, rb) in rblocks.into_iter().enumerate() { + if let Ok(b) = rb { let be = &entries[i]; let mut blocknr = be.blocknr; let mut bitmap = unpack::(b.get_data())?; @@ -262,9 +258,9 @@ fn repair_space_map(ctx: &Context, entries: Vec, sm: ASpaceMap) -> R checksum::write_checksum(b.get_data(), checksum::BT::BITMAP)?; write_blocks.push(b); + } else { + return Err(anyhow!("Unable to reread bitmap blocks for repair")); } - - i += 1; } engine.write_many(&write_blocks[0..])?; @@ -305,20 +301,17 @@ fn spawn_progress_thread( sm: Arc>, nr_allocated_metadata: u64, report: Arc, -) -> Result<(JoinHandle<()>, Arc>)> { +) -> Result<(JoinHandle<()>, Arc)> { let tid; - let stop_progress = Arc::new(Mutex::new(false)); + let stop_progress = Arc::new(AtomicBool::new(false)); { let stop_progress = stop_progress.clone(); tid = thread::spawn(move || { let interval = std::time::Duration::from_millis(250); loop { - { - let stop_progress = stop_progress.lock().unwrap(); - if *stop_progress { - break; - } + if stop_progress.load(Ordering::Relaxed) { + break; } let sm = sm.lock().unwrap(); @@ -364,7 +357,7 @@ fn check_mapping_bottom_level( if roots.len() > 64 { let errs = Arc::new(Mutex::new(Vec::new())); - for (_thin_id, (path, root)) in roots { + for (path, root) in roots.values() { let data_sm = data_sm.clone(); let root = *root; let v = BottomLevelVisitor { data_sm }; @@ -381,12 +374,12 @@ fn check_mapping_bottom_level( } ctx.pool.join(); let errs = Arc::try_unwrap(errs).unwrap().into_inner().unwrap(); - if errs.len() > 0 { + if !errs.is_empty() { ctx.report.fatal(&format!("{}", aggregate_error(errs))); failed = true; } } else { - for (_thin_id, (path, root)) in roots { + for (path, root) in roots.values() { let w = w.clone(); let data_sm = data_sm.clone(); let root = *root; @@ -468,7 +461,12 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> { // Device details. We read this once to get the number of thin devices, and hence the // maximum metadata ref count. Then create metadata space map, and reread to increment // the ref counts for that metadata. - let devs = btree_to_map::(&mut path, engine.clone(), opts.ignore_non_fatal, sb.details_root)?; + let devs = btree_to_map::( + &mut path, + engine.clone(), + opts.ignore_non_fatal, + sb.details_root, + )?; let nr_devs = devs.len(); let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32); inc_superblock(&metadata_sm)?; @@ -574,21 +572,18 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> { bail_out(&ctx, "metadata space map")?; if opts.auto_repair { - if data_leaks.len() > 0 { + if !data_leaks.is_empty() { ctx.report.info("Repairing data leaks."); repair_space_map(&ctx, data_leaks, data_sm.clone())?; } - if metadata_leaks.len() > 0 { + if !metadata_leaks.is_empty() { ctx.report.info("Repairing metadata leaks."); repair_space_map(&ctx, metadata_leaks, metadata_sm.clone())?; } } - { - let mut stop_progress = stop_progress.lock().unwrap(); - *stop_progress = true; - } + stop_progress.store(true, Ordering::Relaxed); tid.join().unwrap(); Ok(()) diff --git a/src/thin/dump.rs b/src/thin/dump.rs index cb62330..1385902 100644 --- a/src/thin/dump.rs +++ b/src/thin/dump.rs @@ -38,7 +38,7 @@ impl RunBuilder { self.run = Some(xml::Map { thin_begin: thin_block, data_begin: data_block, - time: time, + time, len: 1, }); None @@ -59,7 +59,7 @@ impl RunBuilder { self.run.replace(Map { thin_begin: thin_block, data_begin: data_block, - time: time, + time, len: 1, }) } @@ -99,7 +99,7 @@ impl<'a> MappingVisitor<'a> { impl<'a> NodeVisitor for MappingVisitor<'a> { fn visit( &self, - _path: &Vec, + _path: &[u64], _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], @@ -118,7 +118,7 @@ impl<'a> NodeVisitor for MappingVisitor<'a> { Ok(()) } - fn visit_again(&self, _path: &Vec, b: u64) -> btree::Result<()> { + fn visit_again(&self, _path: &[u64], b: u64) -> btree::Result<()> { let mut inner = self.inner.lock().unwrap(); inner .md_out @@ -337,7 +337,7 @@ fn build_metadata(ctx: &Context, sb: &Superblock) -> Result { let (mut shared, sm) = find_shared_nodes(ctx, &roots)?; // Add in the roots, because they may not be shared. - for (_thin_id, (_path, root)) in &roots { + for (_path, root) in roots.values() { shared.insert(*root); } @@ -354,7 +354,7 @@ fn build_metadata(ctx: &Context, sb: &Superblock) -> Result { let kr = KeyRange::new(); // FIXME: finish devs.push(Device { thin_id: thin_id as u32, - detail: detail.clone(), + detail: *detail, map: Mapping { kr, entries: es.to_vec(), @@ -381,7 +381,7 @@ fn build_metadata(ctx: &Context, sb: &Superblock) -> Result { //------------------------------------------ -fn gather_entries(g: &mut Gatherer, es: &Vec) { +fn gather_entries(g: &mut Gatherer, es: &[Entry]) { g.new_seq(); for e in es { match e { @@ -395,7 +395,7 @@ fn gather_entries(g: &mut Gatherer, es: &Vec) { } } -fn entries_to_runs(runs: &BTreeMap>, es: &Vec) -> Vec { +fn entries_to_runs(runs: &BTreeMap>, es: &[Entry]) -> Vec { use Entry::*; let mut result = Vec::new(); @@ -542,7 +542,7 @@ fn emit_leaves(ctx: &Context, out: &mut dyn xml::MetadataVisitor, ls: &[u64]) -> fn emit_entries( ctx: &Context, out: &mut xml::XmlWriter, - entries: &Vec, + entries: &[Entry], ) -> Result<()> { let mut leaves = Vec::new(); @@ -552,7 +552,7 @@ fn emit_entries( leaves.push(*b); } Entry::Ref(id) => { - if leaves.len() > 0 { + if !leaves.is_empty() { emit_leaves(&ctx, out, &leaves[0..])?; leaves.clear(); } @@ -562,7 +562,7 @@ fn emit_entries( } } - if leaves.len() > 0 { + if !leaves.is_empty() { emit_leaves(&ctx, out, &leaves[0..])?; } diff --git a/src/thin/runs.rs b/src/thin/runs.rs index a149ba6..451e7c7 100644 --- a/src/thin/runs.rs +++ b/src/thin/runs.rs @@ -139,6 +139,12 @@ impl Gatherer { } } +impl Default for Gatherer { + fn default() -> Self { + Self::new() + } +} + //------------------------------------------ #[cfg(test)] From 763b2d14b2356497b3a9b0fb9870d42af3a359b3 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 9 Feb 2021 14:34:26 +0000 Subject: [PATCH 20/33] [io_engine] Add Block::zeroed() constructor --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/io_engine.rs | 7 +++++++ src/report.rs | 1 + 4 files changed, 16 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 761163b..e4ed583 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -671,6 +671,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + [[package]] name = "shared_child" version = "0.3.4" @@ -798,6 +804,7 @@ dependencies = [ "quickcheck", "quickcheck_macros", "rand", + "safemem", "tempfile", "termion", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index a98b956..b47c1b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ num-derive = "0.3" num-traits = "0.2" quick-xml = "0.18" rand = "0.7" +safemem = "0.3.3" tempfile = "3.1" threadpool = "1.8" thiserror = "1.0" diff --git a/src/io_engine.rs b/src/io_engine.rs index f245a03..879c294 100644 --- a/src/io_engine.rs +++ b/src/io_engine.rs @@ -1,5 +1,6 @@ use io_uring::opcode::{self, types}; use io_uring::IoUring; +use safemem::write_bytes; use std::alloc::{alloc, dealloc, Layout}; use std::fs::File; use std::fs::OpenOptions; @@ -32,6 +33,12 @@ impl Block { Block { loc, data: ptr } } + pub fn zeroed(loc: u64) -> Block { + let r = Self::new(loc); + write_bytes(r.get_data(), 0); + r + } + pub fn get_data<'a>(&self) -> &'a mut [u8] { unsafe { std::slice::from_raw_parts_mut::<'a>(self.data, BLOCK_SIZE) } } diff --git a/src/report.rs b/src/report.rs index 0131f00..e105fd5 100644 --- a/src/report.rs +++ b/src/report.rs @@ -95,6 +95,7 @@ impl Report { //------------------------------------------ +#[allow(dead_code)] struct PBInner { title: String, bar: ProgressBar, From 58cd881340357d97dc1ddfbdc16557423a65c792 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 17 Feb 2021 15:15:57 +0000 Subject: [PATCH 21/33] Fix regression where era_restore wouldn't work with devices. check_file_exists() had an extra parameter added with a default, which was the wrong default for era_restore. --- era/era_restore.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/era/era_restore.cc b/era/era_restore.cc index 0603a56..27dcadc 100644 --- a/era/era_restore.cc +++ b/era/era_restore.cc @@ -37,7 +37,7 @@ namespace { bool metadata_touched = false; try { block_manager::ptr bm = open_bm(*fs.output, block_manager::READ_WRITE); - file_utils::check_file_exists(*fs.output); + file_utils::check_file_exists(*fs.output, false); metadata_touched = true; metadata::ptr md(new metadata(bm, metadata::CREATE)); emitter::ptr restorer = create_restore_emitter(*md); From e3a646e4d2ca423feb4493f0b92e72f6e475aa8b Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 5 Feb 2021 16:15:24 +0800 Subject: [PATCH 22/33] [dbg] Improve error resilience of debugging tools Do not open the metadata. The tool interprets user specified blocks one-by-one. Thus, there's no need to preload the metadata structures. Also remove the unused --ignore-metadata-space-map option. It was designed to control the loading of space maps with the old metadata constructors. --- caching/cache_debug.cc | 42 ++++++++++--------- thin-provisioning/thin_debug.cc | 71 ++++++++++++++++----------------- 2 files changed, 57 insertions(+), 56 deletions(-) diff --git a/caching/cache_debug.cc b/caching/cache_debug.cc index 1053560..155fd94 100644 --- a/caching/cache_debug.cc +++ b/caching/cache_debug.cc @@ -48,7 +48,7 @@ namespace { class help : public dbg::command { virtual void exec(strings const &args, ostream &out) { out << "Commands:" << endl - << " superblock" << endl + << " superblock [block#]" << endl << " mapping_node " << endl << " mapping_array " << endl << " exit" << endl; @@ -82,16 +82,21 @@ namespace { class show_superblock : public dbg::command { public: - explicit show_superblock(metadata::ptr md) - : md_(md) { + explicit show_superblock(block_manager::ptr bm) + : bm_(bm) { } virtual void exec(strings const &args, ostream &out) { + if (args.size() > 2) + throw runtime_error("incorrect number of arguments"); + + block_address b = caching::SUPERBLOCK_LOCATION; + if (args.size() == 2) + b = boost::lexical_cast(args[1]); + caching::superblock sb = read_superblock(bm_, b); + formatter::ptr f = create_xml_formatter(); ostringstream version; - - superblock const &sb = md_->sb_; - field(*f, "csum", sb.csum); field(*f, "flags", sb.flags.encode()); field(*f, "blocknr", sb.blocknr); @@ -137,7 +142,7 @@ namespace { } private: - metadata::ptr md_; + block_manager::ptr bm_; }; class uint64_show_traits : public uint64_traits { @@ -167,8 +172,8 @@ namespace { template class show_btree_node : public dbg::command { public: - explicit show_btree_node(metadata::ptr md) - : md_(md) { + explicit show_btree_node(block_manager::ptr bm) + : bm_(bm) { } virtual void exec(strings const &args, ostream &out) { @@ -178,7 +183,7 @@ namespace { throw runtime_error("incorrect number of arguments"); block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = md_->tm_->read_lock(block); + block_manager::read_ref rr = bm_->read_lock(block); node_ref n = btree_detail::to_node(rr); if (n.get_type() == INTERNAL) @@ -211,15 +216,15 @@ namespace { f->output(out, 0); } - metadata::ptr md_; + block_manager::ptr bm_; }; template class show_array_block : public dbg::command { typedef array_block rblock; public: - explicit show_array_block(metadata::ptr md) - : md_(md) { + explicit show_array_block(block_manager::ptr bm) + : bm_(bm) { } virtual void exec(strings const& args, ostream &out) { @@ -227,7 +232,7 @@ namespace { throw runtime_error("incorrect number of arguments"); block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = md_->tm_->read_lock(block); + block_manager::read_ref rr = bm_->read_lock(block); rblock b(rr, typename ValueTraits::ref_counter()); show_array_entries(b, out); @@ -252,7 +257,7 @@ namespace { f->output(out, 0); } - metadata::ptr md_; + block_manager::ptr bm_; }; //-------------------------------- @@ -262,12 +267,11 @@ namespace { try { block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY); - metadata::ptr md(new metadata(bm)); command_interpreter::ptr interp = create_command_interpreter(cin, cout); interp->register_command("hello", command::ptr(new hello)); - interp->register_command("superblock", command::ptr(new show_superblock(md))); - interp->register_command("mapping_node", command::ptr(new show_btree_node(md))); - interp->register_command("mapping_array", command::ptr(new show_array_block(md))); + interp->register_command("superblock", command::ptr(new show_superblock(bm))); + interp->register_command("mapping_node", command::ptr(new show_btree_node(bm))); + interp->register_command("mapping_array", command::ptr(new show_array_block(bm))); interp->register_command("help", command::ptr(new help)); interp->register_command("exit", command::ptr(new exit_handler(interp))); interp->enter_main_loop(); diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index e458d7d..37a5f18 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -48,7 +48,7 @@ namespace { class help : public dbg::command { virtual void exec(strings const &args, ostream &out) { out << "Commands:" << endl - << " superblock" << endl + << " superblock [block#]" << endl << " m1_node " << endl << " m2_node " << endl << " detail_node " << endl @@ -85,15 +85,20 @@ namespace { class show_superblock : public dbg::command { public: - explicit show_superblock(metadata::ptr md) - : md_(md) { + explicit show_superblock(block_manager::ptr bm) + : bm_(bm) { } virtual void exec(strings const &args, ostream &out) { + if (args.size() > 2) + throw runtime_error("incorrect number of arguments"); + + block_address b = superblock_detail::SUPERBLOCK_LOCATION; + if (args.size() == 2) + b = boost::lexical_cast(args[1]); + superblock_detail::superblock sb = read_superblock(bm_, b); + formatter::ptr f = create_xml_formatter(); - - thin_provisioning::superblock_detail::superblock const &sb = md_->sb_; - field(*f, "csum", sb.csum_); field(*f, "flags", sb.flags_); field(*f, "blocknr", sb.blocknr_); @@ -134,7 +139,7 @@ namespace { } private: - metadata::ptr md_; + block_manager::ptr bm_; }; class device_details_show_traits : public thin_provisioning::device_tree_detail::device_details_traits { @@ -185,8 +190,8 @@ namespace { template class show_btree_node : public dbg::command { public: - explicit show_btree_node(metadata::ptr md) - : md_(md) { + explicit show_btree_node(block_manager::ptr bm) + : bm_(bm) { } virtual void exec(strings const &args, ostream &out) { @@ -196,7 +201,7 @@ namespace { throw runtime_error("incorrect number of arguments"); block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = md_->tm_->read_lock(block); + block_manager::read_ref rr = bm_->read_lock(block); node_ref n = btree_detail::to_node(rr); if (n.get_type() == INTERNAL) @@ -229,32 +234,27 @@ namespace { f->output(out, 0); } - metadata::ptr md_; + block_manager::ptr bm_; }; class show_index_block : public dbg::command { public: - explicit show_index_block(metadata::ptr md) - : md_(md) { + explicit show_index_block(block_manager::ptr bm) + : bm_(bm) { } virtual void exec(strings const &args, ostream &out) { if (args.size() != 2) throw runtime_error("incorrect number of arguments"); - // manually load metadata_index, without using index_validator() block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = md_->tm_->read_lock(block); - sm_disk_detail::sm_root_disk const *d = - reinterpret_cast(md_->sb_.metadata_space_map_root_); - sm_disk_detail::sm_root v; - sm_disk_detail::sm_root_traits::unpack(*d, v); - block_address nr_indexes = base::div_up(v.nr_blocks_, ENTRIES_PER_BLOCK); + // no checksum validation for debugging purpose + block_manager::read_ref rr = bm_->read_lock(block); sm_disk_detail::metadata_index const *mdi = reinterpret_cast(rr.data()); - show_metadata_index(mdi, nr_indexes, out); + show_metadata_index(mdi, sm_disk_detail::MAX_METADATA_BITMAPS, out); } private: @@ -267,6 +267,10 @@ namespace { sm_disk_detail::index_entry ie; for (block_address i = 0; i < nr_indexes; i++) { sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), ie); + + if (!ie.blocknr_ && !ie.nr_free_ && !ie.none_free_before_) + continue; + formatter::ptr f2 = create_xml_formatter(); index_entry_show_traits::show(f2, "value", ie); f->child(boost::lexical_cast(i), f2); @@ -275,25 +279,24 @@ namespace { } unsigned const ENTRIES_PER_BLOCK = (MD_BLOCK_SIZE - sizeof(persistent_data::sm_disk_detail::bitmap_header)) * 4; - metadata::ptr md_; + block_manager::ptr bm_; }; //-------------------------------- - int debug_(string const &path, bool ignore_metadata_sm) { + int debug_(string const &path) { using dbg::command; try { block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY, 1); - metadata::ptr md(new metadata(bm, false)); command_interpreter::ptr interp = create_command_interpreter(cin, cout); interp->register_command("hello", command::ptr(new hello)); - interp->register_command("superblock", command::ptr(new show_superblock(md))); - interp->register_command("m1_node", command::ptr(new show_btree_node(md))); - interp->register_command("m2_node", command::ptr(new show_btree_node(md))); - interp->register_command("detail_node", command::ptr(new show_btree_node(md))); - interp->register_command("index_block", command::ptr(new show_index_block(md))); - interp->register_command("index_node", command::ptr(new show_btree_node(md))); + interp->register_command("superblock", command::ptr(new show_superblock(bm))); + interp->register_command("m1_node", command::ptr(new show_btree_node(bm))); + interp->register_command("m2_node", command::ptr(new show_btree_node(bm))); + interp->register_command("detail_node", command::ptr(new show_btree_node(bm))); + interp->register_command("index_block", command::ptr(new show_index_block(bm))); + interp->register_command("index_node", command::ptr(new show_btree_node(bm))); interp->register_command("help", command::ptr(new help)); interp->register_command("exit", command::ptr(new exit_handler(interp))); interp->enter_main_loop(); @@ -329,10 +332,8 @@ thin_debug_cmd::run(int argc, char **argv) const struct option longopts[] = { { "help", no_argument, NULL, 'h'}, { "version", no_argument, NULL, 'V'}, - { "ignore-metadata-sm", no_argument, NULL, 1}, { NULL, no_argument, NULL, 0 } }; - bool ignore_metadata_sm = false; while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { switch(c) { @@ -343,10 +344,6 @@ thin_debug_cmd::run(int argc, char **argv) case 'V': cerr << THIN_PROVISIONING_TOOLS_VERSION << endl; return 0; - - case 1: - ignore_metadata_sm = true; - break; } } @@ -355,5 +352,5 @@ thin_debug_cmd::run(int argc, char **argv) exit(1); } - return debug_(argv[optind], ignore_metadata_sm); + return debug_(argv[optind]); } From 1bc88bfde88f189e719e611bcd326c2ba4d5c5e3 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 7 Feb 2021 02:44:08 +0800 Subject: [PATCH 23/33] [cache_debug] Simplify the trait types --- caching/cache_debug.cc | 59 +++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/caching/cache_debug.cc b/caching/cache_debug.cc index 155fd94..4f0c68d 100644 --- a/caching/cache_debug.cc +++ b/caching/cache_debug.cc @@ -49,8 +49,8 @@ namespace { virtual void exec(strings const &args, ostream &out) { out << "Commands:" << endl << " superblock [block#]" << endl - << " mapping_node " << endl - << " mapping_array " << endl + << " block_node " << endl + << " mapping_block " << endl << " exit" << endl; } }; @@ -145,31 +145,28 @@ namespace { block_manager::ptr bm_; }; + // FIXME: duplication class uint64_show_traits : public uint64_traits { public: + typedef uint64_traits value_trait; + static void show(formatter::ptr f, string const &key, uint64_t const &value) { field(*f, key, boost::lexical_cast(value)); } }; - // FIXME: Expose the array ValueTraits parameter or not? - // Since the block_traits isn't related to the ValueTraits. - class block_show_traits : public persistent_data::array::block_traits { - public: - static void show(formatter::ptr f, string const &key, block_address const &value) { - field(*f, "block", value); - } - }; - class mapping_show_traits : public caching::mapping_traits { public: + typedef mapping_traits value_trait; + static void show(formatter::ptr f, string const &key, caching::mapping const &value) { field(*f, "oblock", value.oblock_); field(*f, "flags", value.flags_); } }; - template + // FIXME: duplication + template class show_btree_node : public dbg::command { public: explicit show_btree_node(block_manager::ptr bm) @@ -185,22 +182,22 @@ namespace { block_address block = boost::lexical_cast(args[1]); block_manager::read_ref rr = bm_->read_lock(block); - node_ref n = btree_detail::to_node(rr); + node_ref n = btree_detail::to_node(rr); if (n.get_type() == INTERNAL) show_node(n, out); else { - node_ref n = btree_detail::to_node(rr); - show_node(n, out); + node_ref n = btree_detail::to_node(rr); + show_node(n, out); } } private: - template - void show_node(node_ref n, ostream &out) { + template + void show_node(node_ref n, ostream &out) { formatter::ptr f = create_xml_formatter(); field(*f, "csum", n.get_checksum()); - field(*f, "blocknr", n.get_location()); + field(*f, "blocknr", n.get_block_nr()); field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); field(*f, "nr_entries", n.get_nr_entries()); field(*f, "max_entries", n.get_max_entries()); @@ -209,8 +206,8 @@ namespace { for (unsigned i = 0; i < n.get_nr_entries(); i++) { formatter::ptr f2 = create_xml_formatter(); field(*f2, "key", n.key_at(i)); - VT::show(f2, "value", n.value_at(i)); - f->child("child", f2); + ST::show(f2, "value", n.value_at(i)); + f->child(boost::lexical_cast(i), f2); } f->output(out, 0); @@ -219,12 +216,13 @@ namespace { block_manager::ptr bm_; }; - template + template class show_array_block : public dbg::command { - typedef array_block rblock; + typedef array_block rblock; public: - explicit show_array_block(block_manager::ptr bm) - : bm_(bm) { + explicit show_array_block(block_manager::ptr bm, + typename ShowTraits::ref_counter rc) + : bm_(bm), rc_(rc) { } virtual void exec(strings const& args, ostream &out) { @@ -234,7 +232,7 @@ namespace { block_address block = boost::lexical_cast(args[1]); block_manager::read_ref rr = bm_->read_lock(block); - rblock b(rr, typename ValueTraits::ref_counter()); + rblock b(rr, rc_); show_array_entries(b, out); } @@ -249,15 +247,15 @@ namespace { for (unsigned i = 0; i < nr_entries; i++) { formatter::ptr f2 = create_xml_formatter(); - field(*f2, "index", i); - ValueTraits::show(f2, "value", b.get(i)); - f->child("child", f2); + ShowTraits::show(f2, "value", b.get(i)); + f->child(boost::lexical_cast(i), f2); } f->output(out, 0); } block_manager::ptr bm_; + typename ShowTraits::ref_counter rc_; }; //-------------------------------- @@ -270,8 +268,9 @@ namespace { command_interpreter::ptr interp = create_command_interpreter(cin, cout); interp->register_command("hello", command::ptr(new hello)); interp->register_command("superblock", command::ptr(new show_superblock(bm))); - interp->register_command("mapping_node", command::ptr(new show_btree_node(bm))); - interp->register_command("mapping_array", command::ptr(new show_array_block(bm))); + interp->register_command("block_node", command::ptr(new show_btree_node(bm))); + interp->register_command("mapping_block", command::ptr(new show_array_block(bm, + mapping_show_traits::ref_counter()))); interp->register_command("help", command::ptr(new help)); interp->register_command("exit", command::ptr(new exit_handler(interp))); interp->enter_main_loop(); From afbd913e2233d8baf967f867c2e6be874707253f Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 7 Feb 2021 16:03:58 +0800 Subject: [PATCH 24/33] [era] Add era_debug --- Makefile.in | 1 + era/commands.h | 12 ++ era/devel_commands.cc | 2 + era/era_debug.cc | 346 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 361 insertions(+) create mode 100644 era/era_debug.cc diff --git a/Makefile.in b/Makefile.in index 949e30a..0392d8e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -131,6 +131,7 @@ DEVTOOLS_SOURCE=\ caching/cache_debug.cc \ caching/devel_commands.cc \ era/devel_commands.cc \ + era/era_debug.cc \ thin-provisioning/damage_generator.cc \ thin-provisioning/devel_commands.cc \ thin-provisioning/thin_debug.cc \ diff --git a/era/commands.h b/era/commands.h index 8e7a92c..1ccaf0b 100644 --- a/era/commands.h +++ b/era/commands.h @@ -34,6 +34,18 @@ namespace era { virtual int run(int argc, char **argv); }; + //------------------------------------------------------ + + class era_debug_cmd : public base::command { + public: + era_debug_cmd(); + + virtual void usage(std::ostream &out) const; + virtual int run(int argc, char **argv); + }; + + //------------------------------------------------------ + void register_era_commands(base::application &app); } diff --git a/era/devel_commands.cc b/era/devel_commands.cc index eac6827..bfee083 100644 --- a/era/devel_commands.cc +++ b/era/devel_commands.cc @@ -1,12 +1,14 @@ #include "era/commands.h" using namespace base; +using namespace era; //---------------------------------------------------------------- void era::register_era_commands(base::application &app) { + app.add_cmd(command::ptr(new era_debug_cmd)); } //---------------------------------------------------------------- diff --git a/era/era_debug.cc b/era/era_debug.cc new file mode 100644 index 0000000..cf1f682 --- /dev/null +++ b/era/era_debug.cc @@ -0,0 +1,346 @@ +// Copyright (C) 2012 Red Hat, Inc. All rights reserved. +// +// This file is part of the thin-provisioning-tools source. +// +// thin-provisioning-tools is free software: you can redistribute it +// and/or modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation, either version 3 of +// the License, or (at your option) any later version. +// +// thin-provisioning-tools is distributed in the hope that it will be +// useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with thin-provisioning-tools. If not, see +// . + +#include +#include +#include +#include + +#include "base/command_interpreter.h" +#include "base/output_formatter.h" +#include "persistent-data/file_utils.h" +#include "persistent-data/data-structures/btree.h" +#include "persistent-data/data-structures/simple_traits.h" +#include "persistent-data/space-maps/disk_structures.h" +#include "era/commands.h" +#include "era/metadata.h" +#include "version.h" + +using namespace dbg; +using namespace persistent_data; +using namespace std; +using namespace era; + +//---------------------------------------------------------------- + +namespace { + class hello : public dbg::command { + virtual void exec(strings const &args, ostream &out) { + out << "Hello, world!" << endl; + } + }; + + class help : public dbg::command { + virtual void exec(strings const &args, ostream &out) { + out << "Commands:" << endl + << " superblock [block#]" << endl + << " block_node " << endl + << " bitset_block " << endl + << " era_block " << endl + << " writeset_node " << endl + << " exit" << endl; + } + }; + + class exit_handler : public dbg::command { + public: + exit_handler(command_interpreter::ptr interpreter) + : interpreter_(interpreter) { + } + + virtual void exec(strings const &args, ostream &out) { + out << "Goodbye!" << endl; + interpreter_->exit_main_loop(); + } + + command_interpreter::ptr interpreter_; + }; + + // FIXME: duplication + class uint32_show_traits : public uint32_traits { + public: + typedef uint32_traits value_trait; + + static void show(formatter::ptr f, string const &key, uint32_t const &value) { + field(*f, key, boost::lexical_cast(value)); + } + }; + + // FIXME: duplication + class uint64_show_traits : public uint64_traits { + public: + typedef uint64_traits value_trait; + + static void show(formatter::ptr f, string const &key, uint64_t const &value) { + field(*f, key, boost::lexical_cast(value)); + } + }; + + // FIXME: duplication + class sm_root_show_traits : public persistent_data::sm_disk_detail::sm_root_traits { + public: + static void show(formatter::ptr f, string const &key, + persistent_data::sm_disk_detail::sm_root const &value) { + field(*f, "nr_blocks", value.nr_blocks_); + field(*f, "nr_allocated", value.nr_allocated_); + field(*f, "bitmap_root", value.bitmap_root_); + field(*f, "ref_count_root", value.ref_count_root_); + } + }; + + // for displaying the writeset tree + class writeset_show_traits : public era::era_detail_traits { + public: + typedef era_detail_traits value_trait; + + static void show(formatter::ptr f, string const &key, era_detail const &value) { + field(*f, "nr_bits", value.nr_bits); + field(*f, "writeset_root", value.writeset_root); + } + }; + + class show_superblock : public dbg::command { + public: + explicit show_superblock(block_manager::ptr bm) + : bm_(bm) { + } + + virtual void exec(strings const &args, ostream &out) { + if (args.size() > 2) + throw runtime_error("incorrect number of arguments"); + + block_address b = era::SUPERBLOCK_LOCATION; + if (args.size() == 2) + b = boost::lexical_cast(args[1]); + era::superblock sb = read_superblock(bm_, b); + + formatter::ptr f = create_xml_formatter(); + ostringstream version; + field(*f, "csum", sb.csum); + field(*f, "flags", sb.flags.encode()); + field(*f, "blocknr", sb.blocknr); + field(*f, "uuid", sb.uuid); // FIXME: delimit, and handle non-printable chars + field(*f, "magic", sb.magic); + field(*f, "version", sb.version); + + sm_disk_detail::sm_root_disk const *d; + sm_disk_detail::sm_root v; + { + d = reinterpret_cast(sb.metadata_space_map_root); + sm_disk_detail::sm_root_traits::unpack(*d, v); + formatter::ptr f2 = create_xml_formatter(); + sm_root_show_traits::show(f2, "value", v); + f->child("metadata_space_map_root", f2); + } + + field(*f, "data_block_size", sb.data_block_size); + field(*f, "metadata_block_size", sb.metadata_block_size); + field(*f, "nr_blocks", sb.nr_blocks); + + field(*f, "current_era", sb.current_era); + { + formatter::ptr f2 = create_xml_formatter(); + writeset_show_traits::show(f2, "value", sb.current_detail); + f->child("current_writeset", f2); + } + + field(*f, "writeset_tree_root", sb.writeset_tree_root); + field(*f, "era_array_root", sb.era_array_root); + + if (sb.metadata_snap) + field(*f, "metadata_snap", *sb.metadata_snap); + + f->output(out, 0); + } + + private: + block_manager::ptr bm_; + }; + + // FIXME: duplication + template + class show_btree_node : public dbg::command { + public: + explicit show_btree_node(block_manager::ptr bm) + : bm_(bm) { + } + + virtual void exec(strings const &args, ostream &out) { + using namespace persistent_data::btree_detail; + + if (args.size() != 2) + throw runtime_error("incorrect number of arguments"); + + block_address block = boost::lexical_cast(args[1]); + block_manager::read_ref rr = bm_->read_lock(block); + + node_ref n = btree_detail::to_node(rr); + if (n.get_type() == INTERNAL) + show_node(n, out); + else { + node_ref n = btree_detail::to_node(rr); + show_node(n, out); + } + } + + private: + template + void show_node(node_ref n, ostream &out) { + formatter::ptr f = create_xml_formatter(); + + field(*f, "csum", n.get_checksum()); + field(*f, "blocknr", n.get_block_nr()); + field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); + field(*f, "nr_entries", n.get_nr_entries()); + field(*f, "max_entries", n.get_max_entries()); + field(*f, "value_size", n.get_value_size()); + + for (unsigned i = 0; i < n.get_nr_entries(); i++) { + formatter::ptr f2 = create_xml_formatter(); + field(*f2, "key", n.key_at(i)); + ST::show(f2, "value", n.value_at(i)); + f->child(boost::lexical_cast(i), f2); + } + + f->output(out, 0); + } + + block_manager::ptr bm_; + }; + + // FIXME: duplication + template + class show_array_block : public dbg::command { + typedef array_block rblock; + public: + explicit show_array_block(block_manager::ptr bm, + typename ShowTraits::ref_counter rc) + : bm_(bm), rc_(rc) { + } + + virtual void exec(strings const& args, ostream &out) { + if (args.size() != 2) + throw runtime_error("incorrect number of arguments"); + + block_address block = boost::lexical_cast(args[1]); + block_manager::read_ref rr = bm_->read_lock(block); + + rblock b(rr, rc_); + show_array_entries(b, out); + } + + private: + void show_array_entries(rblock const& b, ostream &out) { + formatter::ptr f = create_xml_formatter(); + uint32_t nr_entries = b.nr_entries(); + + field(*f, "max_entries", b.max_entries()); + field(*f, "nr_entries", nr_entries); + field(*f, "value_size", b.value_size()); + + for (unsigned i = 0; i < nr_entries; i++) { + formatter::ptr f2 = create_xml_formatter(); + ShowTraits::show(f2, "value", b.get(i)); + f->child(boost::lexical_cast(i), f2); + } + + f->output(out, 0); + } + + block_manager::ptr bm_; + typename ShowTraits::ref_counter rc_; + }; + + //-------------------------------- + + int debug(string const &path) { + using dbg::command; + + try { + block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY); + transaction_manager::ptr null_tm = open_tm(bm, era::SUPERBLOCK_LOCATION); + command_interpreter::ptr interp = create_command_interpreter(cin, cout); + interp->register_command("hello", command::ptr(new hello)); + interp->register_command("superblock", command::ptr(new show_superblock(bm))); + interp->register_command("block_node", command::ptr(new show_btree_node(bm))); + interp->register_command("bitset_block", command::ptr(new show_array_block(bm, + uint64_show_traits::ref_counter()))); + interp->register_command("era_block", command::ptr(new show_array_block(bm, + uint32_show_traits::ref_counter()))); + interp->register_command("writeset_node", command::ptr(new show_btree_node(bm))); + interp->register_command("help", command::ptr(new help)); + interp->register_command("exit", command::ptr(new exit_handler(interp))); + interp->enter_main_loop(); + + } catch (std::exception &e) { + cerr << e.what(); + return 1; + } + + return 0; + } +} + +//---------------------------------------------------------------- + +era_debug_cmd::era_debug_cmd() + : command("era_debug") +{ +} + +void +era_debug_cmd::usage(std::ostream &out) const +{ + out << "Usage: " << get_name() << " {device|file}" << endl + << "Options:" << endl + << " {-h|--help}" << endl + << " {-V|--version}" << endl; +} + +int +era_debug_cmd::run(int argc, char **argv) +{ + int c; + const char shortopts[] = "hV"; + const struct option longopts[] = { + { "help", no_argument, NULL, 'h'}, + { "version", no_argument, NULL, 'V'}, + { NULL, no_argument, NULL, 0 } + }; + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch(c) { + case 'h': + usage(cout); + return 0; + + case 'V': + cerr << THIN_PROVISIONING_TOOLS_VERSION << endl; + return 0; + } + } + + if (argc == optind) { + usage(cerr); + exit(1); + } + + return debug(argv[optind]); +} + +//---------------------------------------------------------------- From c95e31bef6e9e14000b3003cecb6579986e786e2 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 18 Feb 2021 18:20:01 +0800 Subject: [PATCH 25/33] [era_debug] Display bitset entries in run-length fashion --- era/era_debug.cc | 128 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 2 deletions(-) diff --git a/era/era_debug.cc b/era/era_debug.cc index cf1f682..af6764c 100644 --- a/era/era_debug.cc +++ b/era/era_debug.cc @@ -266,6 +266,131 @@ namespace { typename ShowTraits::ref_counter rc_; }; + // FIXME: duplication + class show_bitset_block : public dbg::command { + typedef array_block rblock; + public: + explicit show_bitset_block(block_manager::ptr bm) + : bm_(bm), + BITS_PER_ARRAY_ENTRY(64) { + } + + virtual void exec(strings const& args, ostream &out) { + if (args.size() != 2) + throw runtime_error("incorrect number of arguments"); + + block_address block = boost::lexical_cast(args[1]); + block_manager::read_ref rr = bm_->read_lock(block); + + rblock b(rr, rc_); + show_bitset_entries(b, out); + } + + private: + void show_bitset_entries(rblock const& b, ostream &out) { + formatter::ptr f = create_xml_formatter(); + uint32_t nr_entries = b.nr_entries(); + + field(*f, "max_entries", b.max_entries()); + field(*f, "nr_entries", nr_entries); + field(*f, "value_size", b.value_size()); + + uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY; + std::pair range = next_set_bits(b, 0); + for (; range.first < end_pos; range = next_set_bits(b, range.second)) { + formatter::ptr f2 = create_xml_formatter(); + field(*f2, "begin", range.first); + field(*f2, "end", range.second); + f->child("set_bits", f2); + } + + f->output(out, 0); + } + + // Returns the range of set bits, starts from the offset. + pair next_set_bits(rblock const &b, uint32_t offset) { + uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY; + uint32_t begin = find_first_set(b, offset); + + if (begin == end_pos) // not found + return make_pair(end_pos, end_pos); + + uint32_t end = find_first_unset(b, begin + 1); + return make_pair(begin, end); + } + + // Returns the position (zero-based) of the first bit set + // in the array block, starts from the offset. + // Returns the pass-the-end position if not found. + uint32_t find_first_set(rblock const &b, uint32_t offset) { + uint32_t entry = offset / BITS_PER_ARRAY_ENTRY; + uint32_t nr_entries = b.nr_entries(); + + if (entry >= nr_entries) + return entry * BITS_PER_ARRAY_ENTRY; + + uint32_t idx = offset % BITS_PER_ARRAY_ENTRY; + uint64_t v = b.get(entry++) >> idx; + while (!v && entry < nr_entries) { + v = b.get(entry++); + idx = 0; + } + + if (!v) // not found + return entry * BITS_PER_ARRAY_ENTRY; + + return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + ffsll(static_cast(v)) - 1; + } + + // Returns the position (zero-based) of the first zero bit + // in the array block, starts from the offset. + // Returns the pass-the-end position if not found. + // FIXME: improve efficiency + uint32_t find_first_unset(rblock const& b, uint32_t offset) { + uint32_t entry = offset / BITS_PER_ARRAY_ENTRY; + uint32_t nr_entries = b.nr_entries(); + + if (entry >= nr_entries) + return entry * BITS_PER_ARRAY_ENTRY; + + uint32_t idx = offset % BITS_PER_ARRAY_ENTRY; + uint64_t v = b.get(entry++); + while (all_bits_set(v, idx) && entry < nr_entries) { + v = b.get(entry++); + idx = 0; + } + + if (all_bits_set(v, idx)) // not found + return entry * BITS_PER_ARRAY_ENTRY; + + return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + count_leading_bits(v, idx); + } + + // Returns true if all the bits beyond the position are set. + bool all_bits_set(uint64_t v, uint32_t offset) { + return (v >> offset) == (numeric_limits::max() >> offset); + } + + // Counts the number of leading 1's in the given value, starts from the offset + // FIXME: improve efficiency + uint32_t count_leading_bits(uint64_t v, uint32_t offset) { + uint32_t count = 0; + + v >>= offset; + while (v & 0x1) { + v >>= 1; + count++; + } + + return count; + } + + block_manager::ptr bm_; + uint64_traits::ref_counter rc_; + + const uint32_t BITS_PER_ARRAY_ENTRY; + }; + //-------------------------------- int debug(string const &path) { @@ -278,8 +403,7 @@ namespace { interp->register_command("hello", command::ptr(new hello)); interp->register_command("superblock", command::ptr(new show_superblock(bm))); interp->register_command("block_node", command::ptr(new show_btree_node(bm))); - interp->register_command("bitset_block", command::ptr(new show_array_block(bm, - uint64_show_traits::ref_counter()))); + interp->register_command("bitset_block", command::ptr(new show_bitset_block(bm))); interp->register_command("era_block", command::ptr(new show_array_block(bm, uint32_show_traits::ref_counter()))); interp->register_command("writeset_node", command::ptr(new show_btree_node(bm))); From a81cef44674dd71e532fc3a11d6a232257a0059c Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 19 Feb 2021 14:06:21 +0800 Subject: [PATCH 26/33] [dbg] Pull out common code into dbg-lib - Modularize common routines - Extract the block_dumper interface for displaying blocks - Remove inheritance from show_traits --- Makefile.in | 9 +- caching/cache_debug.cc | 167 ++---------- dbg-lib/array_block_dumper.h | 53 ++++ dbg-lib/bitset_block_dumper.cc | 137 ++++++++++ dbg-lib/bitset_block_dumper.h | 14 + dbg-lib/block_dumper.h | 21 ++ dbg-lib/btree_node_dumper.h | 59 +++++ {base => dbg-lib}/command_interpreter.cc | 2 +- {base => dbg-lib}/command_interpreter.h | 0 dbg-lib/commands.cc | 79 ++++++ dbg-lib/commands.h | 24 ++ dbg-lib/index_block_dumper.cc | 51 ++++ dbg-lib/index_block_dumper.h | 10 + {base => dbg-lib}/output_formatter.cc | 2 +- {base => dbg-lib}/output_formatter.h | 0 dbg-lib/simple_show_traits.cc | 19 ++ dbg-lib/simple_show_traits.h | 27 ++ dbg-lib/sm_show_traits.cc | 27 ++ dbg-lib/sm_show_traits.h | 29 +++ era/era_debug.cc | 314 +++-------------------- thin-provisioning/thin_debug.cc | 183 ++----------- 21 files changed, 640 insertions(+), 587 deletions(-) create mode 100644 dbg-lib/array_block_dumper.h create mode 100644 dbg-lib/bitset_block_dumper.cc create mode 100644 dbg-lib/bitset_block_dumper.h create mode 100644 dbg-lib/block_dumper.h create mode 100644 dbg-lib/btree_node_dumper.h rename {base => dbg-lib}/command_interpreter.cc (97%) rename {base => dbg-lib}/command_interpreter.h (100%) create mode 100644 dbg-lib/commands.cc create mode 100644 dbg-lib/commands.h create mode 100644 dbg-lib/index_block_dumper.cc create mode 100644 dbg-lib/index_block_dumper.h rename {base => dbg-lib}/output_formatter.cc (98%) rename {base => dbg-lib}/output_formatter.h (100%) create mode 100644 dbg-lib/simple_show_traits.cc create mode 100644 dbg-lib/simple_show_traits.h create mode 100644 dbg-lib/sm_show_traits.cc create mode 100644 dbg-lib/sm_show_traits.h diff --git a/Makefile.in b/Makefile.in index 0392d8e..11dbf75 100644 --- a/Makefile.in +++ b/Makefile.in @@ -126,10 +126,15 @@ TOOLS_SOURCE=\ thin-provisioning/thin_trim.cc DEVTOOLS_SOURCE=\ - base/command_interpreter.cc \ - base/output_formatter.cc \ caching/cache_debug.cc \ caching/devel_commands.cc \ + dbg-lib/bitset_block_dumper.cc \ + dbg-lib/command_interpreter.cc \ + dbg-lib/commands.cc \ + dbg-lib/index_block_dumper.cc \ + dbg-lib/output_formatter.cc \ + dbg-lib/simple_show_traits.cc \ + dbg-lib/sm_show_traits.cc \ era/devel_commands.cc \ era/era_debug.cc \ thin-provisioning/damage_generator.cc \ diff --git a/caching/cache_debug.cc b/caching/cache_debug.cc index 4f0c68d..4f9e033 100644 --- a/caching/cache_debug.cc +++ b/caching/cache_debug.cc @@ -21,11 +21,13 @@ #include #include -#include "base/command_interpreter.h" -#include "base/output_formatter.h" +#include "dbg-lib/array_block_dumper.h" +#include "dbg-lib/btree_node_dumper.h" +#include "dbg-lib/command_interpreter.h" +#include "dbg-lib/commands.h" +#include "dbg-lib/output_formatter.h" +#include "dbg-lib/sm_show_traits.h" #include "persistent-data/file_utils.h" -#include "persistent-data/data-structures/btree.h" -#include "persistent-data/data-structures/simple_traits.h" #include "persistent-data/space-maps/disk_structures.h" #include "caching/commands.h" #include "caching/metadata.h" @@ -39,12 +41,6 @@ using namespace caching; //---------------------------------------------------------------- namespace { - class hello : public dbg::command { - virtual void exec(strings const &args, ostream &out) { - out << "Hello, world!" << endl; - } - }; - class help : public dbg::command { virtual void exec(strings const &args, ostream &out) { out << "Commands:" << endl @@ -55,31 +51,6 @@ namespace { } }; - class exit_handler : public dbg::command { - public: - exit_handler(command_interpreter::ptr interpreter) - : interpreter_(interpreter) { - } - - virtual void exec(strings const &args, ostream &out) { - out << "Goodbye!" << endl; - interpreter_->exit_main_loop(); - } - - command_interpreter::ptr interpreter_; - }; - - class sm_root_show_traits : public persistent_data::sm_disk_detail::sm_root_traits { - public: - static void show(formatter::ptr f, string const &key, - persistent_data::sm_disk_detail::sm_root const &value) { - field(*f, "nr_blocks", value.nr_blocks_); - field(*f, "nr_allocated", value.nr_allocated_); - field(*f, "bitmap_root", value.bitmap_root_); - field(*f, "ref_count_root", value.ref_count_root_); - } - }; - class show_superblock : public dbg::command { public: explicit show_superblock(block_manager::ptr bm) @@ -145,16 +116,6 @@ namespace { block_manager::ptr bm_; }; - // FIXME: duplication - class uint64_show_traits : public uint64_traits { - public: - typedef uint64_traits value_trait; - - static void show(formatter::ptr f, string const &key, uint64_t const &value) { - field(*f, key, boost::lexical_cast(value)); - } - }; - class mapping_show_traits : public caching::mapping_traits { public: typedef mapping_traits value_trait; @@ -165,114 +126,34 @@ namespace { } }; - // FIXME: duplication - template - class show_btree_node : public dbg::command { - public: - explicit show_btree_node(block_manager::ptr bm) - : bm_(bm) { - } - - virtual void exec(strings const &args, ostream &out) { - using namespace persistent_data::btree_detail; - - if (args.size() != 2) - throw runtime_error("incorrect number of arguments"); - - block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = bm_->read_lock(block); - - node_ref n = btree_detail::to_node(rr); - if (n.get_type() == INTERNAL) - show_node(n, out); - else { - node_ref n = btree_detail::to_node(rr); - show_node(n, out); - } - } - - private: - template - void show_node(node_ref n, ostream &out) { - formatter::ptr f = create_xml_formatter(); - - field(*f, "csum", n.get_checksum()); - field(*f, "blocknr", n.get_block_nr()); - field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); - field(*f, "nr_entries", n.get_nr_entries()); - field(*f, "max_entries", n.get_max_entries()); - field(*f, "value_size", n.get_value_size()); - - for (unsigned i = 0; i < n.get_nr_entries(); i++) { - formatter::ptr f2 = create_xml_formatter(); - field(*f2, "key", n.key_at(i)); - ST::show(f2, "value", n.value_at(i)); - f->child(boost::lexical_cast(i), f2); - } - - f->output(out, 0); - } - - block_manager::ptr bm_; - }; - - template - class show_array_block : public dbg::command { - typedef array_block rblock; - public: - explicit show_array_block(block_manager::ptr bm, - typename ShowTraits::ref_counter rc) - : bm_(bm), rc_(rc) { - } - - virtual void exec(strings const& args, ostream &out) { - if (args.size() != 2) - throw runtime_error("incorrect number of arguments"); - - block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = bm_->read_lock(block); - - rblock b(rr, rc_); - show_array_entries(b, out); - } - - private: - void show_array_entries(rblock const& b, ostream &out) { - formatter::ptr f = create_xml_formatter(); - uint32_t nr_entries = b.nr_entries(); - - field(*f, "max_entries", b.max_entries()); - field(*f, "nr_entries", nr_entries); - field(*f, "value_size", b.value_size()); - - for (unsigned i = 0; i < nr_entries; i++) { - formatter::ptr f2 = create_xml_formatter(); - ShowTraits::show(f2, "value", b.get(i)); - f->child(boost::lexical_cast(i), f2); - } - - f->output(out, 0); - } - - block_manager::ptr bm_; - typename ShowTraits::ref_counter rc_; - }; - //-------------------------------- + template + dbg::command::ptr + create_btree_node_handler(block_manager::ptr bm) { + return create_block_handler(bm, create_btree_node_dumper()); + } + + template + dbg::command::ptr + create_array_block_handler(block_manager::ptr bm, + typename ShowTraits::value_trait::ref_counter rc) { + return create_block_handler(bm, create_array_block_dumper(rc)); + } + int debug(string const &path) { using dbg::command; try { block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY); command_interpreter::ptr interp = create_command_interpreter(cin, cout); - interp->register_command("hello", command::ptr(new hello)); + interp->register_command("hello", create_hello_handler()); interp->register_command("superblock", command::ptr(new show_superblock(bm))); - interp->register_command("block_node", command::ptr(new show_btree_node(bm))); - interp->register_command("mapping_block", command::ptr(new show_array_block(bm, - mapping_show_traits::ref_counter()))); + interp->register_command("block_node", create_btree_node_handler(bm)); + interp->register_command("mapping_block", create_array_block_handler(bm, + mapping_traits::ref_counter())); interp->register_command("help", command::ptr(new help)); - interp->register_command("exit", command::ptr(new exit_handler(interp))); + interp->register_command("exit", create_exit_handler(interp)); interp->enter_main_loop(); } catch (std::exception &e) { diff --git a/dbg-lib/array_block_dumper.h b/dbg-lib/array_block_dumper.h new file mode 100644 index 0000000..b63fa3a --- /dev/null +++ b/dbg-lib/array_block_dumper.h @@ -0,0 +1,53 @@ +#include "dbg-lib/block_dumper.h" +#include "dbg-lib/output_formatter.h" +#include "persistent-data/data-structures/array_block.h" + +//---------------------------------------------------------------- + +namespace dbg { + using persistent_data::block_manager; + using persistent_data::array_block; + + template + class array_block_dumper : public block_dumper { + public: + array_block_dumper(typename ShowTraits::value_trait::ref_counter rc) + : rc_(rc) { + } + + virtual void show(block_manager::read_ref &rr, std::ostream &out) { + rblock b(rr, rc_); + show_array_entries(b, out); + } + + private: + typedef array_block rblock; + + void show_array_entries(rblock const& b, std::ostream &out) { + formatter::ptr f = create_xml_formatter(); + uint32_t nr_entries = b.nr_entries(); + + field(*f, "max_entries", b.max_entries()); + field(*f, "nr_entries", nr_entries); + field(*f, "value_size", b.value_size()); + + for (unsigned i = 0; i < nr_entries; i++) { + formatter::ptr f2 = create_xml_formatter(); + ShowTraits::show(f2, "value", b.get(i)); + f->child(boost::lexical_cast(i), f2); + } + + f->output(out, 0); + } + + typename ShowTraits::value_trait::ref_counter rc_; + }; + + template + block_dumper::ptr + create_array_block_dumper(typename ShowTraits::value_trait::ref_counter rc) { + return block_dumper::ptr(new array_block_dumper(rc)); + } +} + +//---------------------------------------------------------------- diff --git a/dbg-lib/bitset_block_dumper.cc b/dbg-lib/bitset_block_dumper.cc new file mode 100644 index 0000000..8213ee3 --- /dev/null +++ b/dbg-lib/bitset_block_dumper.cc @@ -0,0 +1,137 @@ +#include "dbg-lib/bitset_block_dumper.h" +#include "dbg-lib/output_formatter.h" +#include "persistent-data/data-structures/array_block.h" +#include "persistent-data/data-structures/simple_traits.h" + +using namespace dbg; +using namespace persistent_data; + +//---------------------------------------------------------------- + +namespace { + class bitset_block_dumper : public dbg::block_dumper { + typedef array_block rblock; + public: + explicit bitset_block_dumper() + : BITS_PER_ARRAY_ENTRY(64) { + } + + virtual void show(block_manager::read_ref &rr, ostream &out) { + rblock b(rr, rc_); + show_bitset_entries(b, out); + } + + private: + void show_bitset_entries(rblock const& b, ostream &out) { + formatter::ptr f = create_xml_formatter(); + uint32_t nr_entries = b.nr_entries(); + + field(*f, "max_entries", b.max_entries()); + field(*f, "nr_entries", nr_entries); + field(*f, "value_size", b.value_size()); + + uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY; + std::pair range = next_set_bits(b, 0); + for (; range.first < end_pos; range = next_set_bits(b, range.second)) { + formatter::ptr f2 = create_xml_formatter(); + field(*f2, "begin", range.first); + field(*f2, "end", range.second); + f->child("set_bits", f2); + } + + f->output(out, 0); + } + + // Returns the range of set bits, starts from the offset. + pair next_set_bits(rblock const &b, uint32_t offset) { + uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY; + uint32_t begin = find_first_set(b, offset); + + if (begin == end_pos) // not found + return make_pair(end_pos, end_pos); + + uint32_t end = find_first_unset(b, begin + 1); + return make_pair(begin, end); + } + + // Returns the position (zero-based) of the first bit set + // in the array block, starts from the offset. + // Returns the pass-the-end position if not found. + uint32_t find_first_set(rblock const &b, uint32_t offset) { + uint32_t entry = offset / BITS_PER_ARRAY_ENTRY; + uint32_t nr_entries = b.nr_entries(); + + if (entry >= nr_entries) + return entry * BITS_PER_ARRAY_ENTRY; + + uint32_t idx = offset % BITS_PER_ARRAY_ENTRY; + uint64_t v = b.get(entry++) >> idx; + while (!v && entry < nr_entries) { + v = b.get(entry++); + idx = 0; + } + + if (!v) // not found + return entry * BITS_PER_ARRAY_ENTRY; + + return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + ffsll(static_cast(v)) - 1; + } + + // Returns the position (zero-based) of the first zero bit + // in the array block, starts from the offset. + // Returns the pass-the-end position if not found. + // FIXME: improve efficiency + uint32_t find_first_unset(rblock const& b, uint32_t offset) { + uint32_t entry = offset / BITS_PER_ARRAY_ENTRY; + uint32_t nr_entries = b.nr_entries(); + + if (entry >= nr_entries) + return entry * BITS_PER_ARRAY_ENTRY; + + uint32_t idx = offset % BITS_PER_ARRAY_ENTRY; + uint64_t v = b.get(entry++); + while (all_bits_set(v, idx) && entry < nr_entries) { + v = b.get(entry++); + idx = 0; + } + + if (all_bits_set(v, idx)) // not found + return entry * BITS_PER_ARRAY_ENTRY; + + return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + count_leading_bits(v, idx); + } + + // Returns true if all the bits beyond the position are set. + bool all_bits_set(uint64_t v, uint32_t offset) { + return (v >> offset) == (numeric_limits::max() >> offset); + } + + // Counts the number of leading 1's in the given value, starts from the offset + // FIXME: improve efficiency + uint32_t count_leading_bits(uint64_t v, uint32_t offset) { + uint32_t count = 0; + + v >>= offset; + while (v & 0x1) { + v >>= 1; + count++; + } + + return count; + } + + block_manager::ptr bm_; + uint64_traits::ref_counter rc_; + + const uint32_t BITS_PER_ARRAY_ENTRY; + }; +} + +//---------------------------------------------------------------- + +block_dumper::ptr +dbg::create_bitset_block_dumper() { + return block_dumper::ptr(new bitset_block_dumper()); +} + +//---------------------------------------------------------------- diff --git a/dbg-lib/bitset_block_dumper.h b/dbg-lib/bitset_block_dumper.h new file mode 100644 index 0000000..5c1a816 --- /dev/null +++ b/dbg-lib/bitset_block_dumper.h @@ -0,0 +1,14 @@ +#ifndef DBG_BITSET_BLOCK_DUMPER +#define DBG_BITSET_BLOCK_DUMPER + +#include "dbg-lib/block_dumper.h" + +//---------------------------------------------------------------- + +namespace dbg { + block_dumper::ptr create_bitset_block_dumper(); +} + +//---------------------------------------------------------------- + +#endif diff --git a/dbg-lib/block_dumper.h b/dbg-lib/block_dumper.h new file mode 100644 index 0000000..6d4461f --- /dev/null +++ b/dbg-lib/block_dumper.h @@ -0,0 +1,21 @@ +#ifndef DBG_BLOCK_DUMPER_H +#define DBG_BLOCK_DUMPER_H + +#include "persistent-data/block.h" + +//---------------------------------------------------------------- + +namespace dbg { + class block_dumper { + public: + typedef std::shared_ptr ptr; + + // pass the read_ref by reference since the caller already held the ref-count + virtual void show(persistent_data::block_manager::read_ref &rr, + std::ostream &out) = 0; + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/dbg-lib/btree_node_dumper.h b/dbg-lib/btree_node_dumper.h new file mode 100644 index 0000000..f92bce2 --- /dev/null +++ b/dbg-lib/btree_node_dumper.h @@ -0,0 +1,59 @@ +#ifndef DBG_BTREE_NODE_DUMPER +#define DBG_BTREE_NODE_DUMPER + +#include "dbg-lib/block_dumper.h" +#include "dbg-lib/simple_show_traits.h" +#include "persistent-data/data-structures/btree.h" + +//---------------------------------------------------------------- + +namespace dbg { + using persistent_data::block_manager; + using persistent_data::btree_detail::node_ref; + + template + class btree_node_dumper : public block_dumper { + public: + virtual void show(block_manager::read_ref &rr, std::ostream &out) { + node_ref n = btree_detail::to_node(rr); + if (n.get_type() == INTERNAL) + show_node(n, out); + else { + node_ref n = btree_detail::to_node(rr); + show_node(n, out); + } + } + + private: + template + void show_node(node_ref n, std::ostream &out) { + formatter::ptr f = create_xml_formatter(); + + field(*f, "csum", n.get_checksum()); + field(*f, "blocknr", n.get_block_nr()); + field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); + field(*f, "nr_entries", n.get_nr_entries()); + field(*f, "max_entries", n.get_max_entries()); + field(*f, "value_size", n.get_value_size()); + + for (unsigned i = 0; i < n.get_nr_entries(); i++) { + formatter::ptr f2 = create_xml_formatter(); + field(*f2, "key", n.key_at(i)); + ST::show(f2, "value", n.value_at(i)); + f->child(boost::lexical_cast(i), f2); + } + + f->output(out, 0); + } + }; + + template + block_dumper::ptr + create_btree_node_dumper() { + return block_dumper::ptr(new btree_node_dumper()); + } +} + +//---------------------------------------------------------------- + +#endif diff --git a/base/command_interpreter.cc b/dbg-lib/command_interpreter.cc similarity index 97% rename from base/command_interpreter.cc rename to dbg-lib/command_interpreter.cc index ce7bd97..158831f 100644 --- a/base/command_interpreter.cc +++ b/dbg-lib/command_interpreter.cc @@ -1,4 +1,4 @@ -#include "base/command_interpreter.h" +#include "dbg-lib/command_interpreter.h" using namespace dbg; using namespace std; diff --git a/base/command_interpreter.h b/dbg-lib/command_interpreter.h similarity index 100% rename from base/command_interpreter.h rename to dbg-lib/command_interpreter.h diff --git a/dbg-lib/commands.cc b/dbg-lib/commands.cc new file mode 100644 index 0000000..409cf6f --- /dev/null +++ b/dbg-lib/commands.cc @@ -0,0 +1,79 @@ +#include "dbg-lib/commands.h" +#include "persistent-data/block.h" + +#include +#include + +using namespace dbg; +using namespace persistent_data; +using namespace std; + +//---------------------------------------------------------------- + +namespace { + class hello : public dbg::command { + virtual void exec(dbg::strings const &args, ostream &out) { + out << "Hello, world!" << endl; + } + }; + + class exit_handler : public dbg::command { + public: + exit_handler(command_interpreter::ptr interpreter) + : interpreter_(interpreter) { + } + + virtual void exec(dbg::strings const &args, ostream &out) { + out << "Goodbye!" << endl; + interpreter_->exit_main_loop(); + } + + private: + command_interpreter::ptr interpreter_; + }; + + class block_handler : public dbg::command { + public: + block_handler(block_manager::ptr bm, block_dumper::ptr dumper) + : bm_(bm), dumper_(dumper) { + } + + virtual void exec(dbg::strings const &args, ostream &out) { + if (args.size() != 2) + throw runtime_error("incorrect number of arguments"); + + block_address block = boost::lexical_cast(args[1]); + + // no checksum validation for debugging purpose + block_manager::read_ref rr = bm_->read_lock(block); + + dumper_->show(rr, out); + } + + private: + block_manager::ptr bm_; + block_dumper::ptr dumper_; + }; +} + +//---------------------------------------------------------------- + +command::ptr +dbg::create_hello_handler() +{ + return command::ptr(new hello()); +} + +command::ptr +dbg::create_exit_handler(command_interpreter::ptr interp) +{ + return command::ptr(new exit_handler(interp)); +} + +command::ptr +dbg::create_block_handler(block_manager::ptr bm, block_dumper::ptr dumper) +{ + return command::ptr(new block_handler(bm, dumper)); +} + +//---------------------------------------------------------------- diff --git a/dbg-lib/commands.h b/dbg-lib/commands.h new file mode 100644 index 0000000..76094ad --- /dev/null +++ b/dbg-lib/commands.h @@ -0,0 +1,24 @@ +#ifndef DBG_COMMANDS_H +#define DBG_COMMANDS_H + +#include "dbg-lib/command_interpreter.h" +#include "dbg-lib/block_dumper.h" +#include "persistent-data/block.h" + +//---------------------------------------------------------------- + +namespace dbg { + dbg::command::ptr + create_hello_handler(); + + dbg::command::ptr + create_exit_handler(dbg::command_interpreter::ptr interp); + + dbg::command::ptr + create_block_handler(persistent_data::block_manager::ptr bm, + dbg::block_dumper::ptr dumper); +} + +//---------------------------------------------------------------- + +#endif diff --git a/dbg-lib/index_block_dumper.cc b/dbg-lib/index_block_dumper.cc new file mode 100644 index 0000000..7457d65 --- /dev/null +++ b/dbg-lib/index_block_dumper.cc @@ -0,0 +1,51 @@ +#include "dbg-lib/index_block_dumper.h" +#include "dbg-lib/output_formatter.h" +#include "dbg-lib/sm_show_traits.h" +#include "persistent-data/space-maps/disk_structures.h" + +using namespace dbg; + +//---------------------------------------------------------------- + +namespace { + using persistent_data::block_manager; + + class index_block_dumper : public dbg::block_dumper { + public: + virtual void show(block_manager::read_ref &rr, std::ostream &out) { + sm_disk_detail::metadata_index const *mdi = + reinterpret_cast(rr.data()); + show_metadata_index(mdi, sm_disk_detail::MAX_METADATA_BITMAPS, out); + } + + private: + void show_metadata_index(sm_disk_detail::metadata_index const *mdi, unsigned nr_indexes, std::ostream &out) { + formatter::ptr f = create_xml_formatter(); + field(*f, "csum", to_cpu(mdi->csum_)); + field(*f, "padding", to_cpu(mdi->padding_)); + field(*f, "blocknr", to_cpu(mdi->blocknr_)); + + sm_disk_detail::index_entry ie; + for (unsigned i = 0; i < nr_indexes; i++) { + sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), ie); + + if (!ie.blocknr_ && !ie.nr_free_ && !ie.none_free_before_) + continue; + + formatter::ptr f2 = create_xml_formatter(); + index_entry_show_traits::show(f2, "value", ie); + f->child(boost::lexical_cast(i), f2); + } + f->output(out, 0); + } + }; +} + +//---------------------------------------------------------------- + +block_dumper::ptr +dbg::create_index_block_dumper() { + return block_dumper::ptr(new index_block_dumper()); +} + +//---------------------------------------------------------------- diff --git a/dbg-lib/index_block_dumper.h b/dbg-lib/index_block_dumper.h new file mode 100644 index 0000000..b1ce10a --- /dev/null +++ b/dbg-lib/index_block_dumper.h @@ -0,0 +1,10 @@ +#include "dbg-lib/block_dumper.h" + +//---------------------------------------------------------------- + +namespace dbg { + block_dumper::ptr + create_index_block_dumper(); +} + +//---------------------------------------------------------------- diff --git a/base/output_formatter.cc b/dbg-lib/output_formatter.cc similarity index 98% rename from base/output_formatter.cc rename to dbg-lib/output_formatter.cc index c62058f..95d8bb8 100644 --- a/base/output_formatter.cc +++ b/dbg-lib/output_formatter.cc @@ -1,6 +1,6 @@ #include -#include "base/output_formatter.h" +#include "dbg-lib/output_formatter.h" using namespace dbg; using namespace std; diff --git a/base/output_formatter.h b/dbg-lib/output_formatter.h similarity index 100% rename from base/output_formatter.h rename to dbg-lib/output_formatter.h diff --git a/dbg-lib/simple_show_traits.cc b/dbg-lib/simple_show_traits.cc new file mode 100644 index 0000000..14b12c5 --- /dev/null +++ b/dbg-lib/simple_show_traits.cc @@ -0,0 +1,19 @@ +#include "dbg-lib/simple_show_traits.h" + +using namespace dbg; + +//---------------------------------------------------------------- + +void +uint32_show_traits::show(formatter::ptr f, string const &key, uint32_t const &value) +{ + field(*f, key, boost::lexical_cast(value)); +} + +void +uint64_show_traits::show(formatter::ptr f, string const &key, uint64_t const &value) +{ + field(*f, key, boost::lexical_cast(value)); +} + +//---------------------------------------------------------------- diff --git a/dbg-lib/simple_show_traits.h b/dbg-lib/simple_show_traits.h new file mode 100644 index 0000000..a21c8ea --- /dev/null +++ b/dbg-lib/simple_show_traits.h @@ -0,0 +1,27 @@ +#ifndef DBG_SIMPLE_SHOW_TRAITS_H +#define DBG_SIMPLE_SHOW_TRAITS_H + +#include "dbg-lib/output_formatter.h" +#include "persistent-data/data-structures/simple_traits.h" + +//---------------------------------------------------------------- + +namespace dbg { + class uint32_show_traits { + public: + typedef persistent_data::uint32_traits value_trait; + + static void show(dbg::formatter::ptr f, std::string const &key, uint32_t const &value); + }; + + class uint64_show_traits { + public: + typedef persistent_data::uint64_traits value_trait; + + static void show(dbg::formatter::ptr f, std::string const &key, uint64_t const &value); + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/dbg-lib/sm_show_traits.cc b/dbg-lib/sm_show_traits.cc new file mode 100644 index 0000000..4f14bc9 --- /dev/null +++ b/dbg-lib/sm_show_traits.cc @@ -0,0 +1,27 @@ +#include "dbg-lib/sm_show_traits.h" + +using namespace dbg; +using namespace std; + +//---------------------------------------------------------------- + +void +index_entry_show_traits::show(formatter::ptr f, string const &key, + persistent_data::sm_disk_detail::index_entry const &value) +{ + field(*f, "blocknr", value.blocknr_); + field(*f, "nr_free", value.nr_free_); + field(*f, "none_free_before", value.none_free_before_); +} + +void +sm_root_show_traits::show(formatter::ptr f, string const &key, + persistent_data::sm_disk_detail::sm_root const &value) +{ + field(*f, "nr_blocks", value.nr_blocks_); + field(*f, "nr_allocated", value.nr_allocated_); + field(*f, "bitmap_root", value.bitmap_root_); + field(*f, "ref_count_root", value.ref_count_root_); +} + +//---------------------------------------------------------------- diff --git a/dbg-lib/sm_show_traits.h b/dbg-lib/sm_show_traits.h new file mode 100644 index 0000000..bf85fcf --- /dev/null +++ b/dbg-lib/sm_show_traits.h @@ -0,0 +1,29 @@ +#ifndef DBG_SM_SHOW_TRAITS_H +#define DBG_SM_SHOW_TRAITS_H + +#include "dbg-lib/output_formatter.h" +#include "persistent-data/space-maps/disk_structures.h" + +//---------------------------------------------------------------- + +namespace dbg { + class index_entry_show_traits { + public: + typedef persistent_data::sm_disk_detail::index_entry_traits value_trait; + + static void show(dbg::formatter::ptr f, std::string const &key, + persistent_data::sm_disk_detail::index_entry const &value); + }; + + class sm_root_show_traits { + public: + typedef persistent_data::sm_disk_detail::sm_root_traits value_trait; + + static void show(dbg::formatter::ptr f, std::string const &key, + persistent_data::sm_disk_detail::sm_root const &value); + }; +} + +//---------------------------------------------------------------- + +#endif diff --git a/era/era_debug.cc b/era/era_debug.cc index af6764c..5470b4c 100644 --- a/era/era_debug.cc +++ b/era/era_debug.cc @@ -21,11 +21,14 @@ #include #include -#include "base/command_interpreter.h" -#include "base/output_formatter.h" +#include "dbg-lib/array_block_dumper.h" +#include "dbg-lib/btree_node_dumper.h" +#include "dbg-lib/bitset_block_dumper.h" +#include "dbg-lib/command_interpreter.h" +#include "dbg-lib/commands.h" +#include "dbg-lib/output_formatter.h" +#include "dbg-lib/sm_show_traits.h" #include "persistent-data/file_utils.h" -#include "persistent-data/data-structures/btree.h" -#include "persistent-data/data-structures/simple_traits.h" #include "persistent-data/space-maps/disk_structures.h" #include "era/commands.h" #include "era/metadata.h" @@ -39,12 +42,6 @@ using namespace era; //---------------------------------------------------------------- namespace { - class hello : public dbg::command { - virtual void exec(strings const &args, ostream &out) { - out << "Hello, world!" << endl; - } - }; - class help : public dbg::command { virtual void exec(strings const &args, ostream &out) { out << "Commands:" << endl @@ -57,52 +54,6 @@ namespace { } }; - class exit_handler : public dbg::command { - public: - exit_handler(command_interpreter::ptr interpreter) - : interpreter_(interpreter) { - } - - virtual void exec(strings const &args, ostream &out) { - out << "Goodbye!" << endl; - interpreter_->exit_main_loop(); - } - - command_interpreter::ptr interpreter_; - }; - - // FIXME: duplication - class uint32_show_traits : public uint32_traits { - public: - typedef uint32_traits value_trait; - - static void show(formatter::ptr f, string const &key, uint32_t const &value) { - field(*f, key, boost::lexical_cast(value)); - } - }; - - // FIXME: duplication - class uint64_show_traits : public uint64_traits { - public: - typedef uint64_traits value_trait; - - static void show(formatter::ptr f, string const &key, uint64_t const &value) { - field(*f, key, boost::lexical_cast(value)); - } - }; - - // FIXME: duplication - class sm_root_show_traits : public persistent_data::sm_disk_detail::sm_root_traits { - public: - static void show(formatter::ptr f, string const &key, - persistent_data::sm_disk_detail::sm_root const &value) { - field(*f, "nr_blocks", value.nr_blocks_); - field(*f, "nr_allocated", value.nr_allocated_); - field(*f, "bitmap_root", value.bitmap_root_); - field(*f, "ref_count_root", value.ref_count_root_); - } - }; - // for displaying the writeset tree class writeset_show_traits : public era::era_detail_traits { public: @@ -172,227 +123,26 @@ namespace { block_manager::ptr bm_; }; - // FIXME: duplication - template - class show_btree_node : public dbg::command { - public: - explicit show_btree_node(block_manager::ptr bm) - : bm_(bm) { - } - - virtual void exec(strings const &args, ostream &out) { - using namespace persistent_data::btree_detail; - - if (args.size() != 2) - throw runtime_error("incorrect number of arguments"); - - block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = bm_->read_lock(block); - - node_ref n = btree_detail::to_node(rr); - if (n.get_type() == INTERNAL) - show_node(n, out); - else { - node_ref n = btree_detail::to_node(rr); - show_node(n, out); - } - } - - private: - template - void show_node(node_ref n, ostream &out) { - formatter::ptr f = create_xml_formatter(); - - field(*f, "csum", n.get_checksum()); - field(*f, "blocknr", n.get_block_nr()); - field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); - field(*f, "nr_entries", n.get_nr_entries()); - field(*f, "max_entries", n.get_max_entries()); - field(*f, "value_size", n.get_value_size()); - - for (unsigned i = 0; i < n.get_nr_entries(); i++) { - formatter::ptr f2 = create_xml_formatter(); - field(*f2, "key", n.key_at(i)); - ST::show(f2, "value", n.value_at(i)); - f->child(boost::lexical_cast(i), f2); - } - - f->output(out, 0); - } - - block_manager::ptr bm_; - }; - - // FIXME: duplication - template - class show_array_block : public dbg::command { - typedef array_block rblock; - public: - explicit show_array_block(block_manager::ptr bm, - typename ShowTraits::ref_counter rc) - : bm_(bm), rc_(rc) { - } - - virtual void exec(strings const& args, ostream &out) { - if (args.size() != 2) - throw runtime_error("incorrect number of arguments"); - - block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = bm_->read_lock(block); - - rblock b(rr, rc_); - show_array_entries(b, out); - } - - private: - void show_array_entries(rblock const& b, ostream &out) { - formatter::ptr f = create_xml_formatter(); - uint32_t nr_entries = b.nr_entries(); - - field(*f, "max_entries", b.max_entries()); - field(*f, "nr_entries", nr_entries); - field(*f, "value_size", b.value_size()); - - for (unsigned i = 0; i < nr_entries; i++) { - formatter::ptr f2 = create_xml_formatter(); - ShowTraits::show(f2, "value", b.get(i)); - f->child(boost::lexical_cast(i), f2); - } - - f->output(out, 0); - } - - block_manager::ptr bm_; - typename ShowTraits::ref_counter rc_; - }; - - // FIXME: duplication - class show_bitset_block : public dbg::command { - typedef array_block rblock; - public: - explicit show_bitset_block(block_manager::ptr bm) - : bm_(bm), - BITS_PER_ARRAY_ENTRY(64) { - } - - virtual void exec(strings const& args, ostream &out) { - if (args.size() != 2) - throw runtime_error("incorrect number of arguments"); - - block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = bm_->read_lock(block); - - rblock b(rr, rc_); - show_bitset_entries(b, out); - } - - private: - void show_bitset_entries(rblock const& b, ostream &out) { - formatter::ptr f = create_xml_formatter(); - uint32_t nr_entries = b.nr_entries(); - - field(*f, "max_entries", b.max_entries()); - field(*f, "nr_entries", nr_entries); - field(*f, "value_size", b.value_size()); - - uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY; - std::pair range = next_set_bits(b, 0); - for (; range.first < end_pos; range = next_set_bits(b, range.second)) { - formatter::ptr f2 = create_xml_formatter(); - field(*f2, "begin", range.first); - field(*f2, "end", range.second); - f->child("set_bits", f2); - } - - f->output(out, 0); - } - - // Returns the range of set bits, starts from the offset. - pair next_set_bits(rblock const &b, uint32_t offset) { - uint32_t end_pos = b.nr_entries() * BITS_PER_ARRAY_ENTRY; - uint32_t begin = find_first_set(b, offset); - - if (begin == end_pos) // not found - return make_pair(end_pos, end_pos); - - uint32_t end = find_first_unset(b, begin + 1); - return make_pair(begin, end); - } - - // Returns the position (zero-based) of the first bit set - // in the array block, starts from the offset. - // Returns the pass-the-end position if not found. - uint32_t find_first_set(rblock const &b, uint32_t offset) { - uint32_t entry = offset / BITS_PER_ARRAY_ENTRY; - uint32_t nr_entries = b.nr_entries(); - - if (entry >= nr_entries) - return entry * BITS_PER_ARRAY_ENTRY; - - uint32_t idx = offset % BITS_PER_ARRAY_ENTRY; - uint64_t v = b.get(entry++) >> idx; - while (!v && entry < nr_entries) { - v = b.get(entry++); - idx = 0; - } - - if (!v) // not found - return entry * BITS_PER_ARRAY_ENTRY; - - return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + ffsll(static_cast(v)) - 1; - } - - // Returns the position (zero-based) of the first zero bit - // in the array block, starts from the offset. - // Returns the pass-the-end position if not found. - // FIXME: improve efficiency - uint32_t find_first_unset(rblock const& b, uint32_t offset) { - uint32_t entry = offset / BITS_PER_ARRAY_ENTRY; - uint32_t nr_entries = b.nr_entries(); - - if (entry >= nr_entries) - return entry * BITS_PER_ARRAY_ENTRY; - - uint32_t idx = offset % BITS_PER_ARRAY_ENTRY; - uint64_t v = b.get(entry++); - while (all_bits_set(v, idx) && entry < nr_entries) { - v = b.get(entry++); - idx = 0; - } - - if (all_bits_set(v, idx)) // not found - return entry * BITS_PER_ARRAY_ENTRY; - - return (entry - 1) * BITS_PER_ARRAY_ENTRY + idx + count_leading_bits(v, idx); - } - - // Returns true if all the bits beyond the position are set. - bool all_bits_set(uint64_t v, uint32_t offset) { - return (v >> offset) == (numeric_limits::max() >> offset); - } - - // Counts the number of leading 1's in the given value, starts from the offset - // FIXME: improve efficiency - uint32_t count_leading_bits(uint64_t v, uint32_t offset) { - uint32_t count = 0; - - v >>= offset; - while (v & 0x1) { - v >>= 1; - count++; - } - - return count; - } - - block_manager::ptr bm_; - uint64_traits::ref_counter rc_; - - const uint32_t BITS_PER_ARRAY_ENTRY; - }; - //-------------------------------- + template + dbg::command::ptr + create_btree_node_handler(block_manager::ptr bm) { + return create_block_handler(bm, create_btree_node_dumper()); + } + + template + dbg::command::ptr + create_array_block_handler(block_manager::ptr bm, + typename ShowTraits::value_trait::ref_counter rc) { + return create_block_handler(bm, create_array_block_dumper(rc)); + } + + dbg::command::ptr + create_bitset_block_handler(block_manager::ptr bm) { + return create_block_handler(bm, create_bitset_block_dumper()); + } + int debug(string const &path) { using dbg::command; @@ -400,15 +150,15 @@ namespace { block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY); transaction_manager::ptr null_tm = open_tm(bm, era::SUPERBLOCK_LOCATION); command_interpreter::ptr interp = create_command_interpreter(cin, cout); - interp->register_command("hello", command::ptr(new hello)); + interp->register_command("hello", create_hello_handler()); interp->register_command("superblock", command::ptr(new show_superblock(bm))); - interp->register_command("block_node", command::ptr(new show_btree_node(bm))); - interp->register_command("bitset_block", command::ptr(new show_bitset_block(bm))); - interp->register_command("era_block", command::ptr(new show_array_block(bm, - uint32_show_traits::ref_counter()))); - interp->register_command("writeset_node", command::ptr(new show_btree_node(bm))); + interp->register_command("block_node", create_btree_node_handler(bm)); + interp->register_command("bitset_block", create_bitset_block_handler(bm)); + interp->register_command("era_block", create_array_block_handler(bm, + uint32_traits::ref_counter())); + interp->register_command("writeset_node", create_btree_node_handler(bm)); interp->register_command("help", command::ptr(new help)); - interp->register_command("exit", command::ptr(new exit_handler(interp))); + interp->register_command("exit", create_exit_handler(interp)); interp->enter_main_loop(); } catch (std::exception &e) { diff --git a/thin-provisioning/thin_debug.cc b/thin-provisioning/thin_debug.cc index 37a5f18..9bc1ed4 100644 --- a/thin-provisioning/thin_debug.cc +++ b/thin-provisioning/thin_debug.cc @@ -21,11 +21,14 @@ #include #include -#include "base/command_interpreter.h" #include "base/math_utils.h" -#include "base/output_formatter.h" -#include "persistent-data/data-structures/btree.h" -#include "persistent-data/data-structures/simple_traits.h" +#include "dbg-lib/array_block_dumper.h" +#include "dbg-lib/btree_node_dumper.h" +#include "dbg-lib/index_block_dumper.h" +#include "dbg-lib/command_interpreter.h" +#include "dbg-lib/commands.h" +#include "dbg-lib/output_formatter.h" +#include "dbg-lib/sm_show_traits.h" #include "persistent-data/file_utils.h" #include "persistent-data/space-maps/disk_structures.h" #include "thin-provisioning/commands.h" @@ -39,12 +42,6 @@ using namespace std; using namespace thin_provisioning; namespace { - class hello : public dbg::command { - virtual void exec(strings const &args, ostream &out) { - out << "Hello, world!" << endl; - } - }; - class help : public dbg::command { virtual void exec(strings const &args, ostream &out) { out << "Commands:" << endl @@ -58,31 +55,6 @@ namespace { } }; - class exit_handler : public dbg::command { - public: - exit_handler(command_interpreter::ptr interpreter) - : interpreter_(interpreter) { - } - - virtual void exec(strings const &args, ostream &out) { - out << "Goodbye!" << endl; - interpreter_->exit_main_loop(); - } - - command_interpreter::ptr interpreter_; - }; - - class sm_root_show_traits : public persistent_data::sm_disk_detail::sm_root_traits { - public: - static void show(formatter::ptr f, string const &key, - persistent_data::sm_disk_detail::sm_root const &value) { - field(*f, "nr_blocks", value.nr_blocks_); - field(*f, "nr_allocated", value.nr_allocated_); - field(*f, "bitmap_root", value.bitmap_root_); - field(*f, "ref_count_root", value.ref_count_root_); - } - }; - class show_superblock : public dbg::command { public: explicit show_superblock(block_manager::ptr bm) @@ -142,7 +114,7 @@ namespace { block_manager::ptr bm_; }; - class device_details_show_traits : public thin_provisioning::device_tree_detail::device_details_traits { + class device_details_show_traits { public: typedef thin_provisioning::device_tree_detail::device_details_traits value_trait; @@ -155,16 +127,7 @@ namespace { } }; - class uint64_show_traits : public uint64_traits { - public: - typedef uint64_traits value_trait; - - static void show(formatter::ptr f, string const &key, uint64_t const &value) { - field(*f, key, boost::lexical_cast(value)); - } - }; - - class block_show_traits : public thin_provisioning::mapping_tree_detail::block_traits { + class block_show_traits { public: typedef thin_provisioning::mapping_tree_detail::block_traits value_trait; @@ -175,114 +138,18 @@ namespace { } }; - class index_entry_show_traits : public persistent_data::sm_disk_detail::index_entry_traits { - public: - typedef persistent_data::sm_disk_detail::index_entry_traits value_trait; - - static void show(formatter::ptr f, string const &key, - persistent_data::sm_disk_detail::index_entry const &value) { - field(*f, "blocknr", value.blocknr_); - field(*f, "nr_free", value.nr_free_); - field(*f, "none_free_before", value.none_free_before_); - } - }; + //-------------------------------- template - class show_btree_node : public dbg::command { - public: - explicit show_btree_node(block_manager::ptr bm) - : bm_(bm) { - } + dbg::command::ptr + create_btree_node_handler(block_manager::ptr bm) { + return create_block_handler(bm, create_btree_node_dumper()); + } - virtual void exec(strings const &args, ostream &out) { - using namespace persistent_data::btree_detail; - - if (args.size() != 2) - throw runtime_error("incorrect number of arguments"); - - block_address block = boost::lexical_cast(args[1]); - block_manager::read_ref rr = bm_->read_lock(block); - - node_ref n = btree_detail::to_node(rr); - if (n.get_type() == INTERNAL) - show_node(n, out); - else { - node_ref n = btree_detail::to_node(rr); - show_node(n, out); - } - } - - private: - template - void show_node(node_ref n, ostream &out) { - formatter::ptr f = create_xml_formatter(); - - field(*f, "csum", n.get_checksum()); - field(*f, "blocknr", n.get_block_nr()); - field(*f, "type", n.get_type() == INTERNAL ? "internal" : "leaf"); - field(*f, "nr_entries", n.get_nr_entries()); - field(*f, "max_entries", n.get_max_entries()); - field(*f, "value_size", n.get_value_size()); - - for (unsigned i = 0; i < n.get_nr_entries(); i++) { - formatter::ptr f2 = create_xml_formatter(); - field(*f2, "key", n.key_at(i)); - ST::show(f2, "value", n.value_at(i)); - f->child(boost::lexical_cast(i), f2); - } - - f->output(out, 0); - } - - block_manager::ptr bm_; - }; - - class show_index_block : public dbg::command { - public: - explicit show_index_block(block_manager::ptr bm) - : bm_(bm) { - } - - virtual void exec(strings const &args, ostream &out) { - if (args.size() != 2) - throw runtime_error("incorrect number of arguments"); - - block_address block = boost::lexical_cast(args[1]); - - // no checksum validation for debugging purpose - block_manager::read_ref rr = bm_->read_lock(block); - - sm_disk_detail::metadata_index const *mdi = - reinterpret_cast(rr.data()); - show_metadata_index(mdi, sm_disk_detail::MAX_METADATA_BITMAPS, out); - } - - private: - void show_metadata_index(sm_disk_detail::metadata_index const *mdi, block_address nr_indexes, ostream &out) { - formatter::ptr f = create_xml_formatter(); - field(*f, "csum", to_cpu(mdi->csum_)); - field(*f, "padding", to_cpu(mdi->padding_)); - field(*f, "blocknr", to_cpu(mdi->blocknr_)); - - sm_disk_detail::index_entry ie; - for (block_address i = 0; i < nr_indexes; i++) { - sm_disk_detail::index_entry_traits::unpack(*(mdi->index + i), ie); - - if (!ie.blocknr_ && !ie.nr_free_ && !ie.none_free_before_) - continue; - - formatter::ptr f2 = create_xml_formatter(); - index_entry_show_traits::show(f2, "value", ie); - f->child(boost::lexical_cast(i), f2); - } - f->output(out, 0); - } - - unsigned const ENTRIES_PER_BLOCK = (MD_BLOCK_SIZE - sizeof(persistent_data::sm_disk_detail::bitmap_header)) * 4; - block_manager::ptr bm_; - }; - - //-------------------------------- + dbg::command::ptr + create_index_block_handler(block_manager::ptr bm) { + return create_block_handler(bm, create_index_block_dumper()); + } int debug_(string const &path) { using dbg::command; @@ -290,15 +157,15 @@ namespace { try { block_manager::ptr bm = open_bm(path, block_manager::READ_ONLY, 1); command_interpreter::ptr interp = create_command_interpreter(cin, cout); - interp->register_command("hello", command::ptr(new hello)); + interp->register_command("hello", create_hello_handler()); interp->register_command("superblock", command::ptr(new show_superblock(bm))); - interp->register_command("m1_node", command::ptr(new show_btree_node(bm))); - interp->register_command("m2_node", command::ptr(new show_btree_node(bm))); - interp->register_command("detail_node", command::ptr(new show_btree_node(bm))); - interp->register_command("index_block", command::ptr(new show_index_block(bm))); - interp->register_command("index_node", command::ptr(new show_btree_node(bm))); + interp->register_command("m1_node", create_btree_node_handler(bm)); + interp->register_command("m2_node", create_btree_node_handler(bm)); + interp->register_command("detail_node", create_btree_node_handler(bm)); + interp->register_command("index_block", create_index_block_handler(bm)); + interp->register_command("index_node", create_btree_node_handler(bm)); interp->register_command("help", command::ptr(new help)); - interp->register_command("exit", command::ptr(new exit_handler(interp))); + interp->register_command("exit", create_exit_handler(interp)); interp->enter_main_loop(); } catch (std::exception &e) { From 29f9182078624095e9716eb99dd89f1f39c40900 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 21 Feb 2021 00:49:53 +0800 Subject: [PATCH 27/33] [dbg] Remove nested function template to reduce code size --- dbg-lib/btree_node_dumper.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dbg-lib/btree_node_dumper.h b/dbg-lib/btree_node_dumper.h index f92bce2..44fd2a9 100644 --- a/dbg-lib/btree_node_dumper.h +++ b/dbg-lib/btree_node_dumper.h @@ -17,16 +17,14 @@ namespace dbg { virtual void show(block_manager::read_ref &rr, std::ostream &out) { node_ref n = btree_detail::to_node(rr); if (n.get_type() == INTERNAL) - show_node(n, out); + btree_node_dumper::show_node(n, out); else { node_ref n = btree_detail::to_node(rr); - show_node(n, out); + show_node(n, out); } } - private: - template - void show_node(node_ref n, std::ostream &out) { + static void show_node(node_ref n, std::ostream &out) { formatter::ptr f = create_xml_formatter(); field(*f, "csum", n.get_checksum()); @@ -39,7 +37,7 @@ namespace dbg { for (unsigned i = 0; i < n.get_nr_entries(); i++) { formatter::ptr f2 = create_xml_formatter(); field(*f2, "key", n.key_at(i)); - ST::show(f2, "value", n.value_at(i)); + ShowTraits::show(f2, "value", n.value_at(i)); f->child(boost::lexical_cast(i), f2); } From e8410dec04003ae91d1cfc05cd83bfe160a8cc23 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 21 Feb 2021 15:49:02 +0800 Subject: [PATCH 28/33] [dbg] Add missing commands to cache/era_debug --- caching/cache_debug.cc | 16 ++++++++++++++++ era/era_debug.cc | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/caching/cache_debug.cc b/caching/cache_debug.cc index 4f9e033..838bd24 100644 --- a/caching/cache_debug.cc +++ b/caching/cache_debug.cc @@ -23,8 +23,10 @@ #include "dbg-lib/array_block_dumper.h" #include "dbg-lib/btree_node_dumper.h" +#include "dbg-lib/bitset_block_dumper.h" #include "dbg-lib/command_interpreter.h" #include "dbg-lib/commands.h" +#include "dbg-lib/index_block_dumper.h" #include "dbg-lib/output_formatter.h" #include "dbg-lib/sm_show_traits.h" #include "persistent-data/file_utils.h" @@ -46,6 +48,8 @@ namespace { out << "Commands:" << endl << " superblock [block#]" << endl << " block_node " << endl + << " bitset_block " << endl + << " index_block " << endl << " mapping_block " << endl << " exit" << endl; } @@ -141,6 +145,16 @@ namespace { return create_block_handler(bm, create_array_block_dumper(rc)); } + dbg::command::ptr + create_bitset_block_handler(block_manager::ptr bm) { + return create_block_handler(bm, create_bitset_block_dumper()); + } + + dbg::command::ptr + create_index_block_handler(block_manager::ptr bm) { + return create_block_handler(bm, create_index_block_dumper()); + } + int debug(string const &path) { using dbg::command; @@ -150,6 +164,8 @@ namespace { interp->register_command("hello", create_hello_handler()); interp->register_command("superblock", command::ptr(new show_superblock(bm))); interp->register_command("block_node", create_btree_node_handler(bm)); + interp->register_command("bitset_block", create_bitset_block_handler(bm)); + interp->register_command("index_block", create_index_block_handler(bm)); interp->register_command("mapping_block", create_array_block_handler(bm, mapping_traits::ref_counter())); interp->register_command("help", command::ptr(new help)); diff --git a/era/era_debug.cc b/era/era_debug.cc index 5470b4c..3769d4d 100644 --- a/era/era_debug.cc +++ b/era/era_debug.cc @@ -26,6 +26,7 @@ #include "dbg-lib/bitset_block_dumper.h" #include "dbg-lib/command_interpreter.h" #include "dbg-lib/commands.h" +#include "dbg-lib/index_block_dumper.h" #include "dbg-lib/output_formatter.h" #include "dbg-lib/sm_show_traits.h" #include "persistent-data/file_utils.h" @@ -49,6 +50,7 @@ namespace { << " block_node " << endl << " bitset_block " << endl << " era_block " << endl + << " index_block " << endl << " writeset_node " << endl << " exit" << endl; } @@ -143,6 +145,11 @@ namespace { return create_block_handler(bm, create_bitset_block_dumper()); } + dbg::command::ptr + create_index_block_handler(block_manager::ptr bm) { + return create_block_handler(bm, create_index_block_dumper()); + } + int debug(string const &path) { using dbg::command; @@ -156,6 +163,7 @@ namespace { interp->register_command("bitset_block", create_bitset_block_handler(bm)); interp->register_command("era_block", create_array_block_handler(bm, uint32_traits::ref_counter())); + interp->register_command("index_block", create_index_block_handler(bm)); interp->register_command("writeset_node", create_btree_node_handler(bm)); interp->register_command("help", command::ptr(new help)); interp->register_command("exit", create_exit_handler(interp)); From 2bb3bf65b7e3ce56dbcaa8e498229c0263717235 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Sun, 20 Dec 2020 20:23:30 +0800 Subject: [PATCH 29/33] [cache_check (rust)] Implement basic functions --- src/bin/cache_check.rs | 62 +++++++++++++++ src/cache/check.rs | 166 +++++++++++++++++++++++++++++++++++++++++ src/cache/mod.rs | 1 + 3 files changed, 229 insertions(+) create mode 100644 src/bin/cache_check.rs create mode 100644 src/cache/check.rs diff --git a/src/bin/cache_check.rs b/src/bin/cache_check.rs new file mode 100644 index 0000000..4fed20c --- /dev/null +++ b/src/bin/cache_check.rs @@ -0,0 +1,62 @@ +extern crate clap; +extern crate thinp; + +use clap::{App, Arg}; +use std::path::Path; +use thinp::cache::check::{check, CacheCheckOptions}; + +//------------------------------------------ + +fn main() { + let parser = App::new("cache_check") + .version(thinp::version::TOOLS_VERSION) + .arg( + Arg::with_name("INPUT") + .help("Specify the input device to check") + .required(true) + .index(1), + ) + .arg( + Arg::with_name("SB_ONLY") + .help("Only check the superblock.") + .long("super-block-only") + .value_name("SB_ONLY"), + ) + .arg( + Arg::with_name("SKIP_MAPPINGS") + .help("Don't check the mapping array") + .long("skip-mappings") + .value_name("SKIP_MAPPINGS"), + ) + .arg( + Arg::with_name("SKIP_HINTS") + .help("Don't check the hint array") + .long("skip-hints") + .value_name("SKIP_HINTS"), + ) + .arg( + Arg::with_name("SKIP_DISCARDS") + .help("Don't check the discard bitset") + .long("skip-discards") + .value_name("SKIP_DISCARDS"), + ); + + let matches = parser.get_matches(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + + let opts = CacheCheckOptions { + dev: &input_file, + async_io: false, + sb_only: matches.is_present("SB_ONLY"), + skip_mappings: matches.is_present("SKIP_MAPPINGS"), + skip_hints: matches.is_present("SKIP_HINTS"), + skip_discards: matches.is_present("SKIP_DISCARDS"), + }; + + if let Err(reason) = check(opts) { + eprintln!("{}", reason); + std::process::exit(1); + } +} + +//------------------------------------------ diff --git a/src/cache/check.rs b/src/cache/check.rs new file mode 100644 index 0000000..92895e1 --- /dev/null +++ b/src/cache/check.rs @@ -0,0 +1,166 @@ +use anyhow::anyhow; +use std::marker::PhantomData; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::collections::*; + +use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine}; +use crate::cache::hint::*; +use crate::cache::mapping::*; +use crate::cache::superblock::*; +use crate::pdata::array_walker::*; + +//------------------------------------------ + +const MAX_CONCURRENT_IO: u32 = 1024; + +//------------------------------------------ + +struct CheckMappingVisitor { + allowed_flags: u32, + seen_oblocks: Arc>>, +} + +impl CheckMappingVisitor { + fn new(metadata_version: u32) -> CheckMappingVisitor { + let mut flags: u32 = MappingFlags::Valid as u32; + if metadata_version == 1 { + flags |= MappingFlags::Dirty as u32; + } + CheckMappingVisitor { + allowed_flags: flags, + seen_oblocks: Arc::new(Mutex::new(BTreeSet::new())), + } + } + + fn seen_oblock(&self, b: u64) -> bool { + let seen_oblocks = self.seen_oblocks.lock().unwrap(); + return seen_oblocks.contains(&b); + } + + fn record_oblock(&self, b: u64) { + let mut seen_oblocks = self.seen_oblocks.lock().unwrap(); + seen_oblocks.insert(b); + } + + // FIXME: is it possible to validate flags at an early phase? + // e.g., move to ctor of Mapping? + fn has_unknown_flags(&self, m: &Mapping) -> bool { + return (m.flags & self.allowed_flags) != 0; + } +} + +impl ArrayBlockVisitor for CheckMappingVisitor { + fn visit(&self, _index: u64, m: Mapping) -> anyhow::Result<()> { + if !m.is_valid() { + return Ok(()); + } + + if self.seen_oblock(m.oblock) { + return Err(anyhow!("origin block already mapped")); + } + + self.record_oblock(m.oblock); + + if !self.has_unknown_flags(&m) { + return Err(anyhow!("unknown flags in mapping")); + } + + Ok(()) + } +} + +//------------------------------------------ + +struct CheckHintVisitor { + _not_used: PhantomData, +} + +impl CheckHintVisitor { + fn new() -> CheckHintVisitor { + CheckHintVisitor { + _not_used: PhantomData, + } + } +} + +impl ArrayBlockVisitor> for CheckHintVisitor { + fn visit(&self, _index: u64, _hint: Hint) -> anyhow::Result<()> { + // TODO: check hints + Ok(()) + } +} + +//------------------------------------------ + +// TODO: ignore_non_fatal, clear_needs_check, auto_repair +pub struct CacheCheckOptions<'a> { + pub dev: &'a Path, + pub async_io: bool, + pub sb_only: bool, + pub skip_mappings: bool, + pub skip_hints: bool, + pub skip_discards: bool, +} + +// TODO: thread pool, report +struct Context { + engine: Arc, +} + +fn mk_context(opts: &CacheCheckOptions) -> anyhow::Result { + let engine: Arc; + + if opts.async_io { + engine = Arc::new(AsyncIoEngine::new( + opts.dev, + MAX_CONCURRENT_IO, + false, + )?); + } else { + let nr_threads = std::cmp::max(8, num_cpus::get() * 2); + engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads, false)?); + } + + Ok(Context { + engine, + }) +} + +pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { + let ctx = mk_context(&opts)?; + + let engine = &ctx.engine; + + let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; + + if opts.sb_only { + return Ok(()); + } + + // TODO: factor out into check_mappings() + if !opts.skip_mappings { + let w = ArrayWalker::new(engine.clone(), false); + let c = Box::new(CheckMappingVisitor::new(sb.version)); + w.walk(c, sb.mapping_root)?; + + if sb.version >= 2 { + // TODO: check dirty bitset + } + } + + if !opts.skip_hints && sb.hint_root != 0 { + let w = ArrayWalker::new(engine.clone(), false); + type Width = typenum::U4; // FIXME: check sb.policy_hint_size + let c = Box::new(CheckHintVisitor::::new()); + w.walk(c, sb.hint_root)?; + } + + if !opts.skip_discards { + // TODO: check discard bitset + } + + Ok(()) +} + +//------------------------------------------ diff --git a/src/cache/mod.rs b/src/cache/mod.rs index f1cf219..a120ce6 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1,3 +1,4 @@ +pub mod check; pub mod hint; pub mod mapping; pub mod superblock; From a7ecfba2b4de69b72ec537170a8f37ff75521e4f Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Wed, 24 Feb 2021 20:21:10 +0800 Subject: [PATCH 30/33] [cache (rust)] Fix merge conflicts --- src/pdata/array_walker.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pdata/array_walker.rs b/src/pdata/array_walker.rs index 6322c50..e8287b1 100644 --- a/src/pdata/array_walker.rs +++ b/src/pdata/array_walker.rs @@ -55,7 +55,7 @@ impl NodeVisitor for BlockValueVisitor { // FIXME: return errors fn visit( &self, - path: &Vec, + path: &[u64], _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], @@ -71,7 +71,7 @@ impl NodeVisitor for BlockValueVisitor { } // FIXME: stub - fn visit_again(&self, _path: &Vec, _b: u64) -> Result<()> { + fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> { Ok(()) } From d4299a00d0ac30ab43d5adb826f55b17f12b5abf Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 26 Feb 2021 15:25:53 +0000 Subject: [PATCH 31/33] Make pack_node() and calc_max_entries() public for dm-unit --- src/pdata/btree_builder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pdata/btree_builder.rs b/src/pdata/btree_builder.rs index 44504ca..636762b 100644 --- a/src/pdata/btree_builder.rs +++ b/src/pdata/btree_builder.rs @@ -13,7 +13,7 @@ use crate::write_batcher::*; //------------------------------------------ -fn pack_node(node: &Node, w: &mut W) -> Result<()> { +pub fn pack_node(node: &Node, w: &mut W) -> Result<()> { match node { Node::Internal { header, @@ -60,7 +60,7 @@ fn pack_node(node: &Node, w: &mut W) -> R //------------------------------------------ -fn calc_max_entries() -> usize { +pub fn calc_max_entries() -> usize { let elt_size = 8 + V::disk_size() as usize; ((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize } From 7d983e315553bffebf16f0e7f32bec47f2b2eb86 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Thu, 11 Mar 2021 18:24:10 +0800 Subject: [PATCH 32/33] [btree_builder (rust)] Fix the max_entries --- src/pdata/btree_builder.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pdata/btree_builder.rs b/src/pdata/btree_builder.rs index 636762b..180ac51 100644 --- a/src/pdata/btree_builder.rs +++ b/src/pdata/btree_builder.rs @@ -62,7 +62,8 @@ pub fn pack_node(node: &Node, w: &mut W) pub fn calc_max_entries() -> usize { let elt_size = 8 + V::disk_size() as usize; - ((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize + let total = ((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize + total / 3 * 3 } //------------------------------------------ From 0ff72374f8594bc3064fa388ee7861c06ee33724 Mon Sep 17 00:00:00 2001 From: Ming-Hung Tsai Date: Fri, 12 Mar 2021 00:42:54 +0800 Subject: [PATCH 33/33] [btree_builder (rust)] Fix the max_entries --- src/pdata/btree_builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pdata/btree_builder.rs b/src/pdata/btree_builder.rs index 180ac51..6ae1379 100644 --- a/src/pdata/btree_builder.rs +++ b/src/pdata/btree_builder.rs @@ -62,7 +62,7 @@ pub fn pack_node(node: &Node, w: &mut W) pub fn calc_max_entries() -> usize { let elt_size = 8 + V::disk_size() as usize; - let total = ((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize + let total = ((BLOCK_SIZE - NodeHeader::disk_size() as usize) / elt_size) as usize; total / 3 * 3 }