[era] switch from bloom filters to writesets

Makefile.in

@@ -27,6 +27,7 @@ PROGRAMS=\
 	\
 	era_check \
 	era_dump \
+	era_invalidate \
 	\
 	thin_check \
 	thin_dump \
@@ -53,7 +54,7 @@ SOURCE=\
 	era/era_array.cc \
 	era/era_detail.cc \
 	era/superblock.cc \
-	era/bloom_tree.cc \
+	era/writeset_tree.cc \
 	era/metadata.cc \
 	era/metadata_dump.cc \
 	era/xml_format.cc \
@@ -305,7 +306,7 @@ ERA_CHECK_SOURCE=\
 	base/error_state.cc \
 	base/endian_utils.cc \
 	\
-	era/bloom_tree.cc \
+	era/writeset_tree.cc \
 	era/era_detail.cc \
 	era/era_array.cc \
 	era/metadata.cc \
@@ -335,7 +336,7 @@ ERA_DUMP_SOURCE=\
 	base/error_state.cc \
 	base/endian_utils.cc \
 	\
-	era/bloom_tree.cc \
+	era/writeset_tree.cc \
 	era/era_detail.cc \
 	era/era_array.cc \
 	era/metadata.cc \
@@ -362,6 +363,38 @@ era_dump: $(ERA_DUMP_OBJECTS)  era/era_dump.o
 	@echo "    [LD]  $@"
 	$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
 
+ERA_INVALIDATE_SOURCE=\
+	base/base64.cc \
+	base/error_state.cc \
+	base/endian_utils.cc \
+	\
+	era/writeset_tree.cc \
+	era/era_detail.cc \
+	era/era_array.cc \
+	era/metadata.cc \
+	era/metadata_dump.cc \
+	era/superblock.cc \
+	era/xml_format.cc \
+	\
+	persistent-data/checksum.cc \
+	persistent-data/error_set.cc \
+	persistent-data/file_utils.cc \
+	persistent-data/hex_dump.cc \
+	persistent-data/lock_tracker.cc \
+	persistent-data/data-structures/btree.cc \
+	persistent-data/data-structures/bitset.cc \
+	persistent-data/space_map.cc \
+	persistent-data/space-maps/disk.cc \
+	persistent-data/space-maps/recursive.cc \
+	persistent-data/space-maps/careful_alloc.cc \
+	persistent-data/transaction_manager.cc \
+
+ERA_INVALIDATE_OBJECTS=$(subst .cc,.o,$(ERA_INVALIDATE_SOURCE))
+
+era_invalidate: $(ERA_INVALIDATE_OBJECTS)  era/era_invalidate.o
+	@echo "    [LD]  $@"
+	$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
+
 #----------------------------------------------------------------
 
 DEPEND_FILES=\

era/emitter.h

@@ -20,9 +20,9 @@ namespace era {
 					      uint32_t current_era) = 0;
 		virtual void end_superblock() = 0;
 
-		virtual void begin_bloom(uint32_t era, uint32_t nr_bits, pd::block_address nr_blocks) = 0;
+		virtual void begin_writeset(uint32_t era, uint32_t nr_bits) = 0;
-		virtual void bloom_bit(uint32_t bit, bool value) = 0;
+		virtual void writeset_bit(uint32_t bit, bool value) = 0;
-		virtual void end_bloom() = 0;
+		virtual void end_writeset() = 0;
 
 		virtual void begin_era_array() = 0;
 		virtual void era(pd::block_address block, uint32_t era) = 0;

era/era_check.cc

@@ -14,7 +14,7 @@
 
 #include "base/error_state.h"
 #include "base/nested_output.h"
-#include "era/bloom_tree.h"
+#include "era/writeset_tree.h"
 #include "era/era_array.h"
 #include "era/superblock.h"
 #include "persistent-data/block.h"
@@ -92,14 +92,14 @@ namespace {
 		using reporter_base::get_error;
 	};
 
-	class bloom_tree_reporter : public bloom_tree_detail::damage_visitor, reporter_base {
+	class writeset_tree_reporter : public writeset_tree_detail::damage_visitor, reporter_base {
 	public:
-		bloom_tree_reporter(nested_output &o)
+		writeset_tree_reporter(nested_output &o)
 		: reporter_base(o) {
 		}
 
-		void visit(bloom_tree_detail::missing_eras const &d) {
+		void visit(writeset_tree_detail::missing_eras const &d) {
-			out() << "missing eras from bloom tree" << end_message();
+			out() << "missing eras from writeset tree" << end_message();
 			{
 				nested_output::nest _ = push();
 				out() << d.get_desc() << end_message();
@@ -110,8 +110,8 @@ namespace {
 			mplus_error(FATAL);
 		}
 
-		void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
+		void visit(writeset_tree_detail::damaged_writeset const &d) {
-			out() << "damaged bloom filter" << end_message();
+			out() << "damaged writeset" << end_message();
 			{
 				nested_output::nest _ = push();
 				out() << d.get_desc() << end_message();
@@ -216,22 +216,22 @@ namespace {
 		superblock sb = read_superblock(bm);
 		transaction_manager::ptr tm = open_tm(bm);
 
-		bloom_tree_reporter bt_rep(out);
+		writeset_tree_reporter wt_rep(out);
 		{
 			era_detail_traits::ref_counter rc(tm);
-			bloom_tree bt(tm, sb.bloom_tree_root, rc);
+			writeset_tree wt(tm, sb.writeset_tree_root, rc);
-			check_bloom_tree(tm, bt, bt_rep);
+			check_writeset_tree(tm, wt, wt_rep);
 		}
 
 		era_array_reporter ea_rep(out);
 		{
 			uint32_traits::ref_counter rc;
 			era_array ea(tm, rc, sb.era_array_root, sb.nr_blocks);
-			check_era_array(ea, ea_rep);
+			check_era_array(ea, sb.current_era, ea_rep);
 		}
 
 		return combine_errors(sb_rep.get_error(),
-				      combine_errors(bt_rep.get_error(),
+				      combine_errors(wt_rep.get_error(),
 						     ea_rep.get_error()));
 	}
 

era/era_detail.cc

@@ -7,37 +7,18 @@ using namespace era;
 
 //----------------------------------------------------------------
 
-namespace {
-#if 0
-	le32 pack_hash_detail(uint32_t hash1, uint32_t hash2, uint32_t nr_probes) {
-		throw std::runtime_error("not implemented");
-	}
-
-	void unpack_hash_detail(le32 packed, uint32_t &hash1, uint32_t &hash2, uint32_t &nr_probes) {
-		throw std::runtime_error("not implemented");
-	}
-#endif
-}
-
 void
 era_detail_traits::unpack(disk_type const &disk, value_type &value)
 {
-	value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
 	value.nr_bits = to_cpu<uint32_t>(disk.nr_bits);
-	value.nr_set = to_cpu<uint32_t>(disk.nr_set);
+	value.writeset_root = to_cpu<uint64_t>(disk.writeset_root);
-	value.bloom_root = to_cpu<uint64_t>(disk.bloom_root);
-
-	//unpack_hash_detail(disk.hash_fns_and_probes, value.hash1, value.hash2, value.nr_probes);
 }
 
 void
 era_detail_traits::pack(value_type const &value, disk_type &disk)
 {
-	disk.nr_blocks = to_disk<le32>(value.nr_blocks);
 	disk.nr_bits = to_disk<le32>(value.nr_bits);
-	disk.nr_set = to_disk<le32>(value.nr_set);
+	disk.writeset_root = to_disk<le64>(value.writeset_root);
-	disk.bloom_root = to_disk<le64>(value.bloom_root);
-//	disk.hash_fns_and_probes = pack_hash_detail(value.hash1, value.hash2, value.nr_probes);
 }
 
 //----------------------------------------------------------------

era/era_detail.h

@@ -8,22 +8,13 @@
 
 namespace era {
 	struct era_detail_disk {
-		base::le32 nr_blocks;
 		base::le32 nr_bits;
-		base::le32 nr_set;
+		base::le64 writeset_root;
-		base::le64 bloom_root;
 	} __attribute__ ((packed));
 
 	struct era_detail {
-		uint32_t nr_blocks;
 		uint32_t nr_bits;
-		uint32_t nr_set;
+		uint64_t writeset_root;
-
-		uint32_t hash1;
-		uint32_t hash2;
-		uint32_t nr_probes;
-
-		uint64_t bloom_root;
 	};
 
 	// FIXME: implement

era/era_dump.cc

@@ -5,7 +5,7 @@
 
 #include "version.h"
 #include "era/era_array.h"
-#include "era/bloom_tree.h"
+#include "era/writeset_tree.h"
 #include "era/metadata.h"
 #include "era/metadata_dump.h"
 #include "era/xml_format.h"

era/era_invalidate.cc

@@ -0,0 +1,114 @@
+#include <fstream>
+#include <getopt.h>
+#include <libgen.h>
+#include <iostream>
+
+#include "version.h"
+#include "era/era_array.h"
+#include "era/writeset_tree.h"
+#include "era/metadata.h"
+#include "era/xml_format.h"
+#include "persistent-data/file_utils.h"
+
+using namespace era;
+using namespace std;
+
+//----------------------------------------------------------------
+
+namespace {
+	struct flags {
+		flags() {
+		}
+	};
+
+	//--------------------------------
+
+	void emit_blocks() {
+		
+	}
+
+	//--------------------------------
+
+	string const STDOUT_PATH("-");
+
+	bool want_stdout(string const &output) {
+		return output == STDOUT_PATH;
+	}
+
+	int invalidate(string const &dev, string const &output, flags const &fs) {
+		try {
+			block_manager<>::ptr bm = open_bm(dev, block_io<>::READ_ONLY);
+			metadata::ptr md(new metadata(bm, metadata::OPEN));
+
+			if (want_stdout(output)) {
+				emitter::ptr e = create_xml_emitter(cout);
+				//emit_blocks(md, e, fs);
+			} else {
+				ofstream out(output.c_str());
+				emitter::ptr e = create_xml_emitter(out);
+				//emit_blocks(md, e, fs);
+			}
+
+		} catch (std::exception &e) {
+			cerr << e.what() << endl;
+			return 1;
+		}
+
+		return 0;
+	}
+
+	void usage(ostream &out, string const &cmd) {
+		out << "Usage: " << cmd << " [options] {device|file}" << endl
+		    << "Options:" << endl
+		    << "  {-h|--help}" << endl
+		    << "  {-o <xml file>}" << endl
+		    << "  {-V|--version}" << endl;
+	}
+}
+
+//----------------------------------------------------------------
+
+int main(int argc, char **argv)
+{
+	int c;
+	flags fs;
+	string output("-");
+	char const shortopts[] = "ho:V";
+
+	option const longopts[] = {
+		{ "help", no_argument, NULL, 'h' },
+		{ "output", required_argument, NULL, 'o' },
+		{ "version", no_argument, NULL, 'V' },
+		{ NULL, no_argument, NULL, 0 }
+	};
+
+	while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
+		switch(c) {
+		case 'h':
+			usage(cout, basename(argv[0]));
+			return 0;
+
+		case 'o':
+			output = optarg;
+			break;
+
+		case 'V':
+			cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
+			return 0;
+
+		default:
+			usage(cerr, basename(argv[0]));
+			return 1;
+		}
+	}
+
+	if (argc == optind) {
+		cerr << "No input file provided." << endl;
+		usage(cerr, basename(argv[0]));
+		return 1;
+	}
+
+	return invalidate(argv[optind], output, fs);
+}
+
+//----------------------------------------------------------------

era/metadata.cc

@@ -47,8 +47,8 @@ metadata::open_metadata(block_manager<>::ptr bm)
 	tm_ = open_tm(bm);
 	sb_ = read_superblock(tm_->get_bm());
 
-	bloom_tree_ = bloom_tree::ptr(new bloom_tree(tm_,
+	writeset_tree_ = writeset_tree::ptr(new writeset_tree(tm_,
-						     sb_.bloom_tree_root,
+							      sb_.writeset_tree_root,
 							      era_detail_traits::ref_counter(tm_)));
 
 	era_array_ = era_array::ptr(new era_array(tm_,

era/metadata.h

@@ -10,7 +10,7 @@
 #include "persistent-data/transaction_manager.h"
 
 #include "era/superblock.h"
-#include "era/bloom_tree.h"
+#include "era/writeset_tree.h"
 #include "era/era_array.h"
 
 //----------------------------------------------------------------
@@ -34,7 +34,7 @@ namespace era {
 		tm::ptr tm_;
 		superblock sb_;
 		checked_space_map::ptr metadata_sm_;
-		bloom_tree::ptr bloom_tree_;
+		writeset_tree::ptr writeset_tree_;
 		era_array::ptr era_array_;
 
 	private:

era/metadata_dump.cc

@@ -17,42 +17,42 @@ namespace {
 					 "perhaps you wanted to run with --repair ?");
 	}
 
-	class bloom_tree_emitter : public bloom_tree_detail::bloom_visitor {
+	class writeset_tree_emitter : public writeset_tree_detail::writeset_visitor {
 	public:
-		bloom_tree_emitter(emitter::ptr e)
+		writeset_tree_emitter(emitter::ptr e)
 			: e_(e) {
 		}
 
-		virtual void bloom_begin(uint32_t era, uint32_t nr_blocks, uint32_t nr_bits, uint32_t nr_set) {
+		virtual void writeset_begin(uint32_t era, uint32_t nr_bits) {
-			e_->begin_bloom(era, nr_bits, nr_blocks);
+			e_->begin_writeset(era, nr_bits);
 		}
 
 		virtual void bit(uint32_t bit, bool value) {
-			e_->bloom_bit(bit, value);
+			e_->writeset_bit(bit, value);
 		}
 
-		virtual void bloom_end() {
+		virtual void writeset_end() {
-			e_->end_bloom();
+			e_->end_writeset();
 		}
 
 	private:
 		emitter::ptr e_;
 	};
 
-	struct ignore_bloom_tree_damage : public bloom_tree_detail::damage_visitor {
+	struct ignore_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
-		void visit(bloom_tree_detail::missing_eras const &d) {
+		void visit(writeset_tree_detail::missing_eras const &d) {
 		}
 
-		void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
+		void visit(writeset_tree_detail::damaged_writeset const &d) {
 		}
 	};
 
-	struct fatal_bloom_tree_damage : public bloom_tree_detail::damage_visitor {
+	struct fatal_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
-		void visit(bloom_tree_detail::missing_eras const &d) {
+		void visit(writeset_tree_detail::missing_eras const &d) {
 			raise_metadata_damage();
 		}
 
-		void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
+		void visit(writeset_tree_detail::damaged_writeset const &d) {
 			raise_metadata_damage();
 		}
 	};
@@ -104,15 +104,15 @@ era::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair)
 			    sb.current_era);
 	{
 		{
-			bloom_tree_emitter visitor(e);
+			writeset_tree_emitter visitor(e);
 
-			ignore_bloom_tree_damage ignore;
+			ignore_writeset_tree_damage ignore;
-			fatal_bloom_tree_damage fatal;
+			fatal_writeset_tree_damage fatal;
-			bloom_tree_detail::damage_visitor &dv = repair ?
+			writeset_tree_detail::damage_visitor &dv = repair ?
-				static_cast<bloom_tree_detail::damage_visitor &>(ignore) :
+				static_cast<writeset_tree_detail::damage_visitor &>(ignore) :
-				static_cast<bloom_tree_detail::damage_visitor &>(fatal);
+				static_cast<writeset_tree_detail::damage_visitor &>(fatal);
 
-			walk_bloom_tree(md->tm_, *md->bloom_tree_, visitor, dv);
+			walk_writeset_tree(md->tm_, *md->writeset_tree_, visitor, dv);
 		}
 
 		e->begin_era_array();

era/superblock.cc

@@ -34,7 +34,7 @@ namespace  {
 		le32 current_era;
 		era_detail_disk current_detail;
 
-		le64 bloom_tree_root;
+		le64 writeset_tree_root;
 		le64 era_array_root;
 
 	} __attribute__ ((packed));
@@ -116,7 +116,7 @@ superblock::superblock()
 	  metadata_block_size(8),
 	  nr_blocks(0),
 	  current_era(0),
-	  bloom_tree_root(0),
+	  writeset_tree_root(0),
 	  era_array_root(0)
 {
 	memset(uuid, 0, sizeof(uuid));
@@ -141,7 +141,7 @@ superblock_traits::unpack(disk_type const &disk, value_type &value)
 	value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
 	value.current_era = to_cpu<uint32_t>(disk.current_era);
 	era_detail_traits::unpack(disk.current_detail, value.current_detail);
-	value.bloom_tree_root = to_cpu<uint64_t>(disk.bloom_tree_root);
+	value.writeset_tree_root = to_cpu<uint64_t>(disk.writeset_tree_root);
 	value.era_array_root = to_cpu<uint64_t>(disk.era_array_root);
 }
 
@@ -161,7 +161,7 @@ superblock_traits::pack(value_type const &value, disk_type &disk)
 	disk.nr_blocks = to_disk<le32>(value.nr_blocks);
 	disk.current_era = to_disk<le32>(value.current_era);
 	era_detail_traits::pack(value.current_detail, disk.current_detail);
-	disk.bloom_tree_root = to_disk<le64>(value.bloom_tree_root);
+	disk.writeset_tree_root = to_disk<le64>(value.writeset_tree_root);
 	disk.era_array_root = to_disk<le64>(value.era_array_root);
 }
 
@@ -282,8 +282,8 @@ era::check_superblock(superblock const &sb,
 		visitor.visit(superblock_invalid(msg.str()));
 	}
 
-	if (sb.bloom_tree_root == SUPERBLOCK_LOCATION) {
+	if (sb.writeset_tree_root == SUPERBLOCK_LOCATION) {
-		string msg("bloom tree root points back to the superblock");
+		string msg("writeset tree root points back to the superblock");
 		visitor.visit(superblock_invalid(msg));
 	}
 
@@ -292,9 +292,9 @@ era::check_superblock(superblock const &sb,
 		visitor.visit(superblock_invalid(msg));
 	}
 
-	if (sb.bloom_tree_root == sb.era_array_root) {
+	if (sb.writeset_tree_root == sb.era_array_root) {
 		ostringstream msg;
-		msg << "bloom tree root and era array both point to the same block: "
+		msg << "writeset tree root and era array both point to the same block: "
 		    << sb.era_array_root;
 		visitor.visit(superblock_invalid(msg.str()));
 	}

era/superblock.h

@@ -59,7 +59,7 @@ namespace era {
 		era_detail current_detail;
 
 		// A btree of undigested era_details
-		uint64_t bloom_tree_root;
+		uint64_t writeset_tree_root;
 
 		// Big array holding the digested era/block info.
 		uint64_t era_array_root;

era/writeset_tree.cc

@@ -1,10 +1,10 @@
-#include "era/bloom_tree.h"
+#include "era/writeset_tree.h"
 #include "persistent-data/data-structures/btree_damage_visitor.h"
 #include "persistent-data/data-structures/bitset.h"
 
 using namespace boost;
 using namespace era;
-using namespace bloom_tree_detail;
+using namespace writeset_tree_detail;
 using namespace persistent_data;
 using namespace std;
 
@@ -22,7 +22,7 @@ missing_eras::visit(damage_visitor &v) const {
 	v.visit(*this);
 }
 
-damaged_bloom_filter::damaged_bloom_filter(string const &desc,
+damaged_writeset::damaged_writeset(string const &desc,
 				   uint32_t era,
 				   run<uint32_t> missing_bits)
 	: damage(desc),
@@ -32,7 +32,7 @@ damaged_bloom_filter::damaged_bloom_filter(string const &desc,
 }
 
 void
-damaged_bloom_filter::visit(damage_visitor &v) const
+damaged_writeset::visit(damage_visitor &v) const
 {
 	v.visit(*this);
 }
@@ -40,39 +40,39 @@ damaged_bloom_filter::visit(damage_visitor &v) const
 //----------------------------------------------------------------
 
 namespace {
-	class ll_bloom_visitor : public bitset_detail::bitset_visitor {
+	class ll_writeset_visitor : public bitset_detail::bitset_visitor {
 	public:
 		typedef persistent_data::transaction_manager::ptr tm_ptr;
 
-		ll_bloom_visitor(tm_ptr tm,
+		ll_writeset_visitor(tm_ptr tm,
-				 bloom_tree_detail::bloom_visitor &bloom_v,
+				    writeset_tree_detail::writeset_visitor &writeset_v,
-				 bloom_tree_detail::damage_visitor &dv)
+				    writeset_tree_detail::damage_visitor &dv)
 			: tm_(tm),
-			  bloom_v_(bloom_v),
+			  writeset_v_(writeset_v),
 			  dv_(dv) {
 		}
 
 		void visit(btree_path const &path, era_detail const &era) {
 			era_ = path[0];
-			bitset bs(tm_, era.bloom_root, era.nr_bits);
+			bitset bs(tm_, era.writeset_root, era.nr_bits);
-			bloom_v_.bloom_begin(era_, era.nr_blocks, era.nr_bits, era.nr_set);
+			writeset_v_.writeset_begin(era_, era.nr_bits);
 			bs.walk_bitset(*this);
-			bloom_v_.bloom_end();
+			writeset_v_.writeset_end();
 		}
 
 		void visit(uint32_t index, bool value) {
-			bloom_v_.bit(index, value);
+			writeset_v_.bit(index, value);
 		}
 
 		void visit(bitset_detail::missing_bits const &d) {
-			dv_.visit(bloom_tree_detail::damaged_bloom_filter("missing bits", era_, d.keys_));
+			dv_.visit(writeset_tree_detail::damaged_writeset("missing bits", era_, d.keys_));
 		}
 
 	private:
 		tm_ptr tm_;
 		uint64_t era_;
-		bloom_tree_detail::bloom_visitor &bloom_v_;
+		writeset_tree_detail::writeset_visitor &writeset_v_;
-		bloom_tree_detail::damage_visitor &dv_;
+		writeset_tree_detail::damage_visitor &dv_;
 	};
 
 	class ll_damage_visitor {
@@ -98,38 +98,37 @@ namespace {
 }
 
 void
-era::walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
+era::walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
-		     bloom_tree const &tree,
+			writeset_tree const &tree,
-		     bloom_tree_detail::bloom_visitor &bloom_v,
+			writeset_tree_detail::writeset_visitor &writeset_v,
-		     bloom_tree_detail::damage_visitor &dv)
+			writeset_tree_detail::damage_visitor &dv)
 {
-	ll_bloom_visitor ll_bv(tm, bloom_v, dv);
+	ll_writeset_visitor ll_bv(tm, writeset_v, dv);
 	ll_damage_visitor ll_dv(dv);
 	btree_visit_values(tree, ll_bv, ll_dv);
 }
 
 namespace {
-	class noop_bloom_visitor : public bloom_tree_detail::bloom_visitor {
+	class noop_writeset_visitor : public writeset_tree_detail::writeset_visitor {
 	public:
-		void bloom_begin(uint32_t era, uint32_t nr_blocks,
+		void writeset_begin(uint32_t era, uint32_t nr_bits) {
-				 uint32_t nr_bits, uint32_t nr_set) {
 		}
 
 		void bit(uint32_t index, bool value) {
 		}
 
-		void bloom_end() {
+		void writeset_end() {
 		}
 	};
 };
 
 void
-era::check_bloom_tree(persistent_data::transaction_manager::ptr tm,
+era::check_writeset_tree(persistent_data::transaction_manager::ptr tm,
-		      bloom_tree const &tree,
+			 writeset_tree const &tree,
-		      bloom_tree_detail::damage_visitor &dv)
+			 writeset_tree_detail::damage_visitor &dv)
 {
-	noop_bloom_visitor bv;
+	noop_writeset_visitor bv;
-	walk_bloom_tree(tm, tree, bv, dv);
+	walk_writeset_tree(tm, tree, bv, dv);
 }
 
 //----------------------------------------------------------------

era/writeset_tree.h

@@ -1,5 +1,5 @@
-#ifndef ERA_BLOOM_TREE_H
+#ifndef ERA_WRITESET_TREE_H
-#define ERA_BLOOM_TREE_H
+#define ERA_WRITESET_TREE_H
 
 #include "era/era_detail.h"
 #include "persistent-data/data-structures/btree.h"
@@ -7,7 +7,7 @@
 //----------------------------------------------------------------
 
 namespace era {
-	namespace bloom_tree_detail {
+	namespace writeset_tree_detail {
 		class damage_visitor;
 
 		class damage {
@@ -34,8 +34,8 @@ namespace era {
 			run<uint32_t> eras_;
 		};
 
-		struct damaged_bloom_filter : public damage {
+		struct damaged_writeset : public damage {
-			damaged_bloom_filter(std::string const &desc,
+			damaged_writeset(std::string const &desc,
 					 uint32_t era,
 					 run<uint32_t> missing_bits);
 			virtual void visit(damage_visitor &v) const;
@@ -55,32 +55,31 @@ namespace era {
 			}
 
 			virtual void visit(missing_eras const &d) = 0;
-			virtual void visit(damaged_bloom_filter const &d) = 0;
+			virtual void visit(damaged_writeset const &d) = 0;
 		};
 
-		class bloom_visitor {
+		class writeset_visitor {
 		public:
-			typedef boost::shared_ptr<bloom_visitor> ptr;
+			typedef boost::shared_ptr<writeset_visitor> ptr;
 
-			virtual ~bloom_visitor() {}
+			virtual ~writeset_visitor() {}
 
-			virtual void bloom_begin(uint32_t era, uint32_t nr_blocks,
+			virtual void writeset_begin(uint32_t era, uint32_t nr_bits) = 0;
-						 uint32_t nr_bits, uint32_t nr_set) = 0;
 			virtual void bit(uint32_t index, bool value) = 0;
-			virtual void bloom_end() = 0;
+			virtual void writeset_end() = 0;
 		};
 	}
 
-	typedef persistent_data::btree<1, era_detail_traits> bloom_tree;
+	typedef persistent_data::btree<1, era_detail_traits> writeset_tree;
 
-	void walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
+	void walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
-			     bloom_tree const &tree,
+				writeset_tree const &tree,
-			     bloom_tree_detail::bloom_visitor &bloom_v,
+				writeset_tree_detail::writeset_visitor &writeset_v,
-			     bloom_tree_detail::damage_visitor &dv);
+				writeset_tree_detail::damage_visitor &dv);
 
-	void check_bloom_tree(persistent_data::transaction_manager::ptr tm,
+	void check_writeset_tree(persistent_data::transaction_manager::ptr tm,
-			      bloom_tree const &tree,
+				 writeset_tree const &tree,
-			      bloom_tree_detail::damage_visitor &dv);
+				 writeset_tree_detail::damage_visitor &dv);
 }
 
 //----------------------------------------------------------------

era/xml_format.cc

@@ -33,25 +33,23 @@ namespace {
 			out_ << "</superblock>" << endl;
 		}
 
-		void begin_bloom(uint32_t era, uint32_t nr_bits,
+		void begin_writeset(uint32_t era, uint32_t nr_bits) {
-				 pd::block_address nr_blocks) {
 			indent();
-			out_ << "<bloom era=\"" << era << "\""
+			out_ << "<writeset era=\"" << era << "\""
-			     << " nr_bits=\"" << nr_bits << "\""
+			     << " nr_bits=\"" << nr_bits << "\">" << endl;
-			     << " nr_blocks=\"" << nr_blocks << "\">" << endl;
 			inc();
 		}
 
-		void bloom_bit(uint32_t bit, bool value) {
+		void writeset_bit(uint32_t bit, bool value) {
 			indent();
 			// FIXME: collect all the bits, then uuencode
 			out_ << "<bit bit=\"" << bit << "\" value=\"" << value << "\">" << endl;
 		}
 
-		void end_bloom() {
+		void end_writeset() {
 			dec();
 			indent();
-			out_ << "</bloom>" << endl;
+			out_ << "</writeset>" << endl;
 		}
 
 		void begin_era_array() {

persistent-data/block.tcc

@@ -343,7 +343,7 @@ namespace persistent_data {
 						unsigned max_concurrent_blocks,
 						typename block_io<BlockSize>::mode mode)
 		: io_(new block_io<BlockSize>(path, nr_blocks, mode)),
-		  cache_(max(64u, max_concurrent_blocks)),
+		  cache_(max(1024u, max_concurrent_blocks)),
 		  tracker_(0, nr_blocks)
 	{
 	}

unit-tests/array_t.cc

@@ -20,6 +20,7 @@
 #include "persistent-data/transaction_manager.h"
 #include "persistent-data/space-maps/core.h"
 #include "persistent-data/data-structures/array.h"
+#include "persistent-data/data-structures/simple_traits.h"
 
 #include <vector>
 

unit-tests/bloom_filter_t.cc

@@ -40,17 +40,26 @@ namespace {
 
 			using namespace boost::random;
 
-			mt19937 rng;
 			uniform_int_distribution<uint64_t> uniform_dist(0, max);
 
 			while (r.size() < count) {
-				block_address b = uniform_dist(rng);
+				block_address b = uniform_dist(rng_);
 				r.insert(b);
 			}
 
 			return r;
 		}
 
+		set<block_address> generate_linear_blocks(unsigned count,
+							  block_address max = std::numeric_limits<uint64_t>::max()) {
+			set<block_address> r;
+
+			for (unsigned i = 0; i < count; i++)
+				r.insert(i);
+
+			return r;
+		}
+
 		void commit() {
 			block_manager<>::write_ref superblock(bm_->superblock(SUPERBLOCK));
 		}
@@ -65,6 +74,8 @@ namespace {
 		block_manager<>::ptr bm_;
 		space_map::ptr sm_;
 		transaction_manager::ptr tm_;
+
+		boost::random::mt19937 rng_;
 	};
 }
 
@@ -119,18 +130,75 @@ TEST_F(BloomFilterTests, reload_works)
 	}
 }
 
-TEST_F(BloomFilterTests, count_false_positives)
+unsigned next_power(unsigned n)
 {
-	block_address nr_blocks = 1024 * 1024;
+	unsigned r = 1;
-	block_address written_blocks = nr_blocks / 100;
+	while (r < n)
+		r <<= 1;
 
-	unsigned shift = 1;
+	return r;
+}
 
-	while ((1ull << shift) < (16 * written_blocks))
+unsigned calc_nr_bits(double false_positive_rate, unsigned dirty_blocks_per_era, unsigned nr_probes)
-		shift++;
+{
-	cerr << "bitset size: " << ((1 << shift) / (8 * 1024)) << "k" << endl;
+	double k = (double) nr_probes;
+	double kth_root = exp(log(false_positive_rate) / k); // can be precomputed
 
-	bloom_filter f(tm_, 1 << shift, 6);
+	// FIXME: we need a way to calulate this in kernel?  or should we
+	// just pass in the bloom params on the target line?
+	double tmp = log(1.0 - kth_root);
+	double n = (- k * (double) dirty_blocks_per_era) / tmp;
+
+	return next_power(ceil(n));
+}
+
+unsigned calc_m(double fp, unsigned nr_probes, unsigned n)
+{
+	double k = (double) nr_probes;
+	double kth_root = exp(log(fp) / k);
+	double tmp = log(1.0 - kth_root);
+	double m = (- ((double) n) / k) * tmp;
+
+	return ceil(m);
+}
+
+void print_nr_bits_table(double fp, unsigned nr_probes)
+{
+	cout << "fp = " << fp << ", k = " << nr_probes << endl;
+
+	for (unsigned long long m = 1024; m < (1ull << 25); m *= 2) {
+		unsigned n = calc_nr_bits(fp, m, nr_probes);
+		unsigned actual_m = calc_m(fp, nr_probes, n);
+
+		cout << "    m = " << m << ", n = " << n << ", " << n / (1024 * 8)
+		     << "k, actual_m = " << actual_m << endl;
+	}
+}
+
+// Not really a test
+TEST_F(BloomFilterTests, nr_bits_table)
+{
+	print_nr_bits_table(0.001, 4);
+	print_nr_bits_table(0.001, 6);
+	print_nr_bits_table(0.001, 8);
+	print_nr_bits_table(0.001, 16);
+}
+
+TEST_F(BloomFilterTests, count_false_positives_with_random_inserts)
+{
+	block_address nr_blocks = 1 << 27;
+	block_address written_blocks = nr_blocks / 1024;
+
+	unsigned nr_probes = 6;
+	unsigned n = calc_nr_bits(0.001, written_blocks, nr_probes);
+
+	cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
+
+	double ideal_k = log(2) * ((double) n / (double) written_blocks);
+	cerr << "Ideal k = " << ideal_k << endl;
+
+
+	bloom_filter f(tm_, n, nr_probes);
 
 	set<block_address> bs = generate_random_blocks(written_blocks, nr_blocks);
 	set<block_address>::const_iterator it;
@@ -138,8 +206,6 @@ TEST_F(BloomFilterTests, count_false_positives)
 	for (it = bs.begin(); it != bs.end(); ++it)
 		f.set(*it);
 
-	// f.print_debug(cerr);
-
 	unsigned count = 0;
 	for (unsigned i = 0; i < nr_blocks; i++)
 		if (!bs.count(i) && f.test(i))
@@ -150,4 +216,99 @@ TEST_F(BloomFilterTests, count_false_positives)
 	     << "%" << endl;
 }
 
+TEST_F(BloomFilterTests, count_false_positives_with_linear_inserts)
+{
+	block_address nr_blocks = 1 << 25;
+	block_address written_blocks = nr_blocks / 100;
+
+	double fp = 0.001;
+	unsigned nr_probes = 6;
+	unsigned n = calc_nr_bits(fp, written_blocks, nr_probes);
+
+	cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
+
+	double ideal_k = log(2) * ((double) n / (double) written_blocks);
+	cerr << "Ideal k = " << ideal_k << endl;
+
+
+	bloom_filter f(tm_, n, nr_probes);
+
+	set<block_address> bs = generate_linear_blocks(written_blocks, nr_blocks);
+	set<block_address>::const_iterator it;
+
+	for (it = bs.begin(); it != bs.end(); ++it)
+		f.set(*it);
+
+	unsigned count = 0;
+	for (unsigned i = 0; i < nr_blocks; i++)
+		if (!bs.count(i) && f.test(i))
+			count++;
+
+	double actual_fp = static_cast<double>(count) / static_cast<double>(nr_blocks);
+
+	ASSERT_THAT(actual_fp, Lt(fp));
+
+	cerr << count << " false positives out of " << nr_blocks << ", "
+	     << actual_fp * 100.0 << "%" << endl;
+}
+
+TEST_F(BloomFilterTests, false_positives_over_multiple_eras)
+{
+	unsigned nr_eras = 10;
+	block_address nr_blocks = 1 << 20;
+	block_address written_blocks = nr_blocks / nr_eras;
+
+	double fp = 0.001;
+	unsigned nr_probes = 6;
+	unsigned n = calc_nr_bits(fp, written_blocks, nr_probes);
+
+	cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
+
+	double ideal_k = log(2) * ((double) n / (double) written_blocks);
+	cerr << "Ideal k = " << ideal_k << endl;
+
+	vector<set<block_address> > writes(nr_eras);
+	vector<bloom_filter::ptr> filters(nr_eras);
+
+	for (unsigned era = 0; era < writes.size(); era++) {
+		cerr << "inserting era " << era << endl;
+
+		writes[era] = generate_random_blocks(written_blocks, nr_blocks);
+		set<block_address> const &bs = writes[era];
+
+		filters[era] = bloom_filter::ptr(new bloom_filter(tm_, n, nr_probes));
+		bloom_filter::ptr &f = filters[era];
+
+		set<block_address>::const_iterator it;
+		for (it = bs.begin(); it != bs.end(); ++it)
+			f->set(*it);
+	}
+
+	set<block_address> write_sum;
+	set<block_address> filter_sum;
+	for (unsigned era_plus_1 = writes.size(); era_plus_1 > 0; era_plus_1--) {
+		unsigned era = era_plus_1 - 1;
+
+		set<block_address> const &era_writes = writes[era];
+		write_sum.insert(era_writes.begin(), era_writes.end());
+
+		for (unsigned i = 0; i < nr_blocks; i++)
+			if (filters[era]->test(i))
+				filter_sum.insert(i);
+
+		unsigned count = 0;
+		for (unsigned i = 0; i < nr_blocks; i++) {
+			if (write_sum.count(i) > 0)
+				ASSERT_THAT(filter_sum.count(i), Gt(0ull));
+
+			else if (filter_sum.count(i))
+				count++;
+		}
+
+		cerr << "blocks >= era " << era << ", false positives = "
+		     << static_cast<double>(count * 100) / static_cast<double>(nr_blocks)
+		     << "%" << endl;
+	}
+}
+
 //----------------------------------------------------------------

unit-tests/btree_counter_t.cc

@@ -5,6 +5,7 @@
 #include "persistent-data/data-structures/btree.h"
 #include "persistent-data/data-structures/btree_counter.h"
 #include "persistent-data/space-maps/core.h"
+#include "persistent-data/data-structures/simple_traits.h"
 
 using namespace base;
 using namespace std;

unit-tests/btree_t.cc

@@ -20,6 +20,7 @@
 #include "persistent-data/transaction_manager.h"
 #include "persistent-data/space-maps/core.h"
 #include "persistent-data/data-structures/btree.h"
+#include "persistent-data/data-structures/simple_traits.h"
 
 using namespace std;
 using namespace persistent_data;