[era] switch from bloom filters to writesets
This commit is contained in:
parent
e7bbfbc1fd
commit
8531a2befa
39
Makefile.in
39
Makefile.in
@ -27,6 +27,7 @@ PROGRAMS=\
|
|||||||
\
|
\
|
||||||
era_check \
|
era_check \
|
||||||
era_dump \
|
era_dump \
|
||||||
|
era_invalidate \
|
||||||
\
|
\
|
||||||
thin_check \
|
thin_check \
|
||||||
thin_dump \
|
thin_dump \
|
||||||
@ -53,7 +54,7 @@ SOURCE=\
|
|||||||
era/era_array.cc \
|
era/era_array.cc \
|
||||||
era/era_detail.cc \
|
era/era_detail.cc \
|
||||||
era/superblock.cc \
|
era/superblock.cc \
|
||||||
era/bloom_tree.cc \
|
era/writeset_tree.cc \
|
||||||
era/metadata.cc \
|
era/metadata.cc \
|
||||||
era/metadata_dump.cc \
|
era/metadata_dump.cc \
|
||||||
era/xml_format.cc \
|
era/xml_format.cc \
|
||||||
@ -305,7 +306,7 @@ ERA_CHECK_SOURCE=\
|
|||||||
base/error_state.cc \
|
base/error_state.cc \
|
||||||
base/endian_utils.cc \
|
base/endian_utils.cc \
|
||||||
\
|
\
|
||||||
era/bloom_tree.cc \
|
era/writeset_tree.cc \
|
||||||
era/era_detail.cc \
|
era/era_detail.cc \
|
||||||
era/era_array.cc \
|
era/era_array.cc \
|
||||||
era/metadata.cc \
|
era/metadata.cc \
|
||||||
@ -335,7 +336,7 @@ ERA_DUMP_SOURCE=\
|
|||||||
base/error_state.cc \
|
base/error_state.cc \
|
||||||
base/endian_utils.cc \
|
base/endian_utils.cc \
|
||||||
\
|
\
|
||||||
era/bloom_tree.cc \
|
era/writeset_tree.cc \
|
||||||
era/era_detail.cc \
|
era/era_detail.cc \
|
||||||
era/era_array.cc \
|
era/era_array.cc \
|
||||||
era/metadata.cc \
|
era/metadata.cc \
|
||||||
@ -362,6 +363,38 @@ era_dump: $(ERA_DUMP_OBJECTS) era/era_dump.o
|
|||||||
@echo " [LD] $@"
|
@echo " [LD] $@"
|
||||||
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
|
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
|
||||||
|
|
||||||
|
ERA_INVALIDATE_SOURCE=\
|
||||||
|
base/base64.cc \
|
||||||
|
base/error_state.cc \
|
||||||
|
base/endian_utils.cc \
|
||||||
|
\
|
||||||
|
era/writeset_tree.cc \
|
||||||
|
era/era_detail.cc \
|
||||||
|
era/era_array.cc \
|
||||||
|
era/metadata.cc \
|
||||||
|
era/metadata_dump.cc \
|
||||||
|
era/superblock.cc \
|
||||||
|
era/xml_format.cc \
|
||||||
|
\
|
||||||
|
persistent-data/checksum.cc \
|
||||||
|
persistent-data/error_set.cc \
|
||||||
|
persistent-data/file_utils.cc \
|
||||||
|
persistent-data/hex_dump.cc \
|
||||||
|
persistent-data/lock_tracker.cc \
|
||||||
|
persistent-data/data-structures/btree.cc \
|
||||||
|
persistent-data/data-structures/bitset.cc \
|
||||||
|
persistent-data/space_map.cc \
|
||||||
|
persistent-data/space-maps/disk.cc \
|
||||||
|
persistent-data/space-maps/recursive.cc \
|
||||||
|
persistent-data/space-maps/careful_alloc.cc \
|
||||||
|
persistent-data/transaction_manager.cc \
|
||||||
|
|
||||||
|
ERA_INVALIDATE_OBJECTS=$(subst .cc,.o,$(ERA_INVALIDATE_SOURCE))
|
||||||
|
|
||||||
|
era_invalidate: $(ERA_INVALIDATE_OBJECTS) era/era_invalidate.o
|
||||||
|
@echo " [LD] $@"
|
||||||
|
$(V) $(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $+ $(LIBS)
|
||||||
|
|
||||||
#----------------------------------------------------------------
|
#----------------------------------------------------------------
|
||||||
|
|
||||||
DEPEND_FILES=\
|
DEPEND_FILES=\
|
||||||
|
@ -20,9 +20,9 @@ namespace era {
|
|||||||
uint32_t current_era) = 0;
|
uint32_t current_era) = 0;
|
||||||
virtual void end_superblock() = 0;
|
virtual void end_superblock() = 0;
|
||||||
|
|
||||||
virtual void begin_bloom(uint32_t era, uint32_t nr_bits, pd::block_address nr_blocks) = 0;
|
virtual void begin_writeset(uint32_t era, uint32_t nr_bits) = 0;
|
||||||
virtual void bloom_bit(uint32_t bit, bool value) = 0;
|
virtual void writeset_bit(uint32_t bit, bool value) = 0;
|
||||||
virtual void end_bloom() = 0;
|
virtual void end_writeset() = 0;
|
||||||
|
|
||||||
virtual void begin_era_array() = 0;
|
virtual void begin_era_array() = 0;
|
||||||
virtual void era(pd::block_address block, uint32_t era) = 0;
|
virtual void era(pd::block_address block, uint32_t era) = 0;
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
#include "base/error_state.h"
|
#include "base/error_state.h"
|
||||||
#include "base/nested_output.h"
|
#include "base/nested_output.h"
|
||||||
#include "era/bloom_tree.h"
|
#include "era/writeset_tree.h"
|
||||||
#include "era/era_array.h"
|
#include "era/era_array.h"
|
||||||
#include "era/superblock.h"
|
#include "era/superblock.h"
|
||||||
#include "persistent-data/block.h"
|
#include "persistent-data/block.h"
|
||||||
@ -92,14 +92,14 @@ namespace {
|
|||||||
using reporter_base::get_error;
|
using reporter_base::get_error;
|
||||||
};
|
};
|
||||||
|
|
||||||
class bloom_tree_reporter : public bloom_tree_detail::damage_visitor, reporter_base {
|
class writeset_tree_reporter : public writeset_tree_detail::damage_visitor, reporter_base {
|
||||||
public:
|
public:
|
||||||
bloom_tree_reporter(nested_output &o)
|
writeset_tree_reporter(nested_output &o)
|
||||||
: reporter_base(o) {
|
: reporter_base(o) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(bloom_tree_detail::missing_eras const &d) {
|
void visit(writeset_tree_detail::missing_eras const &d) {
|
||||||
out() << "missing eras from bloom tree" << end_message();
|
out() << "missing eras from writeset tree" << end_message();
|
||||||
{
|
{
|
||||||
nested_output::nest _ = push();
|
nested_output::nest _ = push();
|
||||||
out() << d.get_desc() << end_message();
|
out() << d.get_desc() << end_message();
|
||||||
@ -110,8 +110,8 @@ namespace {
|
|||||||
mplus_error(FATAL);
|
mplus_error(FATAL);
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
|
void visit(writeset_tree_detail::damaged_writeset const &d) {
|
||||||
out() << "damaged bloom filter" << end_message();
|
out() << "damaged writeset" << end_message();
|
||||||
{
|
{
|
||||||
nested_output::nest _ = push();
|
nested_output::nest _ = push();
|
||||||
out() << d.get_desc() << end_message();
|
out() << d.get_desc() << end_message();
|
||||||
@ -216,22 +216,22 @@ namespace {
|
|||||||
superblock sb = read_superblock(bm);
|
superblock sb = read_superblock(bm);
|
||||||
transaction_manager::ptr tm = open_tm(bm);
|
transaction_manager::ptr tm = open_tm(bm);
|
||||||
|
|
||||||
bloom_tree_reporter bt_rep(out);
|
writeset_tree_reporter wt_rep(out);
|
||||||
{
|
{
|
||||||
era_detail_traits::ref_counter rc(tm);
|
era_detail_traits::ref_counter rc(tm);
|
||||||
bloom_tree bt(tm, sb.bloom_tree_root, rc);
|
writeset_tree wt(tm, sb.writeset_tree_root, rc);
|
||||||
check_bloom_tree(tm, bt, bt_rep);
|
check_writeset_tree(tm, wt, wt_rep);
|
||||||
}
|
}
|
||||||
|
|
||||||
era_array_reporter ea_rep(out);
|
era_array_reporter ea_rep(out);
|
||||||
{
|
{
|
||||||
uint32_traits::ref_counter rc;
|
uint32_traits::ref_counter rc;
|
||||||
era_array ea(tm, rc, sb.era_array_root, sb.nr_blocks);
|
era_array ea(tm, rc, sb.era_array_root, sb.nr_blocks);
|
||||||
check_era_array(ea, ea_rep);
|
check_era_array(ea, sb.current_era, ea_rep);
|
||||||
}
|
}
|
||||||
|
|
||||||
return combine_errors(sb_rep.get_error(),
|
return combine_errors(sb_rep.get_error(),
|
||||||
combine_errors(bt_rep.get_error(),
|
combine_errors(wt_rep.get_error(),
|
||||||
ea_rep.get_error()));
|
ea_rep.get_error()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -7,37 +7,18 @@ using namespace era;
|
|||||||
|
|
||||||
//----------------------------------------------------------------
|
//----------------------------------------------------------------
|
||||||
|
|
||||||
namespace {
|
|
||||||
#if 0
|
|
||||||
le32 pack_hash_detail(uint32_t hash1, uint32_t hash2, uint32_t nr_probes) {
|
|
||||||
throw std::runtime_error("not implemented");
|
|
||||||
}
|
|
||||||
|
|
||||||
void unpack_hash_detail(le32 packed, uint32_t &hash1, uint32_t &hash2, uint32_t &nr_probes) {
|
|
||||||
throw std::runtime_error("not implemented");
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
void
|
void
|
||||||
era_detail_traits::unpack(disk_type const &disk, value_type &value)
|
era_detail_traits::unpack(disk_type const &disk, value_type &value)
|
||||||
{
|
{
|
||||||
value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
|
|
||||||
value.nr_bits = to_cpu<uint32_t>(disk.nr_bits);
|
value.nr_bits = to_cpu<uint32_t>(disk.nr_bits);
|
||||||
value.nr_set = to_cpu<uint32_t>(disk.nr_set);
|
value.writeset_root = to_cpu<uint64_t>(disk.writeset_root);
|
||||||
value.bloom_root = to_cpu<uint64_t>(disk.bloom_root);
|
|
||||||
|
|
||||||
//unpack_hash_detail(disk.hash_fns_and_probes, value.hash1, value.hash2, value.nr_probes);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
era_detail_traits::pack(value_type const &value, disk_type &disk)
|
era_detail_traits::pack(value_type const &value, disk_type &disk)
|
||||||
{
|
{
|
||||||
disk.nr_blocks = to_disk<le32>(value.nr_blocks);
|
|
||||||
disk.nr_bits = to_disk<le32>(value.nr_bits);
|
disk.nr_bits = to_disk<le32>(value.nr_bits);
|
||||||
disk.nr_set = to_disk<le32>(value.nr_set);
|
disk.writeset_root = to_disk<le64>(value.writeset_root);
|
||||||
disk.bloom_root = to_disk<le64>(value.bloom_root);
|
|
||||||
// disk.hash_fns_and_probes = pack_hash_detail(value.hash1, value.hash2, value.nr_probes);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//----------------------------------------------------------------
|
//----------------------------------------------------------------
|
||||||
|
@ -8,22 +8,13 @@
|
|||||||
|
|
||||||
namespace era {
|
namespace era {
|
||||||
struct era_detail_disk {
|
struct era_detail_disk {
|
||||||
base::le32 nr_blocks;
|
|
||||||
base::le32 nr_bits;
|
base::le32 nr_bits;
|
||||||
base::le32 nr_set;
|
base::le64 writeset_root;
|
||||||
base::le64 bloom_root;
|
|
||||||
} __attribute__ ((packed));
|
} __attribute__ ((packed));
|
||||||
|
|
||||||
struct era_detail {
|
struct era_detail {
|
||||||
uint32_t nr_blocks;
|
|
||||||
uint32_t nr_bits;
|
uint32_t nr_bits;
|
||||||
uint32_t nr_set;
|
uint64_t writeset_root;
|
||||||
|
|
||||||
uint32_t hash1;
|
|
||||||
uint32_t hash2;
|
|
||||||
uint32_t nr_probes;
|
|
||||||
|
|
||||||
uint64_t bloom_root;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// FIXME: implement
|
// FIXME: implement
|
||||||
|
@ -5,7 +5,7 @@
|
|||||||
|
|
||||||
#include "version.h"
|
#include "version.h"
|
||||||
#include "era/era_array.h"
|
#include "era/era_array.h"
|
||||||
#include "era/bloom_tree.h"
|
#include "era/writeset_tree.h"
|
||||||
#include "era/metadata.h"
|
#include "era/metadata.h"
|
||||||
#include "era/metadata_dump.h"
|
#include "era/metadata_dump.h"
|
||||||
#include "era/xml_format.h"
|
#include "era/xml_format.h"
|
||||||
|
114
era/era_invalidate.cc
Normal file
114
era/era_invalidate.cc
Normal file
@ -0,0 +1,114 @@
|
|||||||
|
#include <fstream>
|
||||||
|
#include <getopt.h>
|
||||||
|
#include <libgen.h>
|
||||||
|
#include <iostream>
|
||||||
|
|
||||||
|
#include "version.h"
|
||||||
|
#include "era/era_array.h"
|
||||||
|
#include "era/writeset_tree.h"
|
||||||
|
#include "era/metadata.h"
|
||||||
|
#include "era/xml_format.h"
|
||||||
|
#include "persistent-data/file_utils.h"
|
||||||
|
|
||||||
|
using namespace era;
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
//----------------------------------------------------------------
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
struct flags {
|
||||||
|
flags() {
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
void emit_blocks() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
//--------------------------------
|
||||||
|
|
||||||
|
string const STDOUT_PATH("-");
|
||||||
|
|
||||||
|
bool want_stdout(string const &output) {
|
||||||
|
return output == STDOUT_PATH;
|
||||||
|
}
|
||||||
|
|
||||||
|
int invalidate(string const &dev, string const &output, flags const &fs) {
|
||||||
|
try {
|
||||||
|
block_manager<>::ptr bm = open_bm(dev, block_io<>::READ_ONLY);
|
||||||
|
metadata::ptr md(new metadata(bm, metadata::OPEN));
|
||||||
|
|
||||||
|
if (want_stdout(output)) {
|
||||||
|
emitter::ptr e = create_xml_emitter(cout);
|
||||||
|
//emit_blocks(md, e, fs);
|
||||||
|
} else {
|
||||||
|
ofstream out(output.c_str());
|
||||||
|
emitter::ptr e = create_xml_emitter(out);
|
||||||
|
//emit_blocks(md, e, fs);
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (std::exception &e) {
|
||||||
|
cerr << e.what() << endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void usage(ostream &out, string const &cmd) {
|
||||||
|
out << "Usage: " << cmd << " [options] {device|file}" << endl
|
||||||
|
<< "Options:" << endl
|
||||||
|
<< " {-h|--help}" << endl
|
||||||
|
<< " {-o <xml file>}" << endl
|
||||||
|
<< " {-V|--version}" << endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//----------------------------------------------------------------
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
int c;
|
||||||
|
flags fs;
|
||||||
|
string output("-");
|
||||||
|
char const shortopts[] = "ho:V";
|
||||||
|
|
||||||
|
option const longopts[] = {
|
||||||
|
{ "help", no_argument, NULL, 'h' },
|
||||||
|
{ "output", required_argument, NULL, 'o' },
|
||||||
|
{ "version", no_argument, NULL, 'V' },
|
||||||
|
{ NULL, no_argument, NULL, 0 }
|
||||||
|
};
|
||||||
|
|
||||||
|
while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) {
|
||||||
|
switch(c) {
|
||||||
|
case 'h':
|
||||||
|
usage(cout, basename(argv[0]));
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
case 'o':
|
||||||
|
output = optarg;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'V':
|
||||||
|
cout << THIN_PROVISIONING_TOOLS_VERSION << endl;
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
default:
|
||||||
|
usage(cerr, basename(argv[0]));
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (argc == optind) {
|
||||||
|
cerr << "No input file provided." << endl;
|
||||||
|
usage(cerr, basename(argv[0]));
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return invalidate(argv[optind], output, fs);
|
||||||
|
}
|
||||||
|
|
||||||
|
//----------------------------------------------------------------
|
@ -47,9 +47,9 @@ metadata::open_metadata(block_manager<>::ptr bm)
|
|||||||
tm_ = open_tm(bm);
|
tm_ = open_tm(bm);
|
||||||
sb_ = read_superblock(tm_->get_bm());
|
sb_ = read_superblock(tm_->get_bm());
|
||||||
|
|
||||||
bloom_tree_ = bloom_tree::ptr(new bloom_tree(tm_,
|
writeset_tree_ = writeset_tree::ptr(new writeset_tree(tm_,
|
||||||
sb_.bloom_tree_root,
|
sb_.writeset_tree_root,
|
||||||
era_detail_traits::ref_counter(tm_)));
|
era_detail_traits::ref_counter(tm_)));
|
||||||
|
|
||||||
era_array_ = era_array::ptr(new era_array(tm_,
|
era_array_ = era_array::ptr(new era_array(tm_,
|
||||||
uint32_traits::ref_counter(),
|
uint32_traits::ref_counter(),
|
||||||
|
@ -10,7 +10,7 @@
|
|||||||
#include "persistent-data/transaction_manager.h"
|
#include "persistent-data/transaction_manager.h"
|
||||||
|
|
||||||
#include "era/superblock.h"
|
#include "era/superblock.h"
|
||||||
#include "era/bloom_tree.h"
|
#include "era/writeset_tree.h"
|
||||||
#include "era/era_array.h"
|
#include "era/era_array.h"
|
||||||
|
|
||||||
//----------------------------------------------------------------
|
//----------------------------------------------------------------
|
||||||
@ -34,7 +34,7 @@ namespace era {
|
|||||||
tm::ptr tm_;
|
tm::ptr tm_;
|
||||||
superblock sb_;
|
superblock sb_;
|
||||||
checked_space_map::ptr metadata_sm_;
|
checked_space_map::ptr metadata_sm_;
|
||||||
bloom_tree::ptr bloom_tree_;
|
writeset_tree::ptr writeset_tree_;
|
||||||
era_array::ptr era_array_;
|
era_array::ptr era_array_;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
@ -17,42 +17,42 @@ namespace {
|
|||||||
"perhaps you wanted to run with --repair ?");
|
"perhaps you wanted to run with --repair ?");
|
||||||
}
|
}
|
||||||
|
|
||||||
class bloom_tree_emitter : public bloom_tree_detail::bloom_visitor {
|
class writeset_tree_emitter : public writeset_tree_detail::writeset_visitor {
|
||||||
public:
|
public:
|
||||||
bloom_tree_emitter(emitter::ptr e)
|
writeset_tree_emitter(emitter::ptr e)
|
||||||
: e_(e) {
|
: e_(e) {
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual void bloom_begin(uint32_t era, uint32_t nr_blocks, uint32_t nr_bits, uint32_t nr_set) {
|
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) {
|
||||||
e_->begin_bloom(era, nr_bits, nr_blocks);
|
e_->begin_writeset(era, nr_bits);
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual void bit(uint32_t bit, bool value) {
|
virtual void bit(uint32_t bit, bool value) {
|
||||||
e_->bloom_bit(bit, value);
|
e_->writeset_bit(bit, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual void bloom_end() {
|
virtual void writeset_end() {
|
||||||
e_->end_bloom();
|
e_->end_writeset();
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
emitter::ptr e_;
|
emitter::ptr e_;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ignore_bloom_tree_damage : public bloom_tree_detail::damage_visitor {
|
struct ignore_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
|
||||||
void visit(bloom_tree_detail::missing_eras const &d) {
|
void visit(writeset_tree_detail::missing_eras const &d) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
|
void visit(writeset_tree_detail::damaged_writeset const &d) {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
struct fatal_bloom_tree_damage : public bloom_tree_detail::damage_visitor {
|
struct fatal_writeset_tree_damage : public writeset_tree_detail::damage_visitor {
|
||||||
void visit(bloom_tree_detail::missing_eras const &d) {
|
void visit(writeset_tree_detail::missing_eras const &d) {
|
||||||
raise_metadata_damage();
|
raise_metadata_damage();
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(bloom_tree_detail::damaged_bloom_filter const &d) {
|
void visit(writeset_tree_detail::damaged_writeset const &d) {
|
||||||
raise_metadata_damage();
|
raise_metadata_damage();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -104,15 +104,15 @@ era::metadata_dump(metadata::ptr md, emitter::ptr e, bool repair)
|
|||||||
sb.current_era);
|
sb.current_era);
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
bloom_tree_emitter visitor(e);
|
writeset_tree_emitter visitor(e);
|
||||||
|
|
||||||
ignore_bloom_tree_damage ignore;
|
ignore_writeset_tree_damage ignore;
|
||||||
fatal_bloom_tree_damage fatal;
|
fatal_writeset_tree_damage fatal;
|
||||||
bloom_tree_detail::damage_visitor &dv = repair ?
|
writeset_tree_detail::damage_visitor &dv = repair ?
|
||||||
static_cast<bloom_tree_detail::damage_visitor &>(ignore) :
|
static_cast<writeset_tree_detail::damage_visitor &>(ignore) :
|
||||||
static_cast<bloom_tree_detail::damage_visitor &>(fatal);
|
static_cast<writeset_tree_detail::damage_visitor &>(fatal);
|
||||||
|
|
||||||
walk_bloom_tree(md->tm_, *md->bloom_tree_, visitor, dv);
|
walk_writeset_tree(md->tm_, *md->writeset_tree_, visitor, dv);
|
||||||
}
|
}
|
||||||
|
|
||||||
e->begin_era_array();
|
e->begin_era_array();
|
||||||
|
@ -34,7 +34,7 @@ namespace {
|
|||||||
le32 current_era;
|
le32 current_era;
|
||||||
era_detail_disk current_detail;
|
era_detail_disk current_detail;
|
||||||
|
|
||||||
le64 bloom_tree_root;
|
le64 writeset_tree_root;
|
||||||
le64 era_array_root;
|
le64 era_array_root;
|
||||||
|
|
||||||
} __attribute__ ((packed));
|
} __attribute__ ((packed));
|
||||||
@ -116,7 +116,7 @@ superblock::superblock()
|
|||||||
metadata_block_size(8),
|
metadata_block_size(8),
|
||||||
nr_blocks(0),
|
nr_blocks(0),
|
||||||
current_era(0),
|
current_era(0),
|
||||||
bloom_tree_root(0),
|
writeset_tree_root(0),
|
||||||
era_array_root(0)
|
era_array_root(0)
|
||||||
{
|
{
|
||||||
memset(uuid, 0, sizeof(uuid));
|
memset(uuid, 0, sizeof(uuid));
|
||||||
@ -141,7 +141,7 @@ superblock_traits::unpack(disk_type const &disk, value_type &value)
|
|||||||
value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
|
value.nr_blocks = to_cpu<uint32_t>(disk.nr_blocks);
|
||||||
value.current_era = to_cpu<uint32_t>(disk.current_era);
|
value.current_era = to_cpu<uint32_t>(disk.current_era);
|
||||||
era_detail_traits::unpack(disk.current_detail, value.current_detail);
|
era_detail_traits::unpack(disk.current_detail, value.current_detail);
|
||||||
value.bloom_tree_root = to_cpu<uint64_t>(disk.bloom_tree_root);
|
value.writeset_tree_root = to_cpu<uint64_t>(disk.writeset_tree_root);
|
||||||
value.era_array_root = to_cpu<uint64_t>(disk.era_array_root);
|
value.era_array_root = to_cpu<uint64_t>(disk.era_array_root);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -161,7 +161,7 @@ superblock_traits::pack(value_type const &value, disk_type &disk)
|
|||||||
disk.nr_blocks = to_disk<le32>(value.nr_blocks);
|
disk.nr_blocks = to_disk<le32>(value.nr_blocks);
|
||||||
disk.current_era = to_disk<le32>(value.current_era);
|
disk.current_era = to_disk<le32>(value.current_era);
|
||||||
era_detail_traits::pack(value.current_detail, disk.current_detail);
|
era_detail_traits::pack(value.current_detail, disk.current_detail);
|
||||||
disk.bloom_tree_root = to_disk<le64>(value.bloom_tree_root);
|
disk.writeset_tree_root = to_disk<le64>(value.writeset_tree_root);
|
||||||
disk.era_array_root = to_disk<le64>(value.era_array_root);
|
disk.era_array_root = to_disk<le64>(value.era_array_root);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -282,8 +282,8 @@ era::check_superblock(superblock const &sb,
|
|||||||
visitor.visit(superblock_invalid(msg.str()));
|
visitor.visit(superblock_invalid(msg.str()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sb.bloom_tree_root == SUPERBLOCK_LOCATION) {
|
if (sb.writeset_tree_root == SUPERBLOCK_LOCATION) {
|
||||||
string msg("bloom tree root points back to the superblock");
|
string msg("writeset tree root points back to the superblock");
|
||||||
visitor.visit(superblock_invalid(msg));
|
visitor.visit(superblock_invalid(msg));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -292,9 +292,9 @@ era::check_superblock(superblock const &sb,
|
|||||||
visitor.visit(superblock_invalid(msg));
|
visitor.visit(superblock_invalid(msg));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sb.bloom_tree_root == sb.era_array_root) {
|
if (sb.writeset_tree_root == sb.era_array_root) {
|
||||||
ostringstream msg;
|
ostringstream msg;
|
||||||
msg << "bloom tree root and era array both point to the same block: "
|
msg << "writeset tree root and era array both point to the same block: "
|
||||||
<< sb.era_array_root;
|
<< sb.era_array_root;
|
||||||
visitor.visit(superblock_invalid(msg.str()));
|
visitor.visit(superblock_invalid(msg.str()));
|
||||||
}
|
}
|
||||||
|
@ -59,7 +59,7 @@ namespace era {
|
|||||||
era_detail current_detail;
|
era_detail current_detail;
|
||||||
|
|
||||||
// A btree of undigested era_details
|
// A btree of undigested era_details
|
||||||
uint64_t bloom_tree_root;
|
uint64_t writeset_tree_root;
|
||||||
|
|
||||||
// Big array holding the digested era/block info.
|
// Big array holding the digested era/block info.
|
||||||
uint64_t era_array_root;
|
uint64_t era_array_root;
|
||||||
|
@ -1,10 +1,10 @@
|
|||||||
#include "era/bloom_tree.h"
|
#include "era/writeset_tree.h"
|
||||||
#include "persistent-data/data-structures/btree_damage_visitor.h"
|
#include "persistent-data/data-structures/btree_damage_visitor.h"
|
||||||
#include "persistent-data/data-structures/bitset.h"
|
#include "persistent-data/data-structures/bitset.h"
|
||||||
|
|
||||||
using namespace boost;
|
using namespace boost;
|
||||||
using namespace era;
|
using namespace era;
|
||||||
using namespace bloom_tree_detail;
|
using namespace writeset_tree_detail;
|
||||||
using namespace persistent_data;
|
using namespace persistent_data;
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
@ -22,9 +22,9 @@ missing_eras::visit(damage_visitor &v) const {
|
|||||||
v.visit(*this);
|
v.visit(*this);
|
||||||
}
|
}
|
||||||
|
|
||||||
damaged_bloom_filter::damaged_bloom_filter(string const &desc,
|
damaged_writeset::damaged_writeset(string const &desc,
|
||||||
uint32_t era,
|
uint32_t era,
|
||||||
run<uint32_t> missing_bits)
|
run<uint32_t> missing_bits)
|
||||||
: damage(desc),
|
: damage(desc),
|
||||||
era_(era),
|
era_(era),
|
||||||
missing_bits_(missing_bits)
|
missing_bits_(missing_bits)
|
||||||
@ -32,7 +32,7 @@ damaged_bloom_filter::damaged_bloom_filter(string const &desc,
|
|||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
damaged_bloom_filter::visit(damage_visitor &v) const
|
damaged_writeset::visit(damage_visitor &v) const
|
||||||
{
|
{
|
||||||
v.visit(*this);
|
v.visit(*this);
|
||||||
}
|
}
|
||||||
@ -40,39 +40,39 @@ damaged_bloom_filter::visit(damage_visitor &v) const
|
|||||||
//----------------------------------------------------------------
|
//----------------------------------------------------------------
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
class ll_bloom_visitor : public bitset_detail::bitset_visitor {
|
class ll_writeset_visitor : public bitset_detail::bitset_visitor {
|
||||||
public:
|
public:
|
||||||
typedef persistent_data::transaction_manager::ptr tm_ptr;
|
typedef persistent_data::transaction_manager::ptr tm_ptr;
|
||||||
|
|
||||||
ll_bloom_visitor(tm_ptr tm,
|
ll_writeset_visitor(tm_ptr tm,
|
||||||
bloom_tree_detail::bloom_visitor &bloom_v,
|
writeset_tree_detail::writeset_visitor &writeset_v,
|
||||||
bloom_tree_detail::damage_visitor &dv)
|
writeset_tree_detail::damage_visitor &dv)
|
||||||
: tm_(tm),
|
: tm_(tm),
|
||||||
bloom_v_(bloom_v),
|
writeset_v_(writeset_v),
|
||||||
dv_(dv) {
|
dv_(dv) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(btree_path const &path, era_detail const &era) {
|
void visit(btree_path const &path, era_detail const &era) {
|
||||||
era_ = path[0];
|
era_ = path[0];
|
||||||
bitset bs(tm_, era.bloom_root, era.nr_bits);
|
bitset bs(tm_, era.writeset_root, era.nr_bits);
|
||||||
bloom_v_.bloom_begin(era_, era.nr_blocks, era.nr_bits, era.nr_set);
|
writeset_v_.writeset_begin(era_, era.nr_bits);
|
||||||
bs.walk_bitset(*this);
|
bs.walk_bitset(*this);
|
||||||
bloom_v_.bloom_end();
|
writeset_v_.writeset_end();
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(uint32_t index, bool value) {
|
void visit(uint32_t index, bool value) {
|
||||||
bloom_v_.bit(index, value);
|
writeset_v_.bit(index, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(bitset_detail::missing_bits const &d) {
|
void visit(bitset_detail::missing_bits const &d) {
|
||||||
dv_.visit(bloom_tree_detail::damaged_bloom_filter("missing bits", era_, d.keys_));
|
dv_.visit(writeset_tree_detail::damaged_writeset("missing bits", era_, d.keys_));
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
tm_ptr tm_;
|
tm_ptr tm_;
|
||||||
uint64_t era_;
|
uint64_t era_;
|
||||||
bloom_tree_detail::bloom_visitor &bloom_v_;
|
writeset_tree_detail::writeset_visitor &writeset_v_;
|
||||||
bloom_tree_detail::damage_visitor &dv_;
|
writeset_tree_detail::damage_visitor &dv_;
|
||||||
};
|
};
|
||||||
|
|
||||||
class ll_damage_visitor {
|
class ll_damage_visitor {
|
||||||
@ -98,38 +98,37 @@ namespace {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
era::walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
|
era::walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
|
||||||
bloom_tree const &tree,
|
writeset_tree const &tree,
|
||||||
bloom_tree_detail::bloom_visitor &bloom_v,
|
writeset_tree_detail::writeset_visitor &writeset_v,
|
||||||
bloom_tree_detail::damage_visitor &dv)
|
writeset_tree_detail::damage_visitor &dv)
|
||||||
{
|
{
|
||||||
ll_bloom_visitor ll_bv(tm, bloom_v, dv);
|
ll_writeset_visitor ll_bv(tm, writeset_v, dv);
|
||||||
ll_damage_visitor ll_dv(dv);
|
ll_damage_visitor ll_dv(dv);
|
||||||
btree_visit_values(tree, ll_bv, ll_dv);
|
btree_visit_values(tree, ll_bv, ll_dv);
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
class noop_bloom_visitor : public bloom_tree_detail::bloom_visitor {
|
class noop_writeset_visitor : public writeset_tree_detail::writeset_visitor {
|
||||||
public:
|
public:
|
||||||
void bloom_begin(uint32_t era, uint32_t nr_blocks,
|
void writeset_begin(uint32_t era, uint32_t nr_bits) {
|
||||||
uint32_t nr_bits, uint32_t nr_set) {
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void bit(uint32_t index, bool value) {
|
void bit(uint32_t index, bool value) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void bloom_end() {
|
void writeset_end() {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
void
|
void
|
||||||
era::check_bloom_tree(persistent_data::transaction_manager::ptr tm,
|
era::check_writeset_tree(persistent_data::transaction_manager::ptr tm,
|
||||||
bloom_tree const &tree,
|
writeset_tree const &tree,
|
||||||
bloom_tree_detail::damage_visitor &dv)
|
writeset_tree_detail::damage_visitor &dv)
|
||||||
{
|
{
|
||||||
noop_bloom_visitor bv;
|
noop_writeset_visitor bv;
|
||||||
walk_bloom_tree(tm, tree, bv, dv);
|
walk_writeset_tree(tm, tree, bv, dv);
|
||||||
}
|
}
|
||||||
|
|
||||||
//----------------------------------------------------------------
|
//----------------------------------------------------------------
|
@ -1,5 +1,5 @@
|
|||||||
#ifndef ERA_BLOOM_TREE_H
|
#ifndef ERA_WRITESET_TREE_H
|
||||||
#define ERA_BLOOM_TREE_H
|
#define ERA_WRITESET_TREE_H
|
||||||
|
|
||||||
#include "era/era_detail.h"
|
#include "era/era_detail.h"
|
||||||
#include "persistent-data/data-structures/btree.h"
|
#include "persistent-data/data-structures/btree.h"
|
||||||
@ -7,7 +7,7 @@
|
|||||||
//----------------------------------------------------------------
|
//----------------------------------------------------------------
|
||||||
|
|
||||||
namespace era {
|
namespace era {
|
||||||
namespace bloom_tree_detail {
|
namespace writeset_tree_detail {
|
||||||
class damage_visitor;
|
class damage_visitor;
|
||||||
|
|
||||||
class damage {
|
class damage {
|
||||||
@ -34,10 +34,10 @@ namespace era {
|
|||||||
run<uint32_t> eras_;
|
run<uint32_t> eras_;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct damaged_bloom_filter : public damage {
|
struct damaged_writeset : public damage {
|
||||||
damaged_bloom_filter(std::string const &desc,
|
damaged_writeset(std::string const &desc,
|
||||||
uint32_t era,
|
uint32_t era,
|
||||||
run<uint32_t> missing_bits);
|
run<uint32_t> missing_bits);
|
||||||
virtual void visit(damage_visitor &v) const;
|
virtual void visit(damage_visitor &v) const;
|
||||||
|
|
||||||
uint32_t era_;
|
uint32_t era_;
|
||||||
@ -55,32 +55,31 @@ namespace era {
|
|||||||
}
|
}
|
||||||
|
|
||||||
virtual void visit(missing_eras const &d) = 0;
|
virtual void visit(missing_eras const &d) = 0;
|
||||||
virtual void visit(damaged_bloom_filter const &d) = 0;
|
virtual void visit(damaged_writeset const &d) = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
class bloom_visitor {
|
class writeset_visitor {
|
||||||
public:
|
public:
|
||||||
typedef boost::shared_ptr<bloom_visitor> ptr;
|
typedef boost::shared_ptr<writeset_visitor> ptr;
|
||||||
|
|
||||||
virtual ~bloom_visitor() {}
|
virtual ~writeset_visitor() {}
|
||||||
|
|
||||||
virtual void bloom_begin(uint32_t era, uint32_t nr_blocks,
|
virtual void writeset_begin(uint32_t era, uint32_t nr_bits) = 0;
|
||||||
uint32_t nr_bits, uint32_t nr_set) = 0;
|
|
||||||
virtual void bit(uint32_t index, bool value) = 0;
|
virtual void bit(uint32_t index, bool value) = 0;
|
||||||
virtual void bloom_end() = 0;
|
virtual void writeset_end() = 0;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
typedef persistent_data::btree<1, era_detail_traits> bloom_tree;
|
typedef persistent_data::btree<1, era_detail_traits> writeset_tree;
|
||||||
|
|
||||||
void walk_bloom_tree(persistent_data::transaction_manager::ptr tm,
|
void walk_writeset_tree(persistent_data::transaction_manager::ptr tm,
|
||||||
bloom_tree const &tree,
|
writeset_tree const &tree,
|
||||||
bloom_tree_detail::bloom_visitor &bloom_v,
|
writeset_tree_detail::writeset_visitor &writeset_v,
|
||||||
bloom_tree_detail::damage_visitor &dv);
|
writeset_tree_detail::damage_visitor &dv);
|
||||||
|
|
||||||
void check_bloom_tree(persistent_data::transaction_manager::ptr tm,
|
void check_writeset_tree(persistent_data::transaction_manager::ptr tm,
|
||||||
bloom_tree const &tree,
|
writeset_tree const &tree,
|
||||||
bloom_tree_detail::damage_visitor &dv);
|
writeset_tree_detail::damage_visitor &dv);
|
||||||
}
|
}
|
||||||
|
|
||||||
//----------------------------------------------------------------
|
//----------------------------------------------------------------
|
@ -33,25 +33,23 @@ namespace {
|
|||||||
out_ << "</superblock>" << endl;
|
out_ << "</superblock>" << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void begin_bloom(uint32_t era, uint32_t nr_bits,
|
void begin_writeset(uint32_t era, uint32_t nr_bits) {
|
||||||
pd::block_address nr_blocks) {
|
|
||||||
indent();
|
indent();
|
||||||
out_ << "<bloom era=\"" << era << "\""
|
out_ << "<writeset era=\"" << era << "\""
|
||||||
<< " nr_bits=\"" << nr_bits << "\""
|
<< " nr_bits=\"" << nr_bits << "\">" << endl;
|
||||||
<< " nr_blocks=\"" << nr_blocks << "\">" << endl;
|
|
||||||
inc();
|
inc();
|
||||||
}
|
}
|
||||||
|
|
||||||
void bloom_bit(uint32_t bit, bool value) {
|
void writeset_bit(uint32_t bit, bool value) {
|
||||||
indent();
|
indent();
|
||||||
// FIXME: collect all the bits, then uuencode
|
// FIXME: collect all the bits, then uuencode
|
||||||
out_ << "<bit bit=\"" << bit << "\" value=\"" << value << "\">" << endl;
|
out_ << "<bit bit=\"" << bit << "\" value=\"" << value << "\">" << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void end_bloom() {
|
void end_writeset() {
|
||||||
dec();
|
dec();
|
||||||
indent();
|
indent();
|
||||||
out_ << "</bloom>" << endl;
|
out_ << "</writeset>" << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void begin_era_array() {
|
void begin_era_array() {
|
||||||
|
@ -343,7 +343,7 @@ namespace persistent_data {
|
|||||||
unsigned max_concurrent_blocks,
|
unsigned max_concurrent_blocks,
|
||||||
typename block_io<BlockSize>::mode mode)
|
typename block_io<BlockSize>::mode mode)
|
||||||
: io_(new block_io<BlockSize>(path, nr_blocks, mode)),
|
: io_(new block_io<BlockSize>(path, nr_blocks, mode)),
|
||||||
cache_(max(64u, max_concurrent_blocks)),
|
cache_(max(1024u, max_concurrent_blocks)),
|
||||||
tracker_(0, nr_blocks)
|
tracker_(0, nr_blocks)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
@ -20,6 +20,7 @@
|
|||||||
#include "persistent-data/transaction_manager.h"
|
#include "persistent-data/transaction_manager.h"
|
||||||
#include "persistent-data/space-maps/core.h"
|
#include "persistent-data/space-maps/core.h"
|
||||||
#include "persistent-data/data-structures/array.h"
|
#include "persistent-data/data-structures/array.h"
|
||||||
|
#include "persistent-data/data-structures/simple_traits.h"
|
||||||
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
@ -40,17 +40,26 @@ namespace {
|
|||||||
|
|
||||||
using namespace boost::random;
|
using namespace boost::random;
|
||||||
|
|
||||||
mt19937 rng;
|
|
||||||
uniform_int_distribution<uint64_t> uniform_dist(0, max);
|
uniform_int_distribution<uint64_t> uniform_dist(0, max);
|
||||||
|
|
||||||
while (r.size() < count) {
|
while (r.size() < count) {
|
||||||
block_address b = uniform_dist(rng);
|
block_address b = uniform_dist(rng_);
|
||||||
r.insert(b);
|
r.insert(b);
|
||||||
}
|
}
|
||||||
|
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
set<block_address> generate_linear_blocks(unsigned count,
|
||||||
|
block_address max = std::numeric_limits<uint64_t>::max()) {
|
||||||
|
set<block_address> r;
|
||||||
|
|
||||||
|
for (unsigned i = 0; i < count; i++)
|
||||||
|
r.insert(i);
|
||||||
|
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
void commit() {
|
void commit() {
|
||||||
block_manager<>::write_ref superblock(bm_->superblock(SUPERBLOCK));
|
block_manager<>::write_ref superblock(bm_->superblock(SUPERBLOCK));
|
||||||
}
|
}
|
||||||
@ -65,6 +74,8 @@ namespace {
|
|||||||
block_manager<>::ptr bm_;
|
block_manager<>::ptr bm_;
|
||||||
space_map::ptr sm_;
|
space_map::ptr sm_;
|
||||||
transaction_manager::ptr tm_;
|
transaction_manager::ptr tm_;
|
||||||
|
|
||||||
|
boost::random::mt19937 rng_;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -119,18 +130,75 @@ TEST_F(BloomFilterTests, reload_works)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(BloomFilterTests, count_false_positives)
|
unsigned next_power(unsigned n)
|
||||||
{
|
{
|
||||||
block_address nr_blocks = 1024 * 1024;
|
unsigned r = 1;
|
||||||
block_address written_blocks = nr_blocks / 100;
|
while (r < n)
|
||||||
|
r <<= 1;
|
||||||
|
|
||||||
unsigned shift = 1;
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
while ((1ull << shift) < (16 * written_blocks))
|
unsigned calc_nr_bits(double false_positive_rate, unsigned dirty_blocks_per_era, unsigned nr_probes)
|
||||||
shift++;
|
{
|
||||||
cerr << "bitset size: " << ((1 << shift) / (8 * 1024)) << "k" << endl;
|
double k = (double) nr_probes;
|
||||||
|
double kth_root = exp(log(false_positive_rate) / k); // can be precomputed
|
||||||
|
|
||||||
bloom_filter f(tm_, 1 << shift, 6);
|
// FIXME: we need a way to calulate this in kernel? or should we
|
||||||
|
// just pass in the bloom params on the target line?
|
||||||
|
double tmp = log(1.0 - kth_root);
|
||||||
|
double n = (- k * (double) dirty_blocks_per_era) / tmp;
|
||||||
|
|
||||||
|
return next_power(ceil(n));
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned calc_m(double fp, unsigned nr_probes, unsigned n)
|
||||||
|
{
|
||||||
|
double k = (double) nr_probes;
|
||||||
|
double kth_root = exp(log(fp) / k);
|
||||||
|
double tmp = log(1.0 - kth_root);
|
||||||
|
double m = (- ((double) n) / k) * tmp;
|
||||||
|
|
||||||
|
return ceil(m);
|
||||||
|
}
|
||||||
|
|
||||||
|
void print_nr_bits_table(double fp, unsigned nr_probes)
|
||||||
|
{
|
||||||
|
cout << "fp = " << fp << ", k = " << nr_probes << endl;
|
||||||
|
|
||||||
|
for (unsigned long long m = 1024; m < (1ull << 25); m *= 2) {
|
||||||
|
unsigned n = calc_nr_bits(fp, m, nr_probes);
|
||||||
|
unsigned actual_m = calc_m(fp, nr_probes, n);
|
||||||
|
|
||||||
|
cout << " m = " << m << ", n = " << n << ", " << n / (1024 * 8)
|
||||||
|
<< "k, actual_m = " << actual_m << endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not really a test
|
||||||
|
TEST_F(BloomFilterTests, nr_bits_table)
|
||||||
|
{
|
||||||
|
print_nr_bits_table(0.001, 4);
|
||||||
|
print_nr_bits_table(0.001, 6);
|
||||||
|
print_nr_bits_table(0.001, 8);
|
||||||
|
print_nr_bits_table(0.001, 16);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(BloomFilterTests, count_false_positives_with_random_inserts)
|
||||||
|
{
|
||||||
|
block_address nr_blocks = 1 << 27;
|
||||||
|
block_address written_blocks = nr_blocks / 1024;
|
||||||
|
|
||||||
|
unsigned nr_probes = 6;
|
||||||
|
unsigned n = calc_nr_bits(0.001, written_blocks, nr_probes);
|
||||||
|
|
||||||
|
cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
|
||||||
|
|
||||||
|
double ideal_k = log(2) * ((double) n / (double) written_blocks);
|
||||||
|
cerr << "Ideal k = " << ideal_k << endl;
|
||||||
|
|
||||||
|
|
||||||
|
bloom_filter f(tm_, n, nr_probes);
|
||||||
|
|
||||||
set<block_address> bs = generate_random_blocks(written_blocks, nr_blocks);
|
set<block_address> bs = generate_random_blocks(written_blocks, nr_blocks);
|
||||||
set<block_address>::const_iterator it;
|
set<block_address>::const_iterator it;
|
||||||
@ -138,8 +206,6 @@ TEST_F(BloomFilterTests, count_false_positives)
|
|||||||
for (it = bs.begin(); it != bs.end(); ++it)
|
for (it = bs.begin(); it != bs.end(); ++it)
|
||||||
f.set(*it);
|
f.set(*it);
|
||||||
|
|
||||||
// f.print_debug(cerr);
|
|
||||||
|
|
||||||
unsigned count = 0;
|
unsigned count = 0;
|
||||||
for (unsigned i = 0; i < nr_blocks; i++)
|
for (unsigned i = 0; i < nr_blocks; i++)
|
||||||
if (!bs.count(i) && f.test(i))
|
if (!bs.count(i) && f.test(i))
|
||||||
@ -150,4 +216,99 @@ TEST_F(BloomFilterTests, count_false_positives)
|
|||||||
<< "%" << endl;
|
<< "%" << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(BloomFilterTests, count_false_positives_with_linear_inserts)
|
||||||
|
{
|
||||||
|
block_address nr_blocks = 1 << 25;
|
||||||
|
block_address written_blocks = nr_blocks / 100;
|
||||||
|
|
||||||
|
double fp = 0.001;
|
||||||
|
unsigned nr_probes = 6;
|
||||||
|
unsigned n = calc_nr_bits(fp, written_blocks, nr_probes);
|
||||||
|
|
||||||
|
cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
|
||||||
|
|
||||||
|
double ideal_k = log(2) * ((double) n / (double) written_blocks);
|
||||||
|
cerr << "Ideal k = " << ideal_k << endl;
|
||||||
|
|
||||||
|
|
||||||
|
bloom_filter f(tm_, n, nr_probes);
|
||||||
|
|
||||||
|
set<block_address> bs = generate_linear_blocks(written_blocks, nr_blocks);
|
||||||
|
set<block_address>::const_iterator it;
|
||||||
|
|
||||||
|
for (it = bs.begin(); it != bs.end(); ++it)
|
||||||
|
f.set(*it);
|
||||||
|
|
||||||
|
unsigned count = 0;
|
||||||
|
for (unsigned i = 0; i < nr_blocks; i++)
|
||||||
|
if (!bs.count(i) && f.test(i))
|
||||||
|
count++;
|
||||||
|
|
||||||
|
double actual_fp = static_cast<double>(count) / static_cast<double>(nr_blocks);
|
||||||
|
|
||||||
|
ASSERT_THAT(actual_fp, Lt(fp));
|
||||||
|
|
||||||
|
cerr << count << " false positives out of " << nr_blocks << ", "
|
||||||
|
<< actual_fp * 100.0 << "%" << endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(BloomFilterTests, false_positives_over_multiple_eras)
|
||||||
|
{
|
||||||
|
unsigned nr_eras = 10;
|
||||||
|
block_address nr_blocks = 1 << 20;
|
||||||
|
block_address written_blocks = nr_blocks / nr_eras;
|
||||||
|
|
||||||
|
double fp = 0.001;
|
||||||
|
unsigned nr_probes = 6;
|
||||||
|
unsigned n = calc_nr_bits(fp, written_blocks, nr_probes);
|
||||||
|
|
||||||
|
cerr << "bitset size: " << (n / (8 * 1024)) << "k" << endl;
|
||||||
|
|
||||||
|
double ideal_k = log(2) * ((double) n / (double) written_blocks);
|
||||||
|
cerr << "Ideal k = " << ideal_k << endl;
|
||||||
|
|
||||||
|
vector<set<block_address> > writes(nr_eras);
|
||||||
|
vector<bloom_filter::ptr> filters(nr_eras);
|
||||||
|
|
||||||
|
for (unsigned era = 0; era < writes.size(); era++) {
|
||||||
|
cerr << "inserting era " << era << endl;
|
||||||
|
|
||||||
|
writes[era] = generate_random_blocks(written_blocks, nr_blocks);
|
||||||
|
set<block_address> const &bs = writes[era];
|
||||||
|
|
||||||
|
filters[era] = bloom_filter::ptr(new bloom_filter(tm_, n, nr_probes));
|
||||||
|
bloom_filter::ptr &f = filters[era];
|
||||||
|
|
||||||
|
set<block_address>::const_iterator it;
|
||||||
|
for (it = bs.begin(); it != bs.end(); ++it)
|
||||||
|
f->set(*it);
|
||||||
|
}
|
||||||
|
|
||||||
|
set<block_address> write_sum;
|
||||||
|
set<block_address> filter_sum;
|
||||||
|
for (unsigned era_plus_1 = writes.size(); era_plus_1 > 0; era_plus_1--) {
|
||||||
|
unsigned era = era_plus_1 - 1;
|
||||||
|
|
||||||
|
set<block_address> const &era_writes = writes[era];
|
||||||
|
write_sum.insert(era_writes.begin(), era_writes.end());
|
||||||
|
|
||||||
|
for (unsigned i = 0; i < nr_blocks; i++)
|
||||||
|
if (filters[era]->test(i))
|
||||||
|
filter_sum.insert(i);
|
||||||
|
|
||||||
|
unsigned count = 0;
|
||||||
|
for (unsigned i = 0; i < nr_blocks; i++) {
|
||||||
|
if (write_sum.count(i) > 0)
|
||||||
|
ASSERT_THAT(filter_sum.count(i), Gt(0ull));
|
||||||
|
|
||||||
|
else if (filter_sum.count(i))
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
cerr << "blocks >= era " << era << ", false positives = "
|
||||||
|
<< static_cast<double>(count * 100) / static_cast<double>(nr_blocks)
|
||||||
|
<< "%" << endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
//----------------------------------------------------------------
|
//----------------------------------------------------------------
|
||||||
|
@ -5,6 +5,7 @@
|
|||||||
#include "persistent-data/data-structures/btree.h"
|
#include "persistent-data/data-structures/btree.h"
|
||||||
#include "persistent-data/data-structures/btree_counter.h"
|
#include "persistent-data/data-structures/btree_counter.h"
|
||||||
#include "persistent-data/space-maps/core.h"
|
#include "persistent-data/space-maps/core.h"
|
||||||
|
#include "persistent-data/data-structures/simple_traits.h"
|
||||||
|
|
||||||
using namespace base;
|
using namespace base;
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
@ -20,6 +20,7 @@
|
|||||||
#include "persistent-data/transaction_manager.h"
|
#include "persistent-data/transaction_manager.h"
|
||||||
#include "persistent-data/space-maps/core.h"
|
#include "persistent-data/space-maps/core.h"
|
||||||
#include "persistent-data/data-structures/btree.h"
|
#include "persistent-data/data-structures/btree.h"
|
||||||
|
#include "persistent-data/data-structures/simple_traits.h"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
using namespace persistent_data;
|
using namespace persistent_data;
|
||||||
|
Loading…
Reference in New Issue
Block a user