emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[space-maps/core] rewrite the core space map to use less memory.

Browse Source
This commit is contained in:
Joe Thornber 2020-05-27 12:00:40 +01:00
parent e92ae10408
commit 29cfdd8979
15 changed files with 268 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile.in
View File

@ -87,6 +87,7 @@ SOURCE=\
persistent-data/file_utils.cc \
persistent-data/hex_dump.cc \
persistent-data/space-maps/careful_alloc.cc \
persistent-data/space-maps/core.cc \
persistent-data/space-maps/disk.cc \
persistent-data/space-maps/recursive.cc \
persistent-data/space_map.cc \

2
caching/metadata.cc
View File

@ -15,7 +15,7 @@ namespace {
if (!nr_blocks)
throw runtime_error("Metadata is not large enough for superblock.");
space_map::ptr sm(new core_map(nr_blocks));
space_map::ptr sm{create_core_map(nr_blocks)};
sm->inc(SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;

2
era/metadata.cc
View File

@ -13,7 +13,7 @@ namespace {
if (!nr_blocks)
throw runtime_error("Metadata is not large enough for superblock.");
space_map::ptr sm(new core_map(nr_blocks));
space_map::ptr sm{create_core_map(nr_blocks)};
sm->inc(SUPERBLOCK_LOCATION);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;

2
persistent-data/file_utils.cc
View File

@ -59,7 +59,7 @@ persistent_data::open_tm(block_manager::ptr bm, block_address superblock_locatio
if (!nr_blocks)
throw runtime_error("Metadata is not large enough for superblock.");
space_map::ptr sm(new core_map(nr_blocks));
space_map::ptr sm{create_core_map(nr_blocks)};
sm->inc(superblock_location);
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;

248
persistent-data/space-maps/core.cc Normal file
View File

@ -0,0 +1,248 @@
// Copyright (C) 2020 Red Hat, Inc. All rights reserved.
//
// This file is part of the thin-provisioning-tools source.
//
// thin-provisioning-tools is free software: you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation, either version 3 of
// the License, or (at your option) any later version.
//
// thin-provisioning-tools is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with thin-provisioning-tools. If not, see
// <http://www.gnu.org/licenses/>.
#include "persistent-data/space-maps/core.h"
#include "persistent-data/math_utils.h"
#include <stdexcept>
using namespace persistent_data;
using namespace std;
//----------------------------------------------------------------
namespace {
constexpr block_address ENTRIES_PER_WORD = 4 * sizeof(uint64_t);
class core_map : public checked_space_map {
public:
typedef std::shared_ptr<core_map> ptr;
core_map(block_address nr_blocks);
block_address get_nr_blocks() const override;
block_address get_nr_free() const override;
ref_t get_count(block_address b) const override;
void set_count(block_address b, ref_t c) override;
void commit() override;
void inc(block_address b, ref_t count) override;
void dec(block_address b, ref_t count) override;
maybe_block find_free(span_iterator &it) override;
bool count_possibly_greater_than_one(block_address b) const override;
void extend(block_address extra_blocks) override;
void count_metadata(block_counter &bc) const override;
// FIXME: meaningless, but this class is only used for testing
size_t root_size() const override;
// FIXME: meaningless, but this class is only used for testing
virtual void copy_root(void *dest, size_t len) const override;
checked_space_map::ptr clone() const override;
private:
ref_t get_count_(block_address b) const;
void set_count_(block_address b, ref_t c);
void check_block_(block_address b) const;
ref_t lookup_bits_(block_address b) const;
void set_bits_(block_address b, ref_t c);
ref_t lookup_exception_(block_address b) const;
block_address nr_blocks_;
block_address nr_free_;
block_address search_start_;
std::vector<uint64_t> bits_;
std::map<block_address, ref_t> exceptions_;
};
core_map::core_map(block_address nr_blocks)
: nr_blocks_(nr_blocks),
nr_free_(nr_blocks),
search_start_(0),
bits_(base::div_up(nr_blocks, ENTRIES_PER_WORD)) {
}
block_address
core_map::get_nr_blocks() const {
return nr_blocks_;
}
block_address
core_map::get_nr_free() const {
return nr_free_;
}
ref_t
core_map::get_count(block_address b) const {
check_block_(b);
return get_count_(b);
}
void
core_map::set_count(block_address b, ref_t c) {
check_block_(b);
set_count_(b, c);
}
void
core_map::commit() {
}
void
core_map::inc(block_address b, ref_t count) {
check_block_(b);
auto old_c = get_count_(b);
set_count_(b, old_c + count);
}
void
core_map::dec(block_address b, ref_t count) {
check_block_(b);
auto old_c = get_count_(b);
set_count_(b, old_c - count);
}
core_map::maybe_block
core_map::find_free(span_iterator &it) {
for (maybe_span ms = it.first(); ms; ms = it.next()) {
for (block_address b = std::max(search_start_, ms->first); b < ms->second; b++) {
check_block_(b);
if (!get_count_(b))
return maybe_block(b);
}
}
return maybe_block();
}
bool
core_map::count_possibly_greater_than_one(block_address b) const {
return get_count(b) > 1;
}
void
core_map::extend(block_address extra_blocks) {
throw std::runtime_error("'extend' not implemented");
}
void
core_map::count_metadata(block_counter &bc) const {
}
// FIXME: meaningless, but this class is only used for testing
size_t
core_map::root_size() const {
return 0;
}
// FIXME: meaningless, but this class is only used for testing
void
core_map::copy_root(void *dest, size_t len) const {
throw std::runtime_error("'copy root' not implemented");
}
checked_space_map::ptr
core_map::clone() const {
return ptr(new core_map(*this));
}
void
core_map::check_block_(block_address b) const {
if (b >= nr_blocks_)
throw std::runtime_error("block out of bounds");
}
ref_t
core_map::get_count_(block_address b) const {
auto c = lookup_bits_(b);
if (c == 3)
c = lookup_exception_(b);
return c;
}
void
core_map::set_count_(block_address b, ref_t c) {
auto old_c = get_count_(b);
if (old_c > 2) {
if (c > 2)
exceptions_[b] = c;
else {
exceptions_.erase(b);
set_bits_(b, c);
}
} else {
if (c > 2) {
set_bits_(b, 3);
exceptions_.insert(make_pair(b, c));
} else
set_bits_(b, c);
}
if (old_c == 0 && c > 0)
nr_free_--;
else if (old_c > 0 && c == 0) {
if (b < search_start_)
search_start_ = b;
nr_free_++;
}
}
ref_t
core_map::lookup_bits_(block_address b) const {
block_address word = b / ENTRIES_PER_WORD;
block_address shift = (b % ENTRIES_PER_WORD) * 2;
return (bits_[word] >> shift) & 0b11;
}
void
core_map::set_bits_(block_address b, ref_t c) {
if (c > 3)
throw runtime_error("bad count");
block_address word = b / ENTRIES_PER_WORD;
block_address shift = (b % ENTRIES_PER_WORD) * 2ull;
auto w = bits_[word] & ~(0b11ull << shift);
bits_[word] = w | (((uint64_t) c) << shift);
}
ref_t
core_map::lookup_exception_(block_address b) const {
auto it = exceptions_.find(b);
if (it == exceptions_.end())
throw runtime_error("core space map exception entry missing");
return it->second;
}
}
//----------------------------------------------------------------
checked_space_map::ptr
persistent_data::create_core_map(block_address nr_blocks)
{
return checked_space_map::ptr(new core_map(nr_blocks));
}
//----------------------------------------------------------------

106
persistent-data/space-maps/core.h
View File

@ -21,112 +21,12 @@
#include "persistent-data/space_map.h"
#include <map>
//----------------------------------------------------------------
namespace persistent_data {
class core_map : public checked_space_map {
public:
typedef std::shared_ptr<core_map> ptr;
core_map(block_address nr_blocks)
: counts_(nr_blocks, 0),
nr_free_(nr_blocks),
search_start_(0ull) {
}
block_address get_nr_blocks() const {
return counts_.size();
}
block_address get_nr_free() const {
return nr_free_;
}
ref_t get_count(block_address b) const {
return counts_[b];
}
void set_count(block_address b, ref_t c) {
if (counts_[b] == 0 && c > 0)
nr_free_--;
else if (counts_[b] > 0 && c == 0) {
if (b < search_start_)
search_start_ = b;
nr_free_++;
}
counts_[b] = c;
}
void commit() {
}
void inc(block_address b, ref_t count) override {
if (counts_[b] == 0)
nr_free_--;
counts_[b] += count;
}
void dec(block_address b, ref_t count) override {
counts_[b] -= count;
if (counts_[b] == 0) {
if (b < search_start_)
search_start_ = b;
nr_free_++;
}
}
maybe_block find_free(span_iterator &it) {
for (maybe_span ms = it.first(); ms; ms = it.next()) {
if (search_start_ >= ms->second)
continue;
for (block_address b = std::max(search_start_, ms->first); b < ms->second; b++) {
if (b >= counts_.size())
throw std::runtime_error("block out of bounds");
if (!counts_[b])
return maybe_block(b);
}
}
return maybe_block();
}
bool count_possibly_greater_than_one(block_address b) const {
return counts_[b] > 1;
}
void extend(block_address extra_blocks) {
throw std::runtime_error("'extend' not implemented");
}
void count_metadata(block_counter &bc) const {
}
// FIXME: meaningless, but this class is only used for testing
size_t root_size() const {
return 0;
}
// FIXME: meaningless, but this class is only used for testing
virtual void copy_root(void *dest, size_t len) const {
throw std::runtime_error("'copy root' not implemented");
}
checked_space_map::ptr clone() const {
return ptr(new core_map(*this));
}
private:
std::vector<ref_t> counts_;
unsigned nr_free_;
block_address search_start_;
};
checked_space_map::ptr create_core_map(block_address nr_blocks);
}
//----------------------------------------------------------------

2
unit-tests/array_block_t.cc
View File

@ -87,7 +87,7 @@ namespace {
transaction_manager::ptr
create_tm() {
block_manager::ptr bm = create_bm(NR_BLOCKS);
space_map::ptr sm(new core_map(NR_BLOCKS));
space_map::ptr sm{create_core_map(NR_BLOCKS)};
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
}

2
unit-tests/array_t.cc
View File

@ -38,7 +38,7 @@ namespace {
public:
ArrayTests()
: bm_(new block_manager("./test.data", NR_BLOCKS, 4, block_manager::READ_WRITE)),
sm_(new core_map(NR_BLOCKS)),
sm_(create_core_map(NR_BLOCKS)),
tm_(bm_, sm_) {
}

2
unit-tests/bitset_t.cc
View File

@ -56,7 +56,7 @@ namespace {
public:
BitsetTests()
: bm_(new block_manager("./test.data", NR_BLOCKS, 4, block_manager::READ_WRITE)),
sm_(new core_map(NR_BLOCKS)),
sm_(create_core_map(NR_BLOCKS)),
tm_(bm_, sm_) {
}

2
unit-tests/btree_counter_t.cc
View File

@ -45,7 +45,7 @@ namespace {
private:
space_map::ptr setup_core_map() {
space_map::ptr sm(new core_map(NR_BLOCKS));
space_map::ptr sm(create_core_map(NR_BLOCKS));
sm->inc(SUPERBLOCK);
return sm;
}

2
unit-tests/btree_damage_visitor_t.cc
View File

@ -302,7 +302,7 @@ namespace {
private:
space_map::ptr setup_core_map() {
space_map::ptr sm(new core_map(NR_BLOCKS));
space_map::ptr sm(create_core_map(NR_BLOCKS));
sm->inc(SUPERBLOCK);
return sm;
}

2
unit-tests/btree_t.cc
View File

@ -35,7 +35,7 @@ namespace {
public:
BtreeTests()
: bm_(new block_manager("./test.data", NR_BLOCKS, 4, block_manager::READ_WRITE)),
sm_(new core_map(NR_BLOCKS)),
sm_(create_core_map(NR_BLOCKS)),
tm_(bm_, sm_) {
}

12
unit-tests/space_map_t.cc
View File

@ -37,14 +37,14 @@ namespace {
public:
SpaceMapTests()
: bm_(new block_manager("./test.data", NR_BLOCKS, MAX_LOCKS, block_manager::READ_WRITE)),
sm_(new core_map(NR_BLOCKS)),
sm_(create_core_map(NR_BLOCKS)),
tm_(bm_, sm_) {
}
struct sm_core_creator {
static space_map::ptr
create(transaction_manager &tm) {
return space_map::ptr(new persistent_data::core_map(NR_BLOCKS));
return create_core_map(NR_BLOCKS);
}
};
@ -52,8 +52,7 @@ namespace {
static space_map::ptr
create(transaction_manager &tm) {
return create_careful_alloc_sm(
checked_space_map::ptr(
new core_map(NR_BLOCKS)));
create_core_map(NR_BLOCKS));
}
};
@ -61,8 +60,7 @@ namespace {
static checked_space_map::ptr
create(transaction_manager &tm) {
return create_recursive_sm(
checked_space_map::ptr(
new core_map(NR_BLOCKS)));
create_core_map(NR_BLOCKS));
}
};
@ -279,7 +277,7 @@ TEST_F(SpaceMapTests, test_metadata_and_disk)
{
block_manager::ptr bm(
new block_manager("./test.data", NR_BLOCKS, MAX_LOCKS, block_manager::READ_WRITE));
space_map::ptr core_sm(new core_map(NR_BLOCKS));
space_map::ptr core_sm{create_core_map(NR_BLOCKS)};
transaction_manager::ptr tm(new transaction_manager(bm, core_sm));
persistent_space_map::ptr metadata_sm = persistent_data::create_metadata_sm(*tm, NR_BLOCKS);
copy_space_maps(metadata_sm, core_sm);

2
unit-tests/test_utils.cc
View File

@ -21,7 +21,7 @@ void test::zero_block(block_manager::ptr bm, block_address b)
transaction_manager::ptr
test::open_temporary_tm(block_manager::ptr bm)
{
space_map::ptr sm(new core_map(bm->get_nr_blocks()));
space_map::ptr sm{create_core_map(bm->get_nr_blocks())};
transaction_manager::ptr tm(new transaction_manager(bm, sm));
return tm;
}

2
unit-tests/transaction_manager_t.cc
View File

@ -34,7 +34,7 @@ namespace {
create_tm() {
block_manager::ptr bm(
new block_manager("./test.data", NR_BLOCKS, MAX_HELD_LOCKS, block_manager::READ_WRITE));
space_map::ptr sm(new core_map(NR_BLOCKS));
space_map::ptr sm{create_core_map(NR_BLOCKS)};
transaction_manager::ptr tm(new transaction_manager(bm, sm));
tm->get_sm()->inc(0);
return tm;