emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[functional-tests] Move thin_check functional tests to Rust.

Browse Source 
They'll be run as part of 'cargo test' now.
This commit is contained in:
Joe Thornber 2020-07-31 11:04:12 +01:00
parent 7243f95380
commit fcfcc60b89
7 changed files with 776 additions and 598 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
Cargo.lock generated
View File

@ -107,6 +107,18 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "duct"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f90a9c3a25aafbd538c7d40a53f83c4487ee8216c12d1c8ef2c01eb2f6ea1553"
dependencies = [
"libc",
"once_cell",
"os_pipe",
"shared_child",
]
[[package]]
name = "env_logger"
version = "0.7.1"
@ -260,6 +272,12 @@ dependencies = [
"libc",
]
[[package]]
name = "json"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
[[package]]
name = "lazy_static"
version = "1.4.0"
@ -369,6 +387,16 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"
[[package]]
name = "os_pipe"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb233f06c2307e1f5ce2ecad9f8121cffbbee2c95428f44ea85222e460d0d213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "pin-project"
version = "0.4.23"
@ -543,6 +571,16 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "shared_child"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cebcf3a403e4deafaf34dc882c4a1b6a648b43e5670aa2e4bb985914eaeb2d2"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "slab"
version = "0.4.2"
@ -603,10 +641,12 @@ dependencies = [
"byteorder",
"clap",
"crc32c",
"duct",
"fixedbitset",
"flate2",
"futures",
"io-uring",
"json",
"libc",
"nix",
"nom",

2
Cargo.toml
View File

@ -10,6 +10,7 @@ anyhow = "1.0"
byteorder = "1.3"
clap = "2.33"
crc32c = "0.4"
duct = "0.13"
fixedbitset = "0.3"
futures = "0.3"
flate2 = "1.0"
@ -27,5 +28,6 @@ threadpool = "1.8"
thiserror = "1.0"
[dev-dependencies]
json = "0.12"
quickcheck = "0.9"
quickcheck_macros = "0.9"

80
functional-tests/thin-functional-tests.scm
View File

@ -67,86 +67,6 @@
;; to run.
(define (register-thin-tests) #t)
;;;-----------------------------------------------------------
;;; thin_check scenarios
;;;-----------------------------------------------------------
(define-scenario (thin-check v)
"thin_check -V"
(run-ok-rcv (stdout _) (thin-check "-V")
(assert-equal tools-version stdout)))
(define-scenario (thin-check version)
"thin_check --version"
(run-ok-rcv (stdout _) (thin-check "--version")
(assert-equal tools-version stdout)))
(define-scenario (thin-check h)
"print help (-h)"
(run-ok-rcv (stdout _) (thin-check "-h")
(assert-equal thin-check-help stdout)))
(define-scenario (thin-check help)
"print help (--help)"
(run-ok-rcv (stdout _) (thin-check "--help")
(assert-equal thin-check-help stdout)))
(define-scenario (thin-check bad-option)
"Unrecognised option should cause failure"
(run-fail (thin-check "--hedgehogs-only")))
(define-scenario (thin-check superblock-only-valid)
"--super-block-only check passes on valid metadata"
(with-valid-metadata (md)
(run-ok (thin-check "--super-block-only" md))))
(define-scenario (thin-check superblock-only-invalid)
"--super-block-only check fails with corrupt metadata"
(with-corrupt-metadata (md)
(run-fail (thin-check "--super-block-only" md))))
(define-scenario (thin-check skip-mappings-valid)
"--skip-mappings check passes on valid metadata"
(with-valid-metadata (md)
(run-ok (thin-check "--skip-mappings" md))))
(define-scenario (thin-check ignore-non-fatal-errors)
"--ignore-non-fatal-errors check passes on valid metadata"
(with-valid-metadata (md)
(run-ok (thin-check "--ignore-non-fatal-errors" md))))
(define-scenario (thin-check quiet)
"--quiet should give no output"
(with-valid-metadata (md)
(run-ok-rcv (stdout stderr) (thin-check "--quiet" md)
(assert-eof stdout)
(assert-eof stderr))))
(define-scenario (thin-check clear-needs-check-flag)
"Accepts --clear-needs-check-flag"
(with-valid-metadata (md)
(run-ok (thin-check "--clear-needs-check-flag" md))))
(define-scenario (thin-check tiny-metadata)
"Prints helpful message in case tiny metadata given"
(with-temp-file-sized ((md "thin.bin" 1024))
(run-fail-rcv (_ stderr) (thin-check md)
(assert-starts-with "Metadata device/file too small. Is this binary metadata?" stderr))))
(define-scenario (thin-check spot-accidental-xml-data)
"Prints helpful message if XML metadata given"
(with-thin-xml (xml)
(system (fmt #f "man bash >> " xml))
(run-fail-rcv (_ stderr) (thin-check xml)
(assert-matches ".*This looks like XML. thin_check only checks the binary metadata format." stderr))))
(define-scenario (thin-check info-fields)
"Outputs info fields"
(with-valid-metadata (md)
(run-ok-rcv (stdout stderr) (thin-check md)
(assert-matches ".*TRANSACTION_ID=[0-9]+.*" stdout)
(assert-matches ".*METADATA_FREE_BLOCKS=[0-9]+.*" stdout))))
;;;-----------------------------------------------------------
;;; thin_restore scenarios
;;;-----------------------------------------------------------

11
tests/common/mod.rs Normal file
View File

@ -0,0 +1,11 @@
use std::path::{Path, PathBuf};
pub mod xml_generator;
pub fn mk_path(dir: &Path, file: &str) -> PathBuf {
let mut p = PathBuf::new();
p.push(dir);
p.push(PathBuf::from(file));
p
}

539
tests/common/xml_generator.rs Normal file
View File

@ -0,0 +1,539 @@
use anyhow::{anyhow, Result};
use rand::prelude::*;
use std::collections::VecDeque;
use std::fs::OpenOptions;
use std::ops::Range;
use std::path::Path;
use thinp::thin::xml;
//------------------------------------------
pub trait XmlGen {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>;
}
pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> {
let xml_out = OpenOptions::new()
.read(false)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
let mut w = xml::XmlWriter::new(xml_out);
g.generate_xml(&mut w)
}
fn common_sb(nr_blocks: u64) -> xml::Superblock {
xml::Superblock {
uuid: "".to_string(),
time: 0,
transaction: 0,
flags: None,
version: None,
data_block_size: 32,
nr_data_blocks: nr_blocks,
metadata_snap: None,
}
}
//------------------------------------------
pub struct EmptyPoolS {}
impl XmlGen for EmptyPoolS {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
v.superblock_b(&common_sb(1024))?;
v.superblock_e()?;
Ok(())
}
}
//------------------------------------------
pub struct SingleThinS {
pub offset: u64,
pub len: u64,
pub old_nr_data_blocks: u64,
pub new_nr_data_blocks: u64,
}
impl SingleThinS {
pub fn new(offset: u64, len: u64, old_nr_data_blocks: u64, new_nr_data_blocks: u64) -> Self {
SingleThinS {
offset,
len,
old_nr_data_blocks,
new_nr_data_blocks,
}
}
}
impl XmlGen for SingleThinS {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
v.device_b(&xml::Device {
dev_id: 0,
mapped_blocks: self.len,
transaction: 0,
creation_time: 0,
snap_time: 0,
})?;
v.map(&xml::Map {
thin_begin: 0,
data_begin: self.offset,
time: 0,
len: self.len,
})?;
v.device_e()?;
v.superblock_e()?;
Ok(())
}
}
//------------------------------------------
pub struct FragmentedS {
pub nr_thins: u32,
pub thin_size: u64,
pub old_nr_data_blocks: u64,
pub new_nr_data_blocks: u64,
}
impl FragmentedS {
pub fn new(nr_thins: u32, thin_size: u64) -> Self {
let old_size = (nr_thins as u64) * thin_size;
FragmentedS {
nr_thins,
thin_size,
old_nr_data_blocks: (nr_thins as u64) * thin_size,
new_nr_data_blocks: old_size * 3 / 4,
}
}
}
#[derive(Clone)]
struct ThinRun {
thin_id: u32,
thin_begin: u64,
len: u64,
}
#[derive(Clone, Debug, Copy)]
struct MappedRun {
thin_id: u32,
thin_begin: u64,
data_begin: u64,
len: u64,
}
fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range<u64>) -> Vec<ThinRun> {
let mut runs = Vec::new();
let mut b = 0u64;
while b < total_len {
let len = u64::min(
total_len - b,
thread_rng().gen_range(run_len.start, run_len.end),
);
runs.push(ThinRun {
thin_id: thin_id,
thin_begin: b,
len,
});
b += len;
}
runs
}
impl XmlGen for FragmentedS {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
// Allocate each thin fully, in runs between 1 and 16.
let mut runs = Vec::new();
for thin in 0..self.nr_thins {
runs.append(&mut mk_runs(thin, self.thin_size, 1..17));
}
// Shuffle
runs.shuffle(&mut rand::thread_rng());
// map across the data
let mut maps = Vec::new();
let mut b = 0;
for r in &runs {
maps.push(MappedRun {
thin_id: r.thin_id,
thin_begin: r.thin_begin,
data_begin: b,
len: r.len,
});
b += r.len;
}
// drop half the mappings, which leaves us free runs
let mut dropped = Vec::new();
for i in 0..maps.len() {
if i % 2 == 0 {
dropped.push(maps[i].clone());
}
}
// Unshuffle. This isn't strictly necc. but makes the xml
// more readable.
use std::cmp::Ordering;
maps.sort_by(|&l, &r| match l.thin_id.cmp(&r.thin_id) {
Ordering::Equal => l.thin_begin.cmp(&r.thin_begin),
o => o,
});
// write the xml
v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
for thin in 0..self.nr_thins {
v.device_b(&xml::Device {
dev_id: thin,
mapped_blocks: self.thin_size,
transaction: 0,
creation_time: 0,
snap_time: 0,
})?;
for m in &dropped {
if m.thin_id != thin {
continue;
}
v.map(&xml::Map {
thin_begin: m.thin_begin,
data_begin: m.data_begin,
time: 0,
len: m.len,
})?;
}
v.device_e()?;
}
v.superblock_e()?;
Ok(())
}
}
//------------------------------------------
struct Allocator {
runs: VecDeque<Range<u64>>,
}
impl Allocator {
fn new_shuffled(total_len: u64, run_len: Range<u64>) -> Allocator {
let mut runs = Vec::new();
let mut b = 0u64;
while b < total_len {
let len = u64::min(
total_len - b,
thread_rng().gen_range(run_len.start, run_len.end),
);
runs.push(b..(b + len));
b += len;
}
runs.shuffle(&mut thread_rng());
let runs: VecDeque<Range<u64>> = runs.iter().map(|r| r.clone()).collect();
Allocator { runs }
}
#[allow(dead_code)]
fn is_empty(&self) -> bool {
self.runs.is_empty()
}
fn alloc(&mut self, len: u64) -> Result<Vec<Range<u64>>> {
let mut len = len;
let mut runs = Vec::new();
while len > 0 {
let r = self.runs.pop_front();
if r.is_none() {
return Err(anyhow!("could not allocate; out of space"));
}
let r = r.unwrap();
let rlen = r.end - r.start;
if len < rlen {
runs.push(r.start..(r.start + len));
// We need to push something back.
self.runs.push_front((r.start + len)..r.end);
len = 0;
} else {
runs.push(r.start..r.end);
len -= rlen;
}
}
Ok(runs)
}
}
// Having explicitly unmapped regions makes it easier to
// apply snapshots.
#[derive(Clone)]
enum Run {
Mapped { data_begin: u64, len: u64 },
UnMapped { len: u64 },
}
impl Run {
#[allow(dead_code)]
fn len(&self) -> u64 {
match self {
Run::Mapped {
data_begin: _data_begin,
len,
} => *len,
Run::UnMapped { len } => *len,
}
}
fn split(&self, n: u64) -> (Option<Run>, Option<Run>) {
if n == 0 {
return (None, Some(self.clone()));
} else {
if self.len() <= n {
return (Some(self.clone()), None);
} else {
match self {
Run::Mapped { data_begin, len } => (
Some(Run::Mapped {
data_begin: *data_begin,
len: n,
}),
Some(Run::Mapped {
data_begin: data_begin + n,
len: len - n,
}),
),
Run::UnMapped { len } => (
Some(Run::UnMapped { len: n }),
Some(Run::UnMapped { len: len - n }),
),
}
}
}
}
}
#[derive(Clone)]
struct ThinDev {
thin_id: u32,
dev_size: u64,
runs: Vec<Run>,
}
impl ThinDev {
fn emit(&self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
v.device_b(&xml::Device {
dev_id: self.thin_id,
mapped_blocks: self.dev_size,
transaction: 0,
creation_time: 0,
snap_time: 0,
})?;
let mut b = 0;
for r in &self.runs {
match r {
Run::Mapped { data_begin, len } => {
v.map(&xml::Map {
thin_begin: b,
data_begin: *data_begin,
time: 0,
len: *len,
})?;
b += len;
}
Run::UnMapped { len } => {
b += len;
}
}
}
v.device_e()?;
Ok(())
}
}
#[derive(Clone)]
enum SnapRunType {
Same,
Diff,
Hole,
}
#[derive(Clone)]
struct SnapRun(SnapRunType, u64);
fn mk_origin(thin_id: u32, total_len: u64, allocator: &mut Allocator) -> Result<ThinDev> {
let mut runs = Vec::new();
let mut b = 0;
while b < total_len {
let len = u64::min(thread_rng().gen_range(16, 64), total_len - b);
match thread_rng().gen_range(0, 2) {
0 => {
for data in allocator.alloc(len)? {
assert!(data.end >= data.start);
runs.push(Run::Mapped {
data_begin: data.start,
len: data.end - data.start,
});
}
}
1 => {
runs.push(Run::UnMapped { len });
}
_ => {
return Err(anyhow!("bad value returned from rng"));
}
};
b += len;
}
Ok(ThinDev {
thin_id,
dev_size: total_len,
runs,
})
}
fn mk_snap_mapping(
total_len: u64,
run_len: Range<u64>,
same_percent: usize,
diff_percent: usize,
) -> Vec<SnapRun> {
let mut runs = Vec::new();
let mut b = 0u64;
while b < total_len {
let len = u64::min(
total_len - b,
thread_rng().gen_range(run_len.start, run_len.end),
);
let n = thread_rng().gen_range(0, 100);
if n < same_percent {
runs.push(SnapRun(SnapRunType::Same, len));
} else if n < diff_percent {
runs.push(SnapRun(SnapRunType::Diff, len));
} else {
runs.push(SnapRun(SnapRunType::Hole, len));
}
b += len;
}
runs
}
fn split_runs(mut n: u64, runs: &Vec<Run>) -> (Vec<Run>, Vec<Run>) {
let mut before = Vec::new();
let mut after = Vec::new();
for r in runs {
match r.split(n) {
(Some(lhs), None) => {
before.push(lhs);
}
(Some(lhs), Some(rhs)) => {
before.push(lhs);
after.push(rhs);
}
(None, Some(rhs)) => {
after.push(rhs);
}
(None, None) => {}
}
n -= r.len();
}
(before, after)
}
fn apply_snap_runs(
origin: &Vec<Run>,
snap: &Vec<SnapRun>,
allocator: &mut Allocator,
) -> Result<Vec<Run>> {
let mut origin = origin.clone();
let mut runs = Vec::new();
for SnapRun(st, slen) in snap {
let (os, rest) = split_runs(*slen, &origin);
match st {
SnapRunType::Same => {
for o in os {
runs.push(o);
}
}
SnapRunType::Diff => {
for data in allocator.alloc(*slen)? {
runs.push(Run::Mapped {
data_begin: data.start,
len: data.end - data.start,
});
}
}
SnapRunType::Hole => {
runs.push(Run::UnMapped { len: *slen });
}
}
origin = rest;
}
Ok(runs)
}
// Snapshots share mappings, not neccessarily the entire ranges.
pub struct SnapS {
pub len: u64,
pub nr_snaps: u32,
// Snaps will differ from the origin by this percentage
pub percent_change: usize,
pub old_nr_data_blocks: u64,
pub new_nr_data_blocks: u64,
}
impl SnapS {
pub fn new(len: u64, nr_snaps: u32, percent_change: usize) -> Self {
let delta = len * (nr_snaps as u64) * (percent_change as u64) / 100;
let old_nr_data_blocks = len + 3 * delta;
let new_nr_data_blocks = len + 2 * delta;
SnapS {
len,
nr_snaps,
percent_change,
old_nr_data_blocks,
new_nr_data_blocks,
}
}
}
impl XmlGen for SnapS {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512);
let origin = mk_origin(0, self.len, &mut allocator)?;
v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
origin.emit(v)?;
v.superblock_e()?;
Ok(())
}
}
//------------------------------------------

173
tests/thin_check.rs Normal file
View File

@ -0,0 +1,173 @@
use anyhow::Result;
use duct::{cmd, Expression};
use std::path::{Path, PathBuf};
use std::str::from_utf8;
use tempfile::{tempdir, TempDir};
use thinp::file_utils;
use thinp::version::TOOLS_VERSION;
mod common;
use common::mk_path;
use common::xml_generator::{write_xml, FragmentedS, SingleThinS};
//------------------------------------------
macro_rules! thin_check {
( $( $arg: expr ),* ) => {
{
use std::ffi::OsString;
let args: &[OsString] = &[$( Into::<OsString>::into($arg) ),*];
duct::cmd("bin/thin_check", args).stdout_capture().stderr_capture()
}
};
}
// Returns stderr, a non zero status must be returned
fn run_fail(command: Expression) -> Result<String> {
let output = command.stderr_capture().unchecked().run()?;
assert!(!output.status.success());
Ok(from_utf8(&output.stderr[0..]).unwrap().to_string())
}
fn mk_valid_md(dir: &TempDir) -> Result<PathBuf> {
let xml = mk_path(dir.path(), "meta.xml");
let md = mk_path(dir.path(), "meta.bin");
let mut gen = SingleThinS::new(0, 1024, 2048, 2048);
write_xml(&xml, &mut gen)?;
let _file = file_utils::create_sized_file(&md, 4096 * 4096);
cmd!("bin/thin_restore", "-i", xml, "-o", &md).run()?;
Ok(md)
}
fn mk_corrupt_md(dir: &TempDir) -> Result<PathBuf> {
let md = mk_path(dir.path(), "meta.bin");
let _file = file_utils::create_sized_file(&md, 4096 * 4096);
Ok(md)
}
fn accepts_flag(flag: &str) -> Result<()> {
let dir = tempdir()?;
let md = mk_valid_md(&dir)?;
thin_check!(flag, &md).run()?;
Ok(())
}
//------------------------------------------
#[test]
fn accepts_v() -> Result<()> {
let stdout = thin_check!("-V").read()?;
assert_eq!(stdout, TOOLS_VERSION);
Ok(())
}
#[test]
fn accepts_version() -> Result<()> {
let stdout = thin_check!("--version").read()?;
assert_eq!(stdout, TOOLS_VERSION);
Ok(())
}
const USAGE: &'static str = "Usage: thin_check [options] {device|file}\nOptions:\n {-q|--quiet}\n {-h|--help}\n {-V|--version}\n {-m|--metadata-snap}\n {--override-mapping-root}\n {--clear-needs-check-flag}\n {--ignore-non-fatal-errors}\n {--skip-mappings}\n {--super-block-only}";
#[test]
fn accepts_h() -> Result<()> {
let stdout = thin_check!("-h").read()?;
assert_eq!(stdout, USAGE);
Ok(())
}
#[test]
fn accepts_help() -> Result<()> {
let stdout = thin_check!("--help").read()?;
assert_eq!(stdout, USAGE);
Ok(())
}
#[test]
fn rejects_bad_option() -> Result<()> {
let stderr = run_fail(thin_check!("--hedgehogs-only"))?;
assert!(stderr.contains("unrecognized option \'--hedgehogs-only\'"));
Ok(())
}
#[test]
fn accepts_superblock_only() -> Result<()> {
accepts_flag("--super-block-only")
}
#[test]
fn accepts_skip_mappings() -> Result<()> {
accepts_flag("--skip-mappings")
}
#[test]
fn accepts_ignore_non_fatal_errors() -> Result<()> {
accepts_flag("--ignore-non-fatal-errors")
}
#[test]
fn accepts_clear_needs_check_flag() -> Result<()> {
accepts_flag("--clear-needs-check-flag")
}
#[test]
fn accepts_quiet() -> Result<()> {
let dir = tempdir()?;
let md = mk_valid_md(&dir)?;
let output = thin_check!("--quiet", &md).run()?;
assert_eq!(output.stdout.len(), 0);
assert_eq!(output.stderr.len(), 0);
Ok(())
}
#[test]
fn detects_corrupt_superblock_with_superblock_only() -> Result<()> {
let dir = tempdir()?;
let md = mk_corrupt_md(&dir)?;
let output = thin_check!("--super-block-only", &md).unchecked().run()?;
assert!(!output.status.success());
Ok(())
}
#[test]
fn prints_help_message_for_tiny_metadata() -> Result<()> {
let dir = tempdir()?;
let md = mk_path(dir.path(), "meta.bin");
let _file = file_utils::create_sized_file(&md, 1024);
let stderr = run_fail(thin_check!(&md))?;
assert!(stderr.contains("Metadata device/file too small. Is this binary metadata?"));
Ok(())
}
#[test]
fn spot_xml_data() -> Result<()> {
let dir = tempdir()?;
let xml = mk_path(dir.path(), "meta.xml");
let mut gen = FragmentedS::new(4, 10240);
write_xml(&xml, &mut gen)?;
let stderr = run_fail(thin_check!(&xml))?;
eprintln!("{}", stderr);
assert!(
stderr.contains("This looks like XML. thin_check only checks the binary metadata format.")
);
Ok(())
}
#[test]
fn prints_info_fields() -> Result<()> {
let dir = tempdir()?;
let md = mk_valid_md(&dir)?;
let stdout = thin_check!(&md).read()?;
assert!(stdout.contains("TRANSACTION_ID="));
assert!(stdout.contains("METADATA_FREE_BLOCKS="));
Ok(())
}
//------------------------------------------

529
tests/thin_shrink.rs
View File

@ -1,16 +1,20 @@
use anyhow::{anyhow, Result};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::prelude::*;
use std::collections::VecDeque;
use std::fs::OpenOptions;
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
use std::ops::Range;
use std::path::{Path, PathBuf};
use tempfile::tempdir;
use thinp::file_utils;
use thinp::thin::xml::{self, Visit};
mod common;
use common::mk_path;
use common::xml_generator::{
write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen,
};
//------------------------------------
#[derive(Debug)]
@ -251,25 +255,6 @@ impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> {
//------------------------------------
fn mk_path(dir: &Path, file: &str) -> PathBuf {
let mut p = PathBuf::new();
p.push(dir);
p.push(PathBuf::from(file));
p
}
fn generate_xml(path: &Path, g: &mut dyn Scenario) -> Result<()> {
let xml_out = OpenOptions::new()
.read(false)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
let mut w = xml::XmlWriter::new(xml_out);
g.generate_xml(&mut w)
}
fn create_data_file(data_path: &Path, xml_path: &Path) -> Result<()> {
let input = OpenOptions::new().read(true).write(false).open(xml_path)?;
@ -304,17 +289,19 @@ fn verify(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> {
}
trait Scenario {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>;
fn get_new_nr_blocks(&self) -> u64;
}
fn test_shrink(scenario: &mut dyn Scenario) -> Result<()> {
fn test_shrink<S>(scenario: &mut S) -> Result<()>
where
S: Scenario + XmlGen,
{
let dir = tempdir()?;
let xml_before = mk_path(dir.path(), "before.xml");
let xml_after = mk_path(dir.path(), "after.xml");
let data_path = mk_path(dir.path(), "metadata.bin");
generate_xml(&xml_before, scenario)?;
write_xml(&xml_before, scenario)?;
create_data_file(&data_path, &xml_before)?;
let mut rng = rand::thread_rng();
@ -332,28 +319,7 @@ fn test_shrink(scenario: &mut dyn Scenario) -> Result<()> {
//------------------------------------
fn common_sb(nr_blocks: u64) -> xml::Superblock {
xml::Superblock {
uuid: "".to_string(),
time: 0,
transaction: 0,
flags: None,
version: None,
data_block_size: 32,
nr_data_blocks: nr_blocks,
metadata_snap: None,
}
}
struct EmptyPoolS {}
impl Scenario for EmptyPoolS {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
v.superblock_b(&common_sb(1024))?;
v.superblock_e()?;
Ok(())
}
fn get_new_nr_blocks(&self) -> u64 {
512
}
@ -367,45 +333,7 @@ fn shrink_empty_pool() -> Result<()> {
//------------------------------------
struct SingleThinS {
offset: u64,
len: u64,
old_nr_data_blocks: u64,
new_nr_data_blocks: u64,
}
impl SingleThinS {
fn new(offset: u64, len: u64, old_nr_data_blocks: u64, new_nr_data_blocks: u64) -> Self {
SingleThinS {
offset,
len,
old_nr_data_blocks,
new_nr_data_blocks,
}
}
}
impl Scenario for SingleThinS {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
v.device_b(&xml::Device {
dev_id: 0,
mapped_blocks: self.len,
transaction: 0,
creation_time: 0,
snap_time: 0,
})?;
v.map(&xml::Map {
thin_begin: 0,
data_begin: self.offset,
time: 0,
len: self.len,
})?;
v.device_e()?;
v.superblock_e()?;
Ok(())
}
fn get_new_nr_blocks(&self) -> u64 {
self.new_nr_data_blocks
}
@ -452,128 +380,7 @@ fn shrink_insufficient_space() -> Result<()> {
//------------------------------------
struct FragmentedS {
nr_thins: u32,
thin_size: u64,
old_nr_data_blocks: u64,
new_nr_data_blocks: u64,
}
impl FragmentedS {
fn new(nr_thins: u32, thin_size: u64) -> Self {
let old_size = (nr_thins as u64) * thin_size;
FragmentedS {
nr_thins,
thin_size,
old_nr_data_blocks: (nr_thins as u64) * thin_size,
new_nr_data_blocks: old_size * 3 / 4,
}
}
}
#[derive(Clone)]
struct ThinRun {
thin_id: u32,
thin_begin: u64,
len: u64,
}
#[derive(Clone, Debug, Copy)]
struct MappedRun {
thin_id: u32,
thin_begin: u64,
data_begin: u64,
len: u64,
}
fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range<u64>) -> Vec<ThinRun> {
let mut runs = Vec::new();
let mut b = 0u64;
while b < total_len {
let len = u64::min(
total_len - b,
thread_rng().gen_range(run_len.start, run_len.end),
);
runs.push(ThinRun {
thin_id: thin_id,
thin_begin: b,
len,
});
b += len;
}
runs
}
impl Scenario for FragmentedS {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
// Allocate each thin fully, in runs between 1 and 16.
let mut runs = Vec::new();
for thin in 0..self.nr_thins {
runs.append(&mut mk_runs(thin, self.thin_size, 1..17));
}
// Shuffle
runs.shuffle(&mut rand::thread_rng());
// map across the data
let mut maps = Vec::new();
let mut b = 0;
for r in &runs {
maps.push(MappedRun {
thin_id: r.thin_id,
thin_begin: r.thin_begin,
data_begin: b,
len: r.len,
});
b += r.len;
}
// drop half the mappings, which leaves us free runs
let mut dropped = Vec::new();
for i in 0..maps.len() {
if i % 2 == 0 {
dropped.push(maps[i].clone());
}
}
// Unshuffle. This isn't strictly necc. but makes the xml
// more readable.
use std::cmp::Ordering;
maps.sort_by(|&l, &r| match l.thin_id.cmp(&r.thin_id) {
Ordering::Equal => l.thin_begin.cmp(&r.thin_begin),
o => o,
});
// write the xml
v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
for thin in 0..self.nr_thins {
v.device_b(&xml::Device {
dev_id: thin,
mapped_blocks: self.thin_size,
transaction: 0,
creation_time: 0,
snap_time: 0,
})?;
for m in &dropped {
if m.thin_id != thin {
continue;
}
v.map(&xml::Map {
thin_begin: m.thin_begin,
data_begin: m.data_begin,
time: 0,
len: m.len,
})?;
}
v.device_e()?;
}
v.superblock_e()?;
Ok(())
}
fn get_new_nr_blocks(&self) -> u64 {
self.new_nr_data_blocks
}
@ -605,321 +412,7 @@ fn shrink_fragmented_thin_64() -> Result<()> {
//------------------------------------
struct Allocator {
runs: VecDeque<Range<u64>>,
}
impl Allocator {
fn new_shuffled(total_len: u64, run_len: Range<u64>) -> Allocator {
let mut runs = Vec::new();
let mut b = 0u64;
while b < total_len {
let len = u64::min(
total_len - b,
thread_rng().gen_range(run_len.start, run_len.end),
);
runs.push(b..(b + len));
b += len;
}
runs.shuffle(&mut thread_rng());
let runs: VecDeque<Range<u64>> = runs.iter().map(|r| r.clone()).collect();
Allocator { runs }
}
fn is_empty(&self) -> bool {
self.runs.is_empty()
}
fn alloc(&mut self, len: u64) -> Result<Vec<Range<u64>>> {
let mut len = len;
let mut runs = Vec::new();
while len > 0 {
let r = self.runs.pop_front();
if r.is_none() {
return Err(anyhow!("could not allocate; out of space"));
}
let mut r = r.unwrap();
let rlen = r.end - r.start;
if len < rlen {
runs.push(r.start..(r.start + len));
// We need to push something back.
self.runs.push_front((r.start + len)..r.end);
len = 0;
} else {
runs.push(r.start..r.end);
len -= rlen;
}
}
Ok(runs)
}
}
// Having explicitly unmapped regions makes it easier to
// apply snapshots.
#[derive(Clone)]
enum Run {
Mapped { data_begin: u64, len: u64 },
UnMapped { len: u64 },
}
impl Run {
fn len(&self) -> u64 {
match self {
Run::Mapped {
data_begin: _data_begin,
len,
} => *len,
Run::UnMapped { len } => *len,
}
}
fn split(&self, n: u64) -> (Option<Run>, Option<Run>) {
if n == 0 {
return (None, Some(self.clone()));
} else {
if self.len() <= n {
return (Some(self.clone()), None);
} else {
match self {
Run::Mapped { data_begin, len } => (
Some(Run::Mapped {
data_begin: *data_begin,
len: n,
}),
Some(Run::Mapped {
data_begin: data_begin + n,
len: len - n,
}),
),
Run::UnMapped { len } => (
Some(Run::UnMapped { len: n }),
Some(Run::UnMapped { len: len - n }),
),
}
}
}
}
}
#[derive(Clone)]
struct ThinDev {
thin_id: u32,
dev_size: u64,
runs: Vec<Run>,
}
impl ThinDev {
fn emit(&self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
v.device_b(&xml::Device {
dev_id: self.thin_id,
mapped_blocks: self.dev_size,
transaction: 0,
creation_time: 0,
snap_time: 0,
})?;
let mut b = 0;
for r in &self.runs {
match r {
Run::Mapped { data_begin, len } => {
v.map(&xml::Map {
thin_begin: b,
data_begin: *data_begin,
time: 0,
len: *len,
})?;
b += len;
}
Run::UnMapped { len } => {
b += len;
}
}
}
v.device_e()?;
Ok(())
}
}
#[derive(Clone)]
enum SnapRunType {
Same,
Diff,
Hole,
}
#[derive(Clone)]
struct SnapRun(SnapRunType, u64);
fn mk_origin(thin_id: u32, total_len: u64, allocator: &mut Allocator) -> Result<ThinDev> {
let mut runs = Vec::new();
let mut b = 0;
while b < total_len {
let len = u64::min(thread_rng().gen_range(16, 64), total_len - b);
match thread_rng().gen_range(0, 2) {
0 => {
for data in allocator.alloc(len)? {
assert!(data.end >= data.start);
runs.push(Run::Mapped {
data_begin: data.start,
len: data.end - data.start,
});
}
}
1 => {
runs.push(Run::UnMapped { len });
}
_ => {
return Err(anyhow!("bad value returned from rng"));
}
};
b += len;
}
Ok(ThinDev {
thin_id,
dev_size: total_len,
runs,
})
}
fn mk_snap_mapping(
total_len: u64,
run_len: Range<u64>,
same_percent: usize,
diff_percent: usize,
) -> Vec<SnapRun> {
let mut runs = Vec::new();
let mut b = 0u64;
while b < total_len {
let len = u64::min(
total_len - b,
thread_rng().gen_range(run_len.start, run_len.end),
);
let n = thread_rng().gen_range(0, 100);
if n < same_percent {
runs.push(SnapRun(SnapRunType::Same, len));
} else if n < diff_percent {
runs.push(SnapRun(SnapRunType::Diff, len));
} else {
runs.push(SnapRun(SnapRunType::Hole, len));
}
b += len;
}
runs
}
fn split_runs(mut n: u64, runs: &Vec<Run>) -> (Vec<Run>, Vec<Run>) {
let mut before = Vec::new();
let mut after = Vec::new();
for r in runs {
match r.split(n) {
(Some(lhs), None) => {
before.push(lhs);
}
(Some(lhs), Some(rhs)) => {
before.push(lhs);
after.push(rhs);
}
(None, Some(rhs)) => {
after.push(rhs);
}
(None, None) => {}
}
n -= r.len();
}
(before, after)
}
fn apply_snap_runs(
origin: &Vec<Run>,
snap: &Vec<SnapRun>,
allocator: &mut Allocator,
) -> Result<Vec<Run>> {
let mut origin = origin.clone();
let mut runs = Vec::new();
for SnapRun(st, slen) in snap {
let (os, rest) = split_runs(*slen, &origin);
match st {
SnapRunType::Same => {
for o in os {
runs.push(o);
}
}
SnapRunType::Diff => {
for data in allocator.alloc(*slen)? {
runs.push(Run::Mapped {
data_begin: data.start,
len: data.end - data.start,
});
}
}
SnapRunType::Hole => {
runs.push(Run::UnMapped { len: *slen });
}
}
origin = rest;
}
Ok(runs)
}
// Snapshots share mappings, not neccessarily the entire ranges.
struct SnapS {
len: u64,
nr_snaps: u32,
// Snaps will differ from the origin by this percentage
percent_change: usize,
old_nr_data_blocks: u64,
new_nr_data_blocks: u64,
}
impl SnapS {
fn new(len: u64, nr_snaps: u32, percent_change: usize) -> Self {
let delta = len * (nr_snaps as u64) * (percent_change as u64) / 100;
let old_nr_data_blocks = len + 3 * delta;
let new_nr_data_blocks = len + 2 * delta;
SnapS {
len,
nr_snaps,
percent_change,
old_nr_data_blocks,
new_nr_data_blocks,
}
}
}
impl Scenario for SnapS {
fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512);
let origin = mk_origin(0, self.len, &mut allocator)?;
v.superblock_b(&common_sb(self.old_nr_data_blocks))?;
origin.emit(v)?;
v.superblock_e()?;
Ok(())
}
fn get_new_nr_blocks(&self) -> u64 {
self.new_nr_data_blocks
}