From 29d56f62a5046186469eb6011c4f6ced93fa4043 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Sun, 14 Jun 2020 08:17:46 +0100 Subject: [PATCH 01/52] wip --- src/bin/thin_check.cc | 55 ++++++++++++++++++++++++++++++++++++++ src/block_manager.rs | 4 +-- src/checksum.rs | 45 +++++++++++++++++++++++++++++++ src/lib.rs | 2 ++ src/pack/toplevel.rs | 39 +-------------------------- src/thin/mod.rs | 1 + src/thin/superblock.rs | 60 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 166 insertions(+), 40 deletions(-) create mode 100644 src/bin/thin_check.cc create mode 100644 src/checksum.rs create mode 100644 src/thin/mod.rs create mode 100644 src/thin/superblock.rs diff --git a/src/bin/thin_check.cc b/src/bin/thin_check.cc new file mode 100644 index 0000000..3ad7c01 --- /dev/null +++ b/src/bin/thin_check.cc @@ -0,0 +1,55 @@ +extern crate clap; +extern crate thinp; + +use clap::{App, Arg}; +use std::process; +use thinp::file_utils; + +use std::process::exit; + +fn main() { + let parser = App::new("thin_check") + .version(thinp::version::TOOLS_VERSION) + .about("Validates thin provisioning metadata on a device or file.") + .arg(Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet") + .value_name("QUIET")) + .arg(Arg::with_name("SB_ONLY") + .help("Only check the superblock.") + .long("super-block-only") + .value_name("SB_ONLY")) + .arg(Arg::with_name("ignore-non-fatal-errors") + .help("Only return a non-zero exit code if a fatal error is found.") + .long("ignore-non-fatal-errors") + .value_name("IGNORE_NON_FATAL")) + .arg(Arg::with_name("clear-needs-check-flag") + .help("Clears the 'needs_check' flag in the superblock") + .long("clear-needs-check") + .value_name("CLEAR_NEEDS_CHECK")) + .arg(Arg::with_name("OVERRIDE_MAPPING_ROOT") + .help("Specify a mapping root to use") + .long("override-mapping-root") + .value_name("OVERRIDE_MAPPING_ROOT") + .takes_value(true)) + .arg(Arg::with_name("METADATA_SNAPSHOT") + .help("Check the metadata snapshot on a live pool") + .short("m") + .long("metadata-snapshot") + .value_name("METADATA_SNAPSHOT")) + + let matches = parser.get_matches(); + let input_file = matches.value_of("INPUT").unwrap(); + let output_file = matches.value_of("OUTPUT").unwrap(); + + if !file_utils::file_exists(input_file) { + eprintln!("Couldn't find input file '{}'.", &input_file); + exit(1); + } + + if let Err(reason) = thinp::pack::toplevel::unpack(&input_file, &output_file) { + println!("Application error: {}", reason); + process::exit(1); + } +} diff --git a/src/block_manager.rs b/src/block_manager.rs index 78dd069..606eb30 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -6,8 +6,8 @@ use std::fs::File; pub const BLOCK_SIZE: usize = 4096; -#[repr(align(4096))] pub struct Block { + pub loc: u64, pub data: [u8; BLOCK_SIZE as usize], } @@ -41,7 +41,7 @@ impl BlockManager { fn read_block(&mut self, b: u64) -> io::Result { - let mut buf = Block {data: [0; BLOCK_SIZE]}; + let mut buf = Block {loc: b, data: [0; BLOCK_SIZE]}; self.input.seek(io::SeekFrom::Start(b * (BLOCK_SIZE as u64)))?; self.input.read_exact(&mut buf.data)?; diff --git a/src/checksum.rs b/src/checksum.rs new file mode 100644 index 0000000..1706532 --- /dev/null +++ b/src/checksum.rs @@ -0,0 +1,45 @@ +use byteorder::{LittleEndian, ReadBytesExt}; +use crc32c::crc32c; + +use std::io::Cursor; + +const BLOCK_SIZE: u64 = 4096; +const MAGIC: u64 = 0xa537a0aa6309ef77; +const SUPERBLOCK_CSUM_XOR: u32 = 160774; +const BITMAP_CSUM_XOR: u32 = 240779; +const INDEX_CSUM_XOR: u32 = 160478; +const BTREE_CSUM_XOR: u32 = 121107; + +fn checksum(buf: &[u8]) -> u32 { + crc32c(&buf[4..]) ^ 0xffffffff +} + +#[derive(PartialEq)] +pub enum BT { + SUPERBLOCK, + NODE, + INDEX, + BITMAP, + UNKNOWN, +} + +pub fn metadata_block_type(buf: &[u8]) -> BT { + if buf.len() != BLOCK_SIZE as usize { + return BT::UNKNOWN; + } + + // The checksum is always stored in the first u32 of the buffer. + let mut rdr = Cursor::new(buf); + let sum_on_disk = rdr.read_u32::().unwrap(); + let csum = checksum(buf); + let btype = csum ^ sum_on_disk; + + match btype { + SUPERBLOCK_CSUM_XOR => BT::SUPERBLOCK, + BTREE_CSUM_XOR => BT::NODE, + BITMAP_CSUM_XOR => BT::BITMAP, + INDEX_CSUM_XOR => BT::INDEX, + _ => BT::UNKNOWN, + } +} + diff --git a/src/lib.rs b/src/lib.rs index cf410c5..a2d5755 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,3 +19,5 @@ pub mod check; pub mod file_utils; pub mod pack; pub mod version; +pub mod thin; +pub mod checksum; diff --git a/src/pack/toplevel.rs b/src/pack/toplevel.rs index bd1757d..305a351 100644 --- a/src/pack/toplevel.rs +++ b/src/pack/toplevel.rs @@ -8,7 +8,6 @@ use std::{ fs::OpenOptions, io, io::prelude::*, - io::Cursor, io::Write, ops::DerefMut, sync::{Arc, Mutex}, @@ -20,14 +19,11 @@ use std::sync::mpsc::{sync_channel, Receiver}; use crate::file_utils; use crate::pack::node_encode::*; +use crate::checksum::*; const BLOCK_SIZE: u64 = 4096; const MAGIC: u64 = 0xa537a0aa6309ef77; const PACK_VERSION: u64 = 3; -const SUPERBLOCK_CSUM_XOR: u32 = 160774; -const BITMAP_CSUM_XOR: u32 = 240779; -const INDEX_CSUM_XOR: u32 = 160478; -const BTREE_CSUM_XOR: u32 = 121107; fn shuffle(v: &mut Vec) { let mut rng = rand::thread_rng(); @@ -209,39 +205,6 @@ where Ok(buf) } -fn checksum(buf: &[u8]) -> u32 { - crc32c::crc32c(&buf[4..]) ^ 0xffffffff -} - -#[derive(PartialEq)] -enum BT { - SUPERBLOCK, - NODE, - INDEX, - BITMAP, - UNKNOWN, -} - -fn metadata_block_type(buf: &[u8]) -> BT { - if buf.len() != BLOCK_SIZE as usize { - return BT::UNKNOWN; - } - - // The checksum is always stored in the first u32 of the buffer. - let mut rdr = Cursor::new(buf); - let sum_on_disk = rdr.read_u32::().unwrap(); - let csum = checksum(buf); - let btype = csum ^ sum_on_disk; - - match btype { - SUPERBLOCK_CSUM_XOR => BT::SUPERBLOCK, - BTREE_CSUM_XOR => BT::NODE, - BITMAP_CSUM_XOR => BT::BITMAP, - INDEX_CSUM_XOR => BT::INDEX, - _ => BT::UNKNOWN, - } -} - fn check(r: &PResult) { match r { Ok(_) => {} diff --git a/src/thin/mod.rs b/src/thin/mod.rs new file mode 100644 index 0000000..ec05bf6 --- /dev/null +++ b/src/thin/mod.rs @@ -0,0 +1 @@ +mod superblock; diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs new file mode 100644 index 0000000..722ea7f --- /dev/null +++ b/src/thin/superblock.rs @@ -0,0 +1,60 @@ +use crate::block_manager::*; +use crate::checksum::*; + +const SPACE_MAP_ROOT_SIZE: usize = 128; + +pub struct Superblock { + block: u64, + uuid: String, + version: u32, + time: u32, + transaction_id: u64, + metadata_snap: u64, + data_sm_root: [u8; SPACE_MAP_ROOT_SIZE], + metadata_sn_root: [u8; SPACE_MAP_ROOT_SIZE], + mapping_root: u64, + details_root: u64, + data_block_size: u32, +} + +pub enum CheckSeverity { + Fatal, + NonFatal, +} + +pub trait CheckError { + fn severity(&self) -> CheckSeverity; + fn block(&self) -> u64; + fn sub_errors(&self) -> Vec>; +} + +enum ErrorType { + BadChecksum, + BadBlockType(&'static str), + BadBlock(u64), + BadVersion(u32), + MetadataSnapOutOfBounds(u64), + MappingRootOutOfBounds(u64), + DetailsRootOutOfBounds(u64), +} + +struct SuperblockError { + severity: CheckSeverity, + kind: ErrorType, +} + +use SuperblockDamage::*; + +//------------------------------ + +pub fn check_type(b: &Block) -> Result<(), Box> { + match metadata_block_type(&b.data[0..]) { + SUPERBLOCK => Ok(()), + NODE => Err(Box::new(BadBlockType("BTree Node"))), + INDEX => Err(Box::new(BadBlockType("Space Map Index"))), + BITMAP => Err(Box::new(BadBlockType("Space Map Bitmap"))), + UNKNOWN => Err(Box::new(BadChecksum)), + } +} + +//------------------------------ From fdf641aff33b3159af0d3d9fed424efc84a00e85 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Sun, 21 Jun 2020 11:50:24 +0100 Subject: [PATCH 02/52] [thin_metadata_{pack,unpak}] use anyhow in toplevel. --- Cargo.lock | 28 ++++++++++++++++++++++++++++ Cargo.toml | 6 ++++-- src/pack/node_encode.rs | 29 ++++++++--------------------- src/pack/toplevel.rs | 35 ++++++++++++++++------------------- 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ca6d78..c1497ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,11 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "arrayvec" version = "0.4.12" @@ -380,6 +385,7 @@ dependencies = [ name = "thinp" version = "0.1.0" dependencies = [ + "anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)", "crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -393,6 +399,25 @@ dependencies = [ "quickcheck 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "quickcheck_macros 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thiserror" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "thiserror-impl 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.30 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -456,6 +481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" "checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum anyhow 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" "checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" @@ -501,6 +527,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" "checksum syn 1.0.30 (registry+https://github.com/rust-lang/crates.io-index)" = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)" = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08" +"checksum thiserror-impl 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)" = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" diff --git a/Cargo.toml b/Cargo.toml index e1e1c7c..fc98546 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" license = "GPL3" [dependencies] +anyhow = "1.0" byteorder = "1.3" clap = "2.33" crc32c = "0.4" @@ -14,9 +15,10 @@ libc = "0.2.71" nix = "0.17" nom = "5.1" num_cpus = "1.13" -rand = "0.7" -num-traits = "0.2" num-derive = "0.3" +num-traits = "0.2" +rand = "0.7" +thiserror = "1.0" [dev-dependencies] quickcheck = "0.9" diff --git a/src/pack/node_encode.rs b/src/pack/node_encode.rs index c5e8370..a8a54a6 100644 --- a/src/pack/node_encode.rs +++ b/src/pack/node_encode.rs @@ -1,3 +1,4 @@ +use thiserror::Error; use std::{io, io::Write}; use nom::{bytes::complete::*, number::complete::*, IResult}; @@ -6,41 +7,27 @@ use crate::pack::vm::*; //------------------------------------------- -#[derive(Debug)] +#[derive(Error, Debug)] pub enum PackError { + #[error("Couldn't parse binary data")] ParseError, - IOError, -} -impl std::error::Error for PackError {} + #[error("Write error")] + WriteError { source: std::io::Error }, +} pub type PResult = Result; fn nom_to_pr(r: IResult<&[u8], T>) -> PResult<(&[u8], T)> { - match r { - Ok(v) => Ok(v), - Err(_) => Err(PackError::ParseError), - } + r.map_err(|_source| PackError::ParseError) } fn io_to_pr(r: io::Result) -> PResult { - match r { - Ok(v) => Ok(v), - Err(_) => Err(PackError::IOError), - } + r.map_err(|source| PackError::WriteError {source}) } //------------------------------------------- -impl std::fmt::Display for PackError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - PackError::ParseError => write!(f, "parse error"), - PackError::IOError => write!(f, "IO error"), - } - } -} - fn run64(i: &[u8], count: usize) -> IResult<&[u8], Vec> { let (i, ns) = nom::multi::many_m_n(count, count, le_u64)(i)?; Ok((i, ns)) diff --git a/src/pack/toplevel.rs b/src/pack/toplevel.rs index bd1757d..97f5eb7 100644 --- a/src/pack/toplevel.rs +++ b/src/pack/toplevel.rs @@ -1,3 +1,4 @@ +use anyhow::{anyhow, Context, Result}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use flate2::{read::ZlibDecoder, write::ZlibEncoder, Compression}; @@ -67,8 +68,8 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec> { vs } -pub fn pack(input_file: &str, output_file: &str) -> Result<(), Box> { - let nr_blocks = get_nr_blocks(&input_file)?; +pub fn pack(input_file: &str, output_file: &str) -> Result<()> { + let nr_blocks = get_nr_blocks(&input_file).context("getting nr blocks")?; let nr_jobs = std::cmp::max(1, std::cmp::min(num_cpus::get() as u64, nr_blocks / 128)); let chunk_vecs = mk_chunk_vecs(nr_blocks, nr_jobs); @@ -85,7 +86,7 @@ pub fn pack(input_file: &str, output_file: &str) -> Result<(), Box> { .truncate(true) .open(output_file)?; - write_header(&output, nr_blocks)?; + write_header(&output, nr_blocks).context("unable to write pack file header")?; let sync_input = Arc::new(Mutex::new(input)); let sync_output = Arc::new(Mutex::new(output)); @@ -108,7 +109,7 @@ fn crunch( input: Arc>, output: Arc>, ranges: Vec<(u64, u64)>, -) -> io::Result<()> +) -> Result<()> where R: Read + Seek, W: Write, @@ -128,7 +129,7 @@ where let kind = metadata_block_type(data); if kind != BT::UNKNOWN { z.write_u64::(b)?; - pack_block(&mut z, kind, &data); + pack_block(&mut z, kind, &data)?; written += 1; if written == 1024 { @@ -242,22 +243,18 @@ fn metadata_block_type(buf: &[u8]) -> BT { } } -fn check(r: &PResult) { - match r { - Ok(_) => {} - Err(PackError::ParseError) => panic!("parse error"), - Err(PackError::IOError) => panic!("io error"), - } -} - -fn pack_block(w: &mut W, kind: BT, buf: &[u8]) { +fn pack_block(w: &mut W, kind: BT, buf: &[u8]) -> Result<()> { match kind { - BT::SUPERBLOCK => check(&pack_superblock(w, buf)), - BT::NODE => check(&pack_btree_node(w, buf)), - BT::INDEX => check(&pack_index(w, buf)), - BT::BITMAP => check(&pack_bitmap(w, buf)), - BT::UNKNOWN => {panic!("asked to pack an unknown block type")} + BT::SUPERBLOCK => pack_superblock(w, buf).context("unable to pack superblock")?, + BT::NODE => pack_btree_node(w, buf).context("unable to pack btree node")?, + BT::INDEX => pack_index(w, buf).context("unable to pack space map index")?, + BT::BITMAP => pack_bitmap(w, buf).context("unable to pack space map bitmap")?, + BT::UNKNOWN => { + return Err(anyhow!("asked to pack an unknown block type")) + } } + + Ok(()) } fn write_zero_block(w: &mut W, b: u64) -> io::Result<()> From 1398cf31d1c29e2e72b6c342bc6bb8a88593a520 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 27 Jul 2020 15:53:42 +0100 Subject: [PATCH 03/52] [thin_check (Rust)] work in progress --- src/bin/thin_check.rs | 84 +++++++++++++++---------- src/block_manager.rs | 139 ++++++++++++++++++++++++++++++++++------- src/lib.rs | 2 - src/thin/check.rs | 20 ++++-- src/thin/superblock.rs | 6 +- 5 files changed, 187 insertions(+), 64 deletions(-) diff --git a/src/bin/thin_check.rs b/src/bin/thin_check.rs index 3ad7c01..46ae7ef 100644 --- a/src/bin/thin_check.rs +++ b/src/bin/thin_check.rs @@ -2,6 +2,7 @@ extern crate clap; extern crate thinp; use clap::{App, Arg}; +use std::path::Path; use std::process; use thinp::file_utils; @@ -9,46 +10,63 @@ use std::process::exit; fn main() { let parser = App::new("thin_check") - .version(thinp::version::TOOLS_VERSION) + .version(thinp::version::TOOLS_VERSION) .about("Validates thin provisioning metadata on a device or file.") - .arg(Arg::with_name("QUIET") - .help("Suppress output messages, return only exit code.") - .short("q") - .long("quiet") - .value_name("QUIET")) - .arg(Arg::with_name("SB_ONLY") - .help("Only check the superblock.") - .long("super-block-only") - .value_name("SB_ONLY")) - .arg(Arg::with_name("ignore-non-fatal-errors") - .help("Only return a non-zero exit code if a fatal error is found.") - .long("ignore-non-fatal-errors") - .value_name("IGNORE_NON_FATAL")) - .arg(Arg::with_name("clear-needs-check-flag") - .help("Clears the 'needs_check' flag in the superblock") - .long("clear-needs-check") - .value_name("CLEAR_NEEDS_CHECK")) - .arg(Arg::with_name("OVERRIDE_MAPPING_ROOT") - .help("Specify a mapping root to use") - .long("override-mapping-root") - .value_name("OVERRIDE_MAPPING_ROOT") - .takes_value(true)) - .arg(Arg::with_name("METADATA_SNAPSHOT") - .help("Check the metadata snapshot on a live pool") - .short("m") - .long("metadata-snapshot") - .value_name("METADATA_SNAPSHOT")) + .arg( + Arg::with_name("QUIET") + .help("Suppress output messages, return only exit code.") + .short("q") + .long("quiet") + .value_name("QUIET"), + ) + .arg( + Arg::with_name("SB_ONLY") + .help("Only check the superblock.") + .long("super-block-only") + .value_name("SB_ONLY"), + ) + .arg( + Arg::with_name("ignore-non-fatal-errors") + .help("Only return a non-zero exit code if a fatal error is found.") + .long("ignore-non-fatal-errors") + .value_name("IGNORE_NON_FATAL"), + ) + .arg( + Arg::with_name("clear-needs-check-flag") + .help("Clears the 'needs_check' flag in the superblock") + .long("clear-needs-check") + .value_name("CLEAR_NEEDS_CHECK"), + ) + .arg( + Arg::with_name("OVERRIDE_MAPPING_ROOT") + .help("Specify a mapping root to use") + .long("override-mapping-root") + .value_name("OVERRIDE_MAPPING_ROOT") + .takes_value(true), + ) + .arg( + Arg::with_name("METADATA_SNAPSHOT") + .help("Check the metadata snapshot on a live pool") + .short("m") + .long("metadata-snapshot") + .value_name("METADATA_SNAPSHOT"), + ) + .arg( + Arg::with_name("INPUT") + .help("Specify the input device to check") + .required(true) + .index(1), + ); let matches = parser.get_matches(); - let input_file = matches.value_of("INPUT").unwrap(); - let output_file = matches.value_of("OUTPUT").unwrap(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); if !file_utils::file_exists(input_file) { - eprintln!("Couldn't find input file '{}'.", &input_file); + eprintln!("Couldn't find input file '{:?}'.", &input_file); exit(1); } - - if let Err(reason) = thinp::pack::toplevel::unpack(&input_file, &output_file) { + + if let Err(reason) = thinp::thin::check::check(&input_file) { println!("Application error: {}", reason); process::exit(1); } diff --git a/src/block_manager.rs b/src/block_manager.rs index 606eb30..df1274e 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -1,51 +1,148 @@ +use anyhow::{anyhow, Result}; +use rio::{self, Completion, Rio}; +use std::alloc::{alloc, dealloc, Layout}; +use std::collections::HashMap; +use std::fs::File; +use std::fs::OpenOptions; use std::io; use std::io::{Read, Seek}; -use std::fs::OpenOptions; use std::os::unix::fs::OpenOptionsExt; -use std::fs::File; +use std::path::Path; +use std::sync::{Arc, Mutex}; pub const BLOCK_SIZE: usize = 4096; +const ALIGN: usize = 4096; +// FIXME: introduce a cache +// FIXME: use O_DIRECT +#[derive(Debug)] pub struct Block { pub loc: u64, - pub data: [u8; BLOCK_SIZE as usize], -} - -pub struct BlockManager { - pub nr_blocks: u64, - input: File, + data: *mut u8, } -fn get_nr_blocks(path: &str) -> io::Result { +impl Block { + pub fn new(loc: u64) -> Block { + let layout = Layout::from_size_align(BLOCK_SIZE, ALIGN).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "out of memory"); + Block { loc, data: ptr } + } + + fn get_data(&self) -> &mut [u8] { + unsafe { std::slice::from_raw_parts_mut::<'static>(self.data, BLOCK_SIZE) } + } +} + +impl Drop for Block { + fn drop(&mut self) { + let layout = Layout::from_size_align(BLOCK_SIZE, ALIGN).unwrap(); + unsafe { + dealloc(self.data, layout); + } + } +} + +//------------------------------------------ + +pub trait IoEngine { + fn get_nr_blocks(&self) -> u64; + fn read(&mut self, blocks: &mut Vec) -> Result<()>; +} + +fn get_nr_blocks(path: &Path) -> io::Result { let metadata = std::fs::metadata(path)?; Ok(metadata.len() / (BLOCK_SIZE as u64)) } -impl BlockManager { - pub fn new(path: &str, _cache_size: usize) -> io::Result { +//------------------------------------------ + +pub struct SyncIoEngine { + nr_blocks: u64, + input: File, +} + +impl SyncIoEngine { + pub fn new(path: &Path) -> Result { let input = OpenOptions::new() .read(true) .write(false) .custom_flags(libc::O_DIRECT) .open(path)?; - Ok(BlockManager { + let ring = rio::new()?; + + Ok(SyncIoEngine { + nr_blocks: get_nr_blocks(path)?, + input, + }) + } +} + +impl IoEngine for SyncIoEngine { + fn get_nr_blocks(&self) -> u64 { + self.nr_blocks + } + + fn read(&mut self, blocks: &mut Vec) -> Result<()> { + for b in blocks.into_iter() { + self.input.seek(io::SeekFrom::Start(0))?; + self.input.read_exact(&mut b.get_data())?; + } + + Ok(()) + } +} + +//------------------------------------------ + +/* +pub struct AsyncIoEngine { + ring: Rio, + nr_blocks: u64, + input: File, +} + +impl AsyncIoEngine { + pub fn new(path: &Path) -> Result { + let input = OpenOptions::new() + .read(true) + .write(false) + .custom_flags(libc::O_DIRECT) + .open(path)?; + + let ring = rio::new()?; + + Ok(IoEngine { + ring, nr_blocks: get_nr_blocks(path)?, input, }) } - pub fn get(&mut self, b: u64) -> io::Result { - self.read_block(b) - } + pub fn read(&self, blocks: &mut Vec) -> Result<()> { + // FIXME: using a bounce buffer as a hack, since b.get_data() will not have + // a big enough lifetime. + let mut bounce_buffer = vec![0; blocks.len() * BLOCK_SIZE]; + let mut completions = Vec::new(); - fn read_block(&mut self, b: u64) -> io::Result - { - let mut buf = Block {loc: b, data: [0; BLOCK_SIZE]}; + for n in 0..blocks.len() { + let b = &blocks[n]; + let at = b.loc * BLOCK_SIZE as u64; + let completion = self.ring.read_at(&self.input, &slice, at); + completions.push(completion); + } - self.input.seek(io::SeekFrom::Start(b * (BLOCK_SIZE as u64)))?; - self.input.read_exact(&mut buf.data)?; + for c in completions { + let n = c.wait()?; + if n != BLOCK_SIZE { + return Err(anyhow!("short read")); + } + } - Ok(buf) + // copy out of the bounce buffer + + Ok(()) } } +*/ diff --git a/src/lib.rs b/src/lib.rs index d63b864..b7b2082 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,11 +16,9 @@ extern crate quickcheck; extern crate quickcheck_macros; pub mod block_manager; -pub mod check; pub mod file_utils; pub mod pack; pub mod shrink; pub mod thin; pub mod version; -pub mod thin; pub mod checksum; diff --git a/src/thin/check.rs b/src/thin/check.rs index a97ca01..580e468 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,13 +1,23 @@ use std::error::Error; +use std::path::Path; +use std::time::{Duration, Instant}; +use std::thread; +use std::sync::{Arc, Mutex}; -use crate::block_manager::BlockManager; +use crate::block_manager::{Block, IoEngine, SyncIoEngine, BLOCK_SIZE}; -pub fn check(dev: &str) -> Result<(), Box> { - let mut bm = BlockManager::new(dev, 1024)?; +pub fn check(dev: &Path) -> Result<(), Box> { + let mut engine = SyncIoEngine::new(dev)?; + let count = 4096; - for b in 0..100 { - let _block = bm.get(b)?; + let mut blocks = Vec::new(); + for n in 0..count { + blocks.push(Block::new(n)); } + let now = Instant::now(); + engine.read(&mut blocks)?; + println!("read {} blocks in {} ms", count, now.elapsed().as_millis()); + Ok(()) } diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index 6b6d701..3b43319 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use crate::block_manager::*; use crate::checksum::*; @@ -12,7 +11,7 @@ pub struct Superblock { transaction_id: u64, metadata_snap: u64, data_sm_root: [u8; SPACE_MAP_ROOT_SIZE], - metadata_sn_root: [u8; SPACE_MAP_ROOT_SIZE], + metadata_sm_root: [u8; SPACE_MAP_ROOT_SIZE], mapping_root: u64, details_root: u64, data_block_size: u32, @@ -44,6 +43,7 @@ struct SuperblockError { kind: ErrorType, } +/* use SuperblockDamage::*; //------------------------------ @@ -57,5 +57,5 @@ pub fn check_type(b: &Block) -> Result<()> { UNKNOWN => Err(Box::new(BadChecksum)), } } - +*/ //------------------------------ From 062a1b8a2cbce1ef4aec3b0cf35a03963b8360c8 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 28 Jul 2020 10:51:48 +0100 Subject: [PATCH 04/52] [thin_check (rust)] Walk mapping tree. Sync IO, no checks beyond checksumming. --- Cargo.lock | 20 ++-- src/block_manager.rs | 23 +++-- src/checksum.rs | 2 +- src/thin/check.rs | 210 ++++++++++++++++++++++++++++++++++++++--- src/thin/superblock.rs | 82 +++++++++++----- 5 files changed, 284 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3bbf30..0ac75d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,9 +26,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" +checksum = "6b602bfe940d21c130f3895acd65221e8a61270debe89d628b9cb4e3ccb8569b" [[package]] name = "arrayvec" @@ -361,18 +361,18 @@ checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" [[package]] name = "pin-project" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" +checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" +checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f" dependencies = [ "proc-macro2", "quote", @@ -393,9 +393,9 @@ checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" [[package]] name = "proc-macro-hack" -version = "0.5.16" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" [[package]] name = "proc-macro-nested" @@ -562,9 +562,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0" +checksum = "4cdb98bcb1f9d81d07b536179c269ea15999b5d14ea958196413869445bb5250" dependencies = [ "proc-macro2", "quote", diff --git a/src/block_manager.rs b/src/block_manager.rs index df1274e..2ce48cf 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -29,8 +29,8 @@ impl Block { Block { loc, data: ptr } } - fn get_data(&self) -> &mut [u8] { - unsafe { std::slice::from_raw_parts_mut::<'static>(self.data, BLOCK_SIZE) } + pub fn get_data<'a>(&self) -> &'a mut [u8] { + unsafe { std::slice::from_raw_parts_mut::<'a>(self.data, BLOCK_SIZE) } } } @@ -47,7 +47,8 @@ impl Drop for Block { pub trait IoEngine { fn get_nr_blocks(&self) -> u64; - fn read(&mut self, blocks: &mut Vec) -> Result<()>; + fn read(&mut self, block: &mut Block) -> Result<()>; + fn read_many(&mut self, blocks: &mut Vec) -> Result<()>; } fn get_nr_blocks(path: &Path) -> io::Result { @@ -70,8 +71,6 @@ impl SyncIoEngine { .custom_flags(libc::O_DIRECT) .open(path)?; - let ring = rio::new()?; - Ok(SyncIoEngine { nr_blocks: get_nr_blocks(path)?, input, @@ -84,10 +83,16 @@ impl IoEngine for SyncIoEngine { self.nr_blocks } - fn read(&mut self, blocks: &mut Vec) -> Result<()> { - for b in blocks.into_iter() { - self.input.seek(io::SeekFrom::Start(0))?; - self.input.read_exact(&mut b.get_data())?; + fn read(&mut self, b: &mut Block) -> Result<()> { + self.input.seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; + self.input.read_exact(&mut b.get_data())?; + + Ok(()) + } + + fn read_many(&mut self, blocks: &mut Vec) -> Result<()> { + for b in blocks { + self.read(b); } Ok(()) diff --git a/src/checksum.rs b/src/checksum.rs index 1706532..25d7205 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -14,7 +14,7 @@ fn checksum(buf: &[u8]) -> u32 { crc32c(&buf[4..]) ^ 0xffffffff } -#[derive(PartialEq)] +#[derive(Debug, PartialEq)] pub enum BT { SUPERBLOCK, NODE, diff --git a/src/thin/check.rs b/src/thin/check.rs index 580e468..cbdf603 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,23 +1,211 @@ +use anyhow::{anyhow, Result}; +use nom::{bytes::complete::*, number::complete::*, IResult}; +use std::collections::HashSet; use std::error::Error; use std::path::Path; -use std::time::{Duration, Instant}; -use std::thread; use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; use crate::block_manager::{Block, IoEngine, SyncIoEngine, BLOCK_SIZE}; +use crate::checksum; +use crate::thin::superblock::*; -pub fn check(dev: &Path) -> Result<(), Box> { - let mut engine = SyncIoEngine::new(dev)?; - let count = 4096; +trait ValueType { + type Value; + fn unpack(data: &[u8]) -> IResult<&[u8], Self::Value>; +} - let mut blocks = Vec::new(); - for n in 0..count { - blocks.push(Block::new(n)); +struct NodeHeader { + is_leaf: bool, + block: u64, + nr_entries: u32, + max_entries: u32, + value_size: u32, +} + +const INTERNAL_NODE: u32 = 1; +const LEAF_NODE: u32 = 2; + +fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> { + let (i, _csum) = le_u32(data)?; + let (i, flags) = le_u32(i)?; + let (i, block) = le_u64(i)?; + let (i, nr_entries) = le_u32(i)?; + let (i, max_entries) = le_u32(i)?; + let (i, value_size) = le_u32(i)?; + let (i, _padding) = le_u32(i)?; + + Ok(( + i, + NodeHeader { + is_leaf: flags == LEAF_NODE, + block, + nr_entries, + max_entries, + value_size, + }, + )) +} + +enum Node { + Internal { + header: NodeHeader, + keys: Vec, + values: Vec, + }, + Leaf { + header: NodeHeader, + keys: Vec, + values: Vec, + }, +} + +fn unpack_node_(data: &[u8]) -> IResult<&[u8], Node> { + use nom::multi::count; + + let (i, header) = unpack_node_header(data)?; + let (i, keys) = count(le_u64, header.nr_entries as usize)(i)?; + + let nr_free = header.max_entries - header.nr_entries; + let (i, _padding) = count(le_u64, nr_free as usize)(i)?; + + if header.is_leaf { + let (i, values) = count(V::unpack, header.nr_entries as usize)(i)?; + Ok(( + i, + Node::Leaf { + header, + keys, + values, + }, + )) + } else { + let (i, values) = count(le_u64, header.nr_entries as usize)(i)?; + Ok(( + i, + Node::Internal { + header, + keys, + values, + }, + )) + } +} + +fn unpack_node(data: &[u8]) -> Result> { + if let Ok((_i, node)) = unpack_node_(data) { + Ok(node) + } else { + Err(anyhow!("couldn't unpack btree node")) + } +} + +struct ValueU64; + +impl ValueType for ValueU64 { + type Value = u64; + fn unpack(i: &[u8]) -> IResult<&[u8], u64> { + le_u64(i) + } +} + +struct BlockTime { + block: u64, + time: u32, +} + +struct ValueBlockTime; + +impl ValueType for ValueBlockTime { + type Value = BlockTime; + fn unpack(i: &[u8]) -> IResult<&[u8], BlockTime> { + let (i, n) = le_u64(i)?; + let block = n >> 24; + let time = n & ((1 << 24) - 1); + + Ok(( + i, + BlockTime { + block, + time: time as u32, + }, + )) + } +} + +enum MappingLevel { + Top, + Bottom, +} + +fn walk_mapping_tree( + engine: &mut E, + seen: &mut HashSet, + level: MappingLevel, + b: u64, +) -> Result<()> { + if seen.contains(&b) { + return Ok(()); + } else { + seen.insert(b); } - let now = Instant::now(); - engine.read(&mut blocks)?; - println!("read {} blocks in {} ms", count, now.elapsed().as_millis()); + let mut b = Block::new(b); + engine.read(&mut b)?; + + let bt = checksum::metadata_block_type(b.get_data()); + if bt != checksum::BT::NODE { + return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); + } + + match level { + MappingLevel::Top => { + let node = unpack_node::(&b.get_data())?; + match node { + Node::Leaf {header: header, keys: _keys, values} => { + for b in &values { + walk_mapping_tree(engine, seen, MappingLevel::Bottom, *b)?; + } + }, + Node::Internal {header: header, keys: _keys, values} => { + for b in &values { + walk_mapping_tree(engine, seen, MappingLevel::Top, *b)?; + } + }, + } + }, + MappingLevel::Bottom => { + let node = unpack_node::(&b.get_data())?; + match node { + Node::Leaf {header: header, keys: _keys, values} => { + // FIXME: check in bounds + }, + Node::Internal {header: header, keys: _keys, values} => { + for b in &values { + walk_mapping_tree(engine, seen, MappingLevel::Bottom, *b)?; + } + }, + } + } + } + + Ok(()) +} + +pub fn check(dev: &Path) -> Result<()> { + let mut engine = SyncIoEngine::new(dev)?; + + let now = Instant::now(); + let sb = read_superblock(&mut engine, SUPERBLOCK_LOCATION)?; + eprintln!("{:?}", sb); + let mut seen = HashSet::new(); + walk_mapping_tree(&mut engine, &mut seen, MappingLevel::Top, sb.mapping_root)?; + println!( + "read superblock, mapping root at {}, {} ms", + sb.mapping_root, + now.elapsed().as_millis() + ); Ok(()) } diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index 3b43319..f7c3546 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -1,20 +1,26 @@ use crate::block_manager::*; +use crate::block_manager::*; use crate::checksum::*; +use anyhow::{anyhow, Result}; +use nom::{bytes::complete::*, number::complete::*, IResult}; +pub const SUPERBLOCK_LOCATION: u64 = 0; +const UUID_SIZE: usize = 16; const SPACE_MAP_ROOT_SIZE: usize = 128; +#[derive(Debug)] pub struct Superblock { - block: u64, - uuid: String, - version: u32, - time: u32, - transaction_id: u64, - metadata_snap: u64, - data_sm_root: [u8; SPACE_MAP_ROOT_SIZE], - metadata_sm_root: [u8; SPACE_MAP_ROOT_SIZE], - mapping_root: u64, - details_root: u64, - data_block_size: u32, + pub block: u64, + //uuid: [u8; UUID_SIZE], + pub version: u32, + pub time: u32, + pub transaction_id: u64, + pub metadata_snap: u64, + //data_sm_root: [u8; SPACE_MAP_ROOT_SIZE], + //metadata_sm_root: [u8; SPACE_MAP_ROOT_SIZE], + pub mapping_root: u64, + pub details_root: u64, + pub data_block_size: u32, } pub enum CheckSeverity { @@ -43,19 +49,51 @@ struct SuperblockError { kind: ErrorType, } -/* -use SuperblockDamage::*; +fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { + let (i, _csum) = le_u32(data)?; + let (i, _flags) = le_u32(i)?; + let (i, block) = le_u64(i)?; + let (i, _uuid) = take(16usize)(i)?; + let (i, _magic) = le_u64(i)?; + let (i, version) = le_u32(i)?; + let (i, time) = le_u32(i)?; + let (i, transaction_id) = le_u64(i)?; + let (i, metadata_snap) = le_u64(i)?; + let (i, _data_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; + let (i, _metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; + let (i, mapping_root) = le_u64(i)?; + let (i, details_root) = le_u64(i)?; + let (i, data_block_size) = le_u32(i)?; + let (i, _metadata_block_size) = le_u32(i)?; + let (i, _metadata_nr_blocks) = le_u64(i)?; -//------------------------------ + Ok(( + i, + Superblock { + block, + //uuid: uuid[0..UUID_SIZE], + version, + time, + transaction_id, + metadata_snap, + //data_sm_root, + //metadata_sm_root, + mapping_root, + details_root, + data_block_size, + }, + )) +} -pub fn check_type(b: &Block) -> Result<()> { - match metadata_block_type(&b.data[0..]) { - SUPERBLOCK => Ok(()), - NODE => Err(Box::new(BadBlockType("BTree Node"))), - INDEX => Err(Box::new(BadBlockType("Space Map Index"))), - BITMAP => Err(Box::new(BadBlockType("Space Map Bitmap"))), - UNKNOWN => Err(Box::new(BadChecksum)), +pub fn read_superblock(engine: &mut E, loc: u64) -> Result { + let mut b = Block::new(loc); + engine.read(&mut b)?; + + if let Ok((_, sb)) = unpack(&b.get_data()) { + Ok(sb) + } else { + Err(anyhow!("couldn't unpack superblock")) } } -*/ + //------------------------------ From a90294e2793e04cb7ab2cc725f47e4b994351c5b Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 28 Jul 2020 11:45:25 +0100 Subject: [PATCH 05/52] [thin_check (rust)] read many blocks at once. We need to switch to io_uring to really get the benefit of this. --- src/thin/check.rs | 84 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 27 deletions(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index cbdf603..a46385a 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -134,25 +134,41 @@ impl ValueType for ValueBlockTime { } } +#[derive(Copy, Clone)] enum MappingLevel { Top, Bottom, } -fn walk_mapping_tree( +fn walk_nodes( engine: &mut E, seen: &mut HashSet, level: MappingLevel, - b: u64, + bs: &Vec, ) -> Result<()> { - if seen.contains(&b) { - return Ok(()); - } else { - seen.insert(b); + let mut blocks = Vec::new(); + for b in bs { + if !seen.contains(b) { + blocks.push(Block::new(*b)); + } } - let mut b = Block::new(b); - engine.read(&mut b)?; + engine.read_many(&mut blocks)?; + + for b in blocks { + walk_node(engine, seen, level, &b); + } + + Ok(()) +} + +fn walk_node( + engine: &mut E, + seen: &mut HashSet, + level: MappingLevel, + b: &Block, +) -> Result<()> { + seen.insert(b.loc); let bt = checksum::metadata_block_type(b.get_data()); if bt != checksum::BT::NODE { @@ -163,29 +179,39 @@ fn walk_mapping_tree( MappingLevel::Top => { let node = unpack_node::(&b.get_data())?; match node { - Node::Leaf {header: header, keys: _keys, values} => { - for b in &values { - walk_mapping_tree(engine, seen, MappingLevel::Bottom, *b)?; - } - }, - Node::Internal {header: header, keys: _keys, values} => { - for b in &values { - walk_mapping_tree(engine, seen, MappingLevel::Top, *b)?; - } - }, + Node::Leaf { + header: header, + keys: _keys, + values, + } => { + walk_nodes(engine, seen, MappingLevel::Bottom, &values)?; + } + Node::Internal { + header: header, + keys: _keys, + values, + } => { + walk_nodes(engine, seen, MappingLevel::Top, &values)?; + } } - }, + } MappingLevel::Bottom => { let node = unpack_node::(&b.get_data())?; match node { - Node::Leaf {header: header, keys: _keys, values} => { + Node::Leaf { + header: header, + keys: _keys, + values, + } => { // FIXME: check in bounds - }, - Node::Internal {header: header, keys: _keys, values} => { - for b in &values { - walk_mapping_tree(engine, seen, MappingLevel::Bottom, *b)?; - } - }, + } + Node::Internal { + header: header, + keys: _keys, + values, + } => { + walk_nodes(engine, seen, MappingLevel::Bottom, &values)?; + } } } } @@ -200,7 +226,11 @@ pub fn check(dev: &Path) -> Result<()> { let sb = read_superblock(&mut engine, SUPERBLOCK_LOCATION)?; eprintln!("{:?}", sb); let mut seen = HashSet::new(); - walk_mapping_tree(&mut engine, &mut seen, MappingLevel::Top, sb.mapping_root)?; + + let mut root = Block::new(sb.mapping_root); + engine.read(&mut root)?; + + walk_node(&mut engine, &mut seen, MappingLevel::Top, &root)?; println!( "read superblock, mapping root at {}, {} ms", sb.mapping_root, From e9abdd9c880c61c4b43e35fdf34044045fd90036 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 28 Jul 2020 12:57:30 +0100 Subject: [PATCH 06/52] [thin_check (rust)] Switch to a different io_uring crate. This one works. --- Cargo.lock | 21 ++++++----- Cargo.toml | 2 +- src/block_manager.rs | 87 ++++++++++++++++++++++++++++++-------------- src/thin/check.rs | 8 ++-- 4 files changed, 76 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ac75d3..0478876 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,6 +250,16 @@ dependencies = [ "libc", ] +[[package]] +name = "io-uring" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31e11f8867575fc79a3e73e5f554d0b7386bc4a6f469039e8a83136c724fd81" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -527,15 +537,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rio" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce02a35f6fbcc9c5ce0674f17d33fb56afbe0bec6f6263affed4b1ebf594d95d" -dependencies = [ - "libc", -] - [[package]] name = "ryu" version = "1.0.5" @@ -605,6 +606,7 @@ dependencies = [ "fixedbitset", "flate2", "futures", + "io-uring", "libc", "nix", "nom", @@ -615,7 +617,6 @@ dependencies = [ "quickcheck", "quickcheck_macros", "rand", - "rio", "tempfile", "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index 0dd5c32..ddcf6bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ crc32c = "0.4" fixedbitset = "0.3" futures = "0.3" flate2 = "1.0" +io-uring = "0.3" libc = "0.2.71" nix = "0.17" nom = "5.1" @@ -21,7 +22,6 @@ num-derive = "0.3" num-traits = "0.2" quick-xml = "0.18" rand = "0.7" -rio = "0.9" tempfile = "3.1" thiserror = "1.0" diff --git a/src/block_manager.rs b/src/block_manager.rs index 2ce48cf..f813012 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -use rio::{self, Completion, Rio}; use std::alloc::{alloc, dealloc, Layout}; use std::collections::HashMap; use std::fs::File; @@ -7,14 +6,17 @@ use std::fs::OpenOptions; use std::io; use std::io::{Read, Seek}; use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::io::AsRawFd; use std::path::Path; use std::sync::{Arc, Mutex}; +use io_uring::opcode::{self, types}; +use io_uring::IoUring; + +//------------------------------------------ pub const BLOCK_SIZE: usize = 4096; const ALIGN: usize = 4096; -// FIXME: introduce a cache -// FIXME: use O_DIRECT #[derive(Debug)] pub struct Block { pub loc: u64, @@ -84,7 +86,8 @@ impl IoEngine for SyncIoEngine { } fn read(&mut self, b: &mut Block) -> Result<()> { - self.input.seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; + self.input + .seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; self.input.read_exact(&mut b.get_data())?; Ok(()) @@ -101,53 +104,83 @@ impl IoEngine for SyncIoEngine { //------------------------------------------ -/* pub struct AsyncIoEngine { - ring: Rio, + ring: IoUring, nr_blocks: u64, input: File, } impl AsyncIoEngine { - pub fn new(path: &Path) -> Result { + pub fn new(path: &Path, queue_len: u32) -> Result { let input = OpenOptions::new() .read(true) .write(false) .custom_flags(libc::O_DIRECT) .open(path)?; - let ring = rio::new()?; - - Ok(IoEngine { - ring, + Ok(AsyncIoEngine { + ring: IoUring::new(queue_len)?, nr_blocks: get_nr_blocks(path)?, input, }) } +} - pub fn read(&self, blocks: &mut Vec) -> Result<()> { - // FIXME: using a bounce buffer as a hack, since b.get_data() will not have - // a big enough lifetime. - let mut bounce_buffer = vec![0; blocks.len() * BLOCK_SIZE]; - let mut completions = Vec::new(); +impl IoEngine for AsyncIoEngine { + fn get_nr_blocks(&self) -> u64 { + self.nr_blocks + } - for n in 0..blocks.len() { - let b = &blocks[n]; - let at = b.loc * BLOCK_SIZE as u64; - let completion = self.ring.read_at(&self.input, &slice, at); - completions.push(completion); + fn read(&mut self, b: &mut Block) -> Result<()> { + let fd = types::Target::Fd(self.input.as_raw_fd()); + let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32).offset(b.loc as i64 * BLOCK_SIZE as i64); + + unsafe { + let mut queue = self.ring.submission().available(); + queue.push(read_e.build().user_data(1)) + .ok() + .expect("queue is full"); } - for c in completions { - let n = c.wait()?; - if n != BLOCK_SIZE { - return Err(anyhow!("short read")); + self.ring.submit_and_wait(1)?; + + let cqes = self.ring.completion().available().collect::>(); + + // FIXME: return proper errors + assert_eq!(cqes.len(), 1); + assert_eq!(cqes[0].user_data(), 1); + assert_eq!(cqes[0].result(), BLOCK_SIZE as i32); + + Ok(()) + } + + fn read_many(&mut self, blocks: &mut Vec) -> Result<()> { + let count = blocks.len(); + let fd = types::Target::Fd(self.input.as_raw_fd()); + + for b in blocks.into_iter() { + let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32).offset(b.loc as i64 * BLOCK_SIZE as i64); + + unsafe { + let mut queue = self.ring.submission().available(); + queue.push(read_e.build().user_data(1)) + .ok() + .expect("queue is full"); } } - // copy out of the bounce buffer + self.ring.submit_and_wait(count)?; + + let cqes = self.ring.completion().available().collect::>(); + + // FIXME: return proper errors + assert_eq!(cqes.len(), count); + for c in &cqes { + assert_eq!(c.result(), BLOCK_SIZE as i32); + } Ok(()) } } -*/ + +//------------------------------------------ diff --git a/src/thin/check.rs b/src/thin/check.rs index a46385a..e22af6f 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; -use crate::block_manager::{Block, IoEngine, SyncIoEngine, BLOCK_SIZE}; +use crate::block_manager::{Block, IoEngine, AsyncIoEngine, SyncIoEngine, BLOCK_SIZE}; use crate::checksum; use crate::thin::superblock::*; @@ -220,7 +220,8 @@ fn walk_node( } pub fn check(dev: &Path) -> Result<()> { - let mut engine = SyncIoEngine::new(dev)?; + //let mut engine = SyncIoEngine::new(dev)?; + let mut engine = AsyncIoEngine::new(dev, 256)?; let now = Instant::now(); let sb = read_superblock(&mut engine, SUPERBLOCK_LOCATION)?; @@ -232,8 +233,7 @@ pub fn check(dev: &Path) -> Result<()> { walk_node(&mut engine, &mut seen, MappingLevel::Top, &root)?; println!( - "read superblock, mapping root at {}, {} ms", - sb.mapping_root, + "read mapping tree in {} ms", now.elapsed().as_millis() ); From d5597d5d368a3cf088298d0c4ee303842b7421e6 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 29 Jul 2020 11:12:03 +0100 Subject: [PATCH 07/52] [thin_check (rust)] Factor out tree walking code --- src/thin/check.rs | 215 +++++++++++++++++++++++++++------------------- 1 file changed, 129 insertions(+), 86 deletions(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index e22af6f..c9a4742 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,4 +1,6 @@ use anyhow::{anyhow, Result}; +use fixedbitset::FixedBitSet; +use futures::executor; use nom::{bytes::complete::*, number::complete::*, IResult}; use std::collections::HashSet; use std::error::Error; @@ -7,10 +9,12 @@ use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; -use crate::block_manager::{Block, IoEngine, AsyncIoEngine, SyncIoEngine, BLOCK_SIZE}; +use crate::block_manager::{AsyncIoEngine, Block, IoEngine, SyncIoEngine, BLOCK_SIZE}; use crate::checksum; use crate::thin::superblock::*; +//------------------------------------------ + trait ValueType { type Value; fn unpack(data: &[u8]) -> IResult<&[u8], Self::Value>; @@ -61,6 +65,27 @@ enum Node { }, } +impl Node { + fn get_header(&self) -> &NodeHeader { + match self { + Node::Internal { + header, + keys: _k, + values: _v, + } => &header, + Node::Leaf { + header, + keys: _k, + values: _v, + } => &header, + } + } + + fn is_leaf(&self) -> bool { + self.get_header().is_leaf + } +} + fn unpack_node_(data: &[u8]) -> IResult<&[u8], Node> { use nom::multi::count; @@ -101,6 +126,8 @@ fn unpack_node(data: &[u8]) -> Result> { } } +//------------------------------------------ + struct ValueU64; impl ValueType for ValueU64 { @@ -110,6 +137,76 @@ impl ValueType for ValueU64 { } } +//------------------------------------------ + +trait NodeVisitor { + fn visit<'a>(&mut self, w: &mut BTreeWalker<'a>, b: &Block, node: &Node) -> Result<()>; +} + +struct BTreeWalker<'a> { + engine: &'a mut dyn IoEngine, + seen: &'a mut FixedBitSet, +} + +impl<'a> BTreeWalker<'a> { + fn new(engine: &'a mut dyn IoEngine, seen: &'a mut FixedBitSet) -> BTreeWalker<'a> { + let nr_blocks = engine.get_nr_blocks() as usize; + assert_eq!(seen.len(), nr_blocks); + let r: BTreeWalker<'a> = BTreeWalker { engine, seen }; + r + } + + fn walk_nodes(&mut self, visitor: &mut NV, bs: &Vec) -> Result<()> + where + NV: NodeVisitor, + V: ValueType, + { + let mut blocks = Vec::new(); + for b in bs { + if !self.seen[*b as usize] { + blocks.push(Block::new(*b)); + } + } + + self.engine.read_many(&mut blocks)?; + + for b in blocks { + self.walk_node(visitor, &b)?; + } + + Ok(()) + } + + fn walk_node(&mut self, visitor: &mut NV, b: &Block) -> Result<()> + where + NV: NodeVisitor, + V: ValueType, + { + self.seen.insert(b.loc as usize); + + let bt = checksum::metadata_block_type(b.get_data()); + if bt != checksum::BT::NODE { + return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); + } + + let node = unpack_node::(&b.get_data())?; + visitor.visit(self, &b, &node)?; + + if let Node::Internal { + header: _h, + keys: _k, + values, + } = node + { + self.walk_nodes(visitor, &values)?; + } + + Ok(()) + } +} + +//------------------------------------------ + struct BlockTime { block: u64, time: u32, @@ -134,90 +231,37 @@ impl ValueType for ValueBlockTime { } } -#[derive(Copy, Clone)] -enum MappingLevel { - Top, - Bottom, +struct TopLevelVisitor {} + +impl NodeVisitor for TopLevelVisitor { + fn visit(&mut self, w: &mut BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf { + header: _h, + keys, + values, + } = node + { + let mut v = BottomLevelVisitor {}; + w.walk_nodes(&mut v, values)?; + } + Ok(()) + } } -fn walk_nodes( - engine: &mut E, - seen: &mut HashSet, - level: MappingLevel, - bs: &Vec, -) -> Result<()> { - let mut blocks = Vec::new(); - for b in bs { - if !seen.contains(b) { - blocks.push(Block::new(*b)); - } +struct BottomLevelVisitor {} + +impl NodeVisitor for BottomLevelVisitor { + fn visit( + &mut self, + _w: &mut BTreeWalker, + _b: &Block, + _node: &Node, + ) -> Result<()> { + Ok(()) } - - engine.read_many(&mut blocks)?; - - for b in blocks { - walk_node(engine, seen, level, &b); - } - - Ok(()) } -fn walk_node( - engine: &mut E, - seen: &mut HashSet, - level: MappingLevel, - b: &Block, -) -> Result<()> { - seen.insert(b.loc); - - let bt = checksum::metadata_block_type(b.get_data()); - if bt != checksum::BT::NODE { - return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); - } - - match level { - MappingLevel::Top => { - let node = unpack_node::(&b.get_data())?; - match node { - Node::Leaf { - header: header, - keys: _keys, - values, - } => { - walk_nodes(engine, seen, MappingLevel::Bottom, &values)?; - } - Node::Internal { - header: header, - keys: _keys, - values, - } => { - walk_nodes(engine, seen, MappingLevel::Top, &values)?; - } - } - } - MappingLevel::Bottom => { - let node = unpack_node::(&b.get_data())?; - match node { - Node::Leaf { - header: header, - keys: _keys, - values, - } => { - // FIXME: check in bounds - } - Node::Internal { - header: header, - keys: _keys, - values, - } => { - walk_nodes(engine, seen, MappingLevel::Bottom, &values)?; - } - } - } - } - - Ok(()) -} +//------------------------------------------ pub fn check(dev: &Path) -> Result<()> { //let mut engine = SyncIoEngine::new(dev)?; @@ -226,16 +270,15 @@ pub fn check(dev: &Path) -> Result<()> { let now = Instant::now(); let sb = read_superblock(&mut engine, SUPERBLOCK_LOCATION)?; eprintln!("{:?}", sb); - let mut seen = HashSet::new(); let mut root = Block::new(sb.mapping_root); engine.read(&mut root)?; - walk_node(&mut engine, &mut seen, MappingLevel::Top, &root)?; - println!( - "read mapping tree in {} ms", - now.elapsed().as_millis() - ); + let mut seen = FixedBitSet::with_capacity(engine.get_nr_blocks() as usize); + let mut w = BTreeWalker::new(&mut engine, &mut seen); + let mut visitor = TopLevelVisitor {}; + let result = w.walk_node(&mut visitor, &root)?; + println!("read mapping tree in {} ms", now.elapsed().as_millis()); Ok(()) } From 4f120911d271f4e6c387961b934f19e927fd0792 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 29 Jul 2020 16:38:52 +0100 Subject: [PATCH 08/52] [thin_check (rust)] First attempt at multithreading the tree walking. Still using a single io engine, so little benefit. --- Cargo.lock | 10 ++++++ Cargo.toml | 1 + src/block_manager.rs | 27 +++++++++++--- src/thin/check.rs | 84 ++++++++++++++++++++++++++++++++------------ 4 files changed, 96 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0478876..a41568a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -619,6 +619,7 @@ dependencies = [ "rand", "tempfile", "thiserror", + "threadpool", ] [[package]] @@ -650,6 +651,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "unicode-width" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index ddcf6bc..e4c1fd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ num-traits = "0.2" quick-xml = "0.18" rand = "0.7" tempfile = "3.1" +threadpool = "1.8" thiserror = "1.0" [dev-dependencies] diff --git a/src/block_manager.rs b/src/block_manager.rs index f813012..8e7dc5d 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -6,7 +6,7 @@ use std::fs::OpenOptions; use std::io; use std::io::{Read, Seek}; use std::os::unix::fs::OpenOptionsExt; -use std::os::unix::io::AsRawFd; +use std::os::unix::io::{RawFd, AsRawFd}; use std::path::Path; use std::sync::{Arc, Mutex}; use io_uring::opcode::{self, types}; @@ -45,6 +45,8 @@ impl Drop for Block { } } +unsafe impl Send for Block {} + //------------------------------------------ pub trait IoEngine { @@ -105,9 +107,11 @@ impl IoEngine for SyncIoEngine { //------------------------------------------ pub struct AsyncIoEngine { + queue_len: u32, ring: IoUring, nr_blocks: u64, - input: File, + fd: RawFd, + input: Arc, } impl AsyncIoEngine { @@ -118,14 +122,29 @@ impl AsyncIoEngine { .custom_flags(libc::O_DIRECT) .open(path)?; - Ok(AsyncIoEngine { + Ok (AsyncIoEngine { + queue_len, ring: IoUring::new(queue_len)?, nr_blocks: get_nr_blocks(path)?, - input, + fd: input.as_raw_fd(), + input: Arc::new(input), }) } } +impl Clone for AsyncIoEngine { + fn clone(&self) -> AsyncIoEngine { + eprintln!("in clone, queue_len = {}", self.queue_len); + AsyncIoEngine { + queue_len: self.queue_len, + ring: IoUring::new(self.queue_len).expect("couldn't create uring"), + nr_blocks: self.nr_blocks, + fd: self.fd, + input: self.input.clone(), + } + } +} + impl IoEngine for AsyncIoEngine { fn get_nr_blocks(&self) -> u64 { self.nr_blocks diff --git a/src/thin/check.rs b/src/thin/check.rs index c9a4742..c867b89 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -6,8 +6,9 @@ use std::collections::HashSet; use std::error::Error; use std::path::Path; use std::sync::{Arc, Mutex}; -use std::thread; +use std::thread::{self, spawn}; use std::time::{Duration, Instant}; +use threadpool::ThreadPool; use crate::block_manager::{AsyncIoEngine, Block, IoEngine, SyncIoEngine, BLOCK_SIZE}; use crate::checksum; @@ -140,19 +141,22 @@ impl ValueType for ValueU64 { //------------------------------------------ trait NodeVisitor { - fn visit<'a>(&mut self, w: &mut BTreeWalker<'a>, b: &Block, node: &Node) -> Result<()>; + fn visit<'a>(&mut self, w: &BTreeWalker, b: &Block, node: &Node) -> Result<()>; } -struct BTreeWalker<'a> { - engine: &'a mut dyn IoEngine, - seen: &'a mut FixedBitSet, +#[derive(Clone)] +struct BTreeWalker { + engine: Arc>, + seen: Arc>, } -impl<'a> BTreeWalker<'a> { - fn new(engine: &'a mut dyn IoEngine, seen: &'a mut FixedBitSet) -> BTreeWalker<'a> { +impl BTreeWalker { + fn new(engine: AsyncIoEngine) -> BTreeWalker { let nr_blocks = engine.get_nr_blocks() as usize; - assert_eq!(seen.len(), nr_blocks); - let r: BTreeWalker<'a> = BTreeWalker { engine, seen }; + let r: BTreeWalker = BTreeWalker { + engine: Arc::new(Mutex::new(engine)), + seen: Arc::new(Mutex::new(FixedBitSet::with_capacity(nr_blocks))), + }; r } @@ -162,13 +166,17 @@ impl<'a> BTreeWalker<'a> { V: ValueType, { let mut blocks = Vec::new(); + let seen = self.seen.lock().unwrap(); for b in bs { - if !self.seen[*b as usize] { + if !seen[*b as usize] { blocks.push(Block::new(*b)); } } + drop(seen); - self.engine.read_many(&mut blocks)?; + let mut engine = self.engine.lock().unwrap(); + engine.read_many(&mut blocks)?; + drop(engine); for b in blocks { self.walk_node(visitor, &b)?; @@ -182,7 +190,9 @@ impl<'a> BTreeWalker<'a> { NV: NodeVisitor, V: ValueType, { - self.seen.insert(b.loc as usize); + let mut seen = self.seen.lock().unwrap(); + seen.insert(b.loc as usize); + drop(seen); let bt = checksum::metadata_block_type(b.get_data()); if bt != checksum::BT::NODE { @@ -234,16 +244,49 @@ impl ValueType for ValueBlockTime { struct TopLevelVisitor {} impl NodeVisitor for TopLevelVisitor { - fn visit(&mut self, w: &mut BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + fn visit(&mut self, w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { if let Node::Leaf { header: _h, keys, values, } = node { - let mut v = BottomLevelVisitor {}; - w.walk_nodes(&mut v, values)?; + let mut blocks = Vec::new(); + let mut thin_ids = Vec::new(); + let seen = w.seen.lock().unwrap(); + for n in 0..keys.len() { + let b = values[n]; + if !seen[b as usize] { + thin_ids.push(keys[n]); + blocks.push(Block::new(b)); + } + } + drop(seen); + + let mut engine = w.engine.lock().unwrap(); + engine.read_many(&mut blocks)?; + drop(engine); + + // FIXME: with a thread pool we need to return errors another way. + let nr_workers = 16; + let pool = ThreadPool::new(nr_workers); + + let mut n = 0; + for b in blocks { + let thin_id = thin_ids[n]; + n += 1; + + let mut w = w.clone(); + pool.execute(move || { + let mut v = BottomLevelVisitor {}; + w.walk_node(&mut v, &b); + eprintln!("checked thin_dev {}", thin_id); + }); + } + + pool.join(); } + Ok(()) } } @@ -251,12 +294,7 @@ impl NodeVisitor for TopLevelVisitor { struct BottomLevelVisitor {} impl NodeVisitor for BottomLevelVisitor { - fn visit( - &mut self, - _w: &mut BTreeWalker, - _b: &Block, - _node: &Node, - ) -> Result<()> { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, _node: &Node) -> Result<()> { Ok(()) } } @@ -275,10 +313,12 @@ pub fn check(dev: &Path) -> Result<()> { engine.read(&mut root)?; let mut seen = FixedBitSet::with_capacity(engine.get_nr_blocks() as usize); - let mut w = BTreeWalker::new(&mut engine, &mut seen); + let mut w = BTreeWalker::new(engine); let mut visitor = TopLevelVisitor {}; let result = w.walk_node(&mut visitor, &root)?; println!("read mapping tree in {} ms", now.elapsed().as_millis()); Ok(()) } + +//------------------------------------------ From 8146fba9d21bf3d6a1f07af170b298ab96228cbf Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 30 Jul 2020 08:47:16 +0100 Subject: [PATCH 09/52] [thin_check (rust)] Move mutex inside IoEngines. Makes it easier to share an engine between different threads. --- src/block_manager.rs | 94 ++++++++++++++++++++++++++------------------ src/thin/check.rs | 2 +- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/src/block_manager.rs b/src/block_manager.rs index 8e7dc5d..ba8bf81 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -1,4 +1,6 @@ use anyhow::{anyhow, Result}; +use io_uring::opcode::{self, types}; +use io_uring::IoUring; use std::alloc::{alloc, dealloc, Layout}; use std::collections::HashMap; use std::fs::File; @@ -6,11 +8,9 @@ use std::fs::OpenOptions; use std::io; use std::io::{Read, Seek}; use std::os::unix::fs::OpenOptionsExt; -use std::os::unix::io::{RawFd, AsRawFd}; +use std::os::unix::io::{AsRawFd, RawFd}; use std::path::Path; use std::sync::{Arc, Mutex}; -use io_uring::opcode::{self, types}; -use io_uring::IoUring; //------------------------------------------ @@ -51,8 +51,8 @@ unsafe impl Send for Block {} pub trait IoEngine { fn get_nr_blocks(&self) -> u64; - fn read(&mut self, block: &mut Block) -> Result<()>; - fn read_many(&mut self, blocks: &mut Vec) -> Result<()>; + fn read(&self, block: &mut Block) -> Result<()>; + fn read_many(&self, blocks: &mut Vec) -> Result<()>; } fn get_nr_blocks(path: &Path) -> io::Result { @@ -62,6 +62,7 @@ fn get_nr_blocks(path: &Path) -> io::Result { //------------------------------------------ +/* pub struct SyncIoEngine { nr_blocks: u64, input: File, @@ -103,10 +104,10 @@ impl IoEngine for SyncIoEngine { Ok(()) } } - +*/ //------------------------------------------ -pub struct AsyncIoEngine { +pub struct AsyncIoEngine_ { queue_len: u32, ring: IoUring, nr_blocks: u64, @@ -114,6 +115,10 @@ pub struct AsyncIoEngine { input: Arc, } +pub struct AsyncIoEngine { + inner: Mutex, +} + impl AsyncIoEngine { pub fn new(path: &Path, queue_len: u32) -> Result { let input = OpenOptions::new() @@ -122,75 +127,86 @@ impl AsyncIoEngine { .custom_flags(libc::O_DIRECT) .open(path)?; - Ok (AsyncIoEngine { - queue_len, - ring: IoUring::new(queue_len)?, - nr_blocks: get_nr_blocks(path)?, - fd: input.as_raw_fd(), - input: Arc::new(input), + Ok(AsyncIoEngine { + inner: Mutex::new(AsyncIoEngine_ { + queue_len, + ring: IoUring::new(queue_len)?, + nr_blocks: get_nr_blocks(path)?, + fd: input.as_raw_fd(), + input: Arc::new(input), + }), }) } } impl Clone for AsyncIoEngine { fn clone(&self) -> AsyncIoEngine { - eprintln!("in clone, queue_len = {}", self.queue_len); - AsyncIoEngine { - queue_len: self.queue_len, - ring: IoUring::new(self.queue_len).expect("couldn't create uring"), - nr_blocks: self.nr_blocks, - fd: self.fd, - input: self.input.clone(), + let inner = self.inner.lock().unwrap(); + eprintln!("in clone, queue_len = {}", inner.queue_len); + AsyncIoEngine {inner: Mutex::new(AsyncIoEngine_ { + queue_len: inner.queue_len, + ring: IoUring::new(inner.queue_len).expect("couldn't create uring"), + nr_blocks: inner.nr_blocks, + fd: inner.fd, + input: inner.input.clone(), + }), } } } impl IoEngine for AsyncIoEngine { fn get_nr_blocks(&self) -> u64 { - self.nr_blocks + let inner = self.inner.lock().unwrap(); + inner.nr_blocks } - fn read(&mut self, b: &mut Block) -> Result<()> { - let fd = types::Target::Fd(self.input.as_raw_fd()); - let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32).offset(b.loc as i64 * BLOCK_SIZE as i64); + fn read(&self, b: &mut Block) -> Result<()> { + let mut inner = self.inner.lock().unwrap(); + let fd = types::Target::Fd(inner.input.as_raw_fd()); + let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32) + .offset(b.loc as i64 * BLOCK_SIZE as i64); unsafe { - let mut queue = self.ring.submission().available(); - queue.push(read_e.build().user_data(1)) + let mut queue = inner.ring.submission().available(); + queue + .push(read_e.build().user_data(1)) .ok() .expect("queue is full"); } - self.ring.submit_and_wait(1)?; + inner.ring.submit_and_wait(1)?; - let cqes = self.ring.completion().available().collect::>(); + let cqes = inner.ring.completion().available().collect::>(); - // FIXME: return proper errors - assert_eq!(cqes.len(), 1); - assert_eq!(cqes[0].user_data(), 1); - assert_eq!(cqes[0].result(), BLOCK_SIZE as i32); + // FIXME: return proper errors + assert_eq!(cqes.len(), 1); + assert_eq!(cqes[0].user_data(), 1); + assert_eq!(cqes[0].result(), BLOCK_SIZE as i32); Ok(()) } - fn read_many(&mut self, blocks: &mut Vec) -> Result<()> { + fn read_many(&self, blocks: &mut Vec) -> Result<()> { + let mut inner = self.inner.lock().unwrap(); let count = blocks.len(); - let fd = types::Target::Fd(self.input.as_raw_fd()); + let fd = types::Target::Fd(inner.input.as_raw_fd()); for b in blocks.into_iter() { - let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32).offset(b.loc as i64 * BLOCK_SIZE as i64); + let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32) + .offset(b.loc as i64 * BLOCK_SIZE as i64); unsafe { - let mut queue = self.ring.submission().available(); - queue.push(read_e.build().user_data(1)) + let mut queue = inner.ring.submission().available(); + queue + .push(read_e.build().user_data(1)) .ok() .expect("queue is full"); } } - self.ring.submit_and_wait(count)?; + inner.ring.submit_and_wait(count)?; - let cqes = self.ring.completion().available().collect::>(); + let cqes = inner.ring.completion().available().collect::>(); // FIXME: return proper errors assert_eq!(cqes.len(), count); diff --git a/src/thin/check.rs b/src/thin/check.rs index c867b89..471fad7 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -10,7 +10,7 @@ use std::thread::{self, spawn}; use std::time::{Duration, Instant}; use threadpool::ThreadPool; -use crate::block_manager::{AsyncIoEngine, Block, IoEngine, SyncIoEngine, BLOCK_SIZE}; +use crate::block_manager::{AsyncIoEngine, Block, IoEngine, BLOCK_SIZE}; use crate::checksum; use crate::thin::superblock::*; From f7623e62643a41d7de873dd1904d7f8fc517a711 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 30 Jul 2020 09:59:02 +0100 Subject: [PATCH 10/52] [thin_check (rust)] remove spurious mutex. --- src/thin/check.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index 471fad7..58c5ee3 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -146,7 +146,7 @@ trait NodeVisitor { #[derive(Clone)] struct BTreeWalker { - engine: Arc>, + engine: Arc, seen: Arc>, } @@ -154,7 +154,7 @@ impl BTreeWalker { fn new(engine: AsyncIoEngine) -> BTreeWalker { let nr_blocks = engine.get_nr_blocks() as usize; let r: BTreeWalker = BTreeWalker { - engine: Arc::new(Mutex::new(engine)), + engine: Arc::new(engine), seen: Arc::new(Mutex::new(FixedBitSet::with_capacity(nr_blocks))), }; r @@ -174,9 +174,7 @@ impl BTreeWalker { } drop(seen); - let mut engine = self.engine.lock().unwrap(); - engine.read_many(&mut blocks)?; - drop(engine); + self.engine.read_many(&mut blocks)?; for b in blocks { self.walk_node(visitor, &b)?; @@ -263,9 +261,7 @@ impl NodeVisitor for TopLevelVisitor { } drop(seen); - let mut engine = w.engine.lock().unwrap(); - engine.read_many(&mut blocks)?; - drop(engine); + w.engine.read_many(&mut blocks)?; // FIXME: with a thread pool we need to return errors another way. let nr_workers = 16; From de172147d36b00c1314f77771a6a94b5a0ccdada Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 30 Jul 2020 10:12:51 +0100 Subject: [PATCH 11/52] [thin_check (rust)] squash a lot of warnings --- src/block_manager.rs | 4 +--- src/checksum.rs | 1 + src/thin/check.rs | 39 ++++++++------------------------------- src/thin/superblock.rs | 6 +++--- 4 files changed, 13 insertions(+), 37 deletions(-) diff --git a/src/block_manager.rs b/src/block_manager.rs index ba8bf81..5abef18 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -1,12 +1,10 @@ -use anyhow::{anyhow, Result}; +use anyhow::Result; use io_uring::opcode::{self, types}; use io_uring::IoUring; use std::alloc::{alloc, dealloc, Layout}; -use std::collections::HashMap; use std::fs::File; use std::fs::OpenOptions; use std::io; -use std::io::{Read, Seek}; use std::os::unix::fs::OpenOptionsExt; use std::os::unix::io::{AsRawFd, RawFd}; use std::path::Path; diff --git a/src/checksum.rs b/src/checksum.rs index 25d7205..9cb3b89 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -4,6 +4,7 @@ use crc32c::crc32c; use std::io::Cursor; const BLOCK_SIZE: u64 = 4096; +#[allow(dead_code)] const MAGIC: u64 = 0xa537a0aa6309ef77; const SUPERBLOCK_CSUM_XOR: u32 = 160774; const BITMAP_CSUM_XOR: u32 = 240779; diff --git a/src/thin/check.rs b/src/thin/check.rs index 58c5ee3..98b343d 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,16 +1,12 @@ use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; -use futures::executor; -use nom::{bytes::complete::*, number::complete::*, IResult}; -use std::collections::HashSet; -use std::error::Error; +use nom::{number::complete::*, IResult}; use std::path::Path; use std::sync::{Arc, Mutex}; -use std::thread::{self, spawn}; -use std::time::{Duration, Instant}; +use std::time::Instant; use threadpool::ThreadPool; -use crate::block_manager::{AsyncIoEngine, Block, IoEngine, BLOCK_SIZE}; +use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; use crate::checksum; use crate::thin::superblock::*; @@ -21,6 +17,7 @@ trait ValueType { fn unpack(data: &[u8]) -> IResult<&[u8], Self::Value>; } +#[allow(dead_code)] struct NodeHeader { is_leaf: bool, block: u64, @@ -29,6 +26,7 @@ struct NodeHeader { value_size: u32, } +#[allow(dead_code)] const INTERNAL_NODE: u32 = 1; const LEAF_NODE: u32 = 2; @@ -66,27 +64,6 @@ enum Node { }, } -impl Node { - fn get_header(&self) -> &NodeHeader { - match self { - Node::Internal { - header, - keys: _k, - values: _v, - } => &header, - Node::Leaf { - header, - keys: _k, - values: _v, - } => &header, - } - } - - fn is_leaf(&self) -> bool { - self.get_header().is_leaf - } -} - fn unpack_node_(data: &[u8]) -> IResult<&[u8], Node> { use nom::multi::count; @@ -215,6 +192,7 @@ impl BTreeWalker { //------------------------------------------ +#[allow(dead_code)] struct BlockTime { block: u64, time: u32, @@ -275,7 +253,7 @@ impl NodeVisitor for TopLevelVisitor { let mut w = w.clone(); pool.execute(move || { let mut v = BottomLevelVisitor {}; - w.walk_node(&mut v, &b); + w.walk_node(&mut v, &b).expect("walk failed"); // FIXME: return error eprintln!("checked thin_dev {}", thin_id); }); } @@ -308,10 +286,9 @@ pub fn check(dev: &Path) -> Result<()> { let mut root = Block::new(sb.mapping_root); engine.read(&mut root)?; - let mut seen = FixedBitSet::with_capacity(engine.get_nr_blocks() as usize); let mut w = BTreeWalker::new(engine); let mut visitor = TopLevelVisitor {}; - let result = w.walk_node(&mut visitor, &root)?; + let _result = w.walk_node(&mut visitor, &root)?; println!("read mapping tree in {} ms", now.elapsed().as_millis()); Ok(()) diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index f7c3546..e52d7f2 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -1,11 +1,9 @@ use crate::block_manager::*; -use crate::block_manager::*; -use crate::checksum::*; use anyhow::{anyhow, Result}; use nom::{bytes::complete::*, number::complete::*, IResult}; pub const SUPERBLOCK_LOCATION: u64 = 0; -const UUID_SIZE: usize = 16; +//const UUID_SIZE: usize = 16; const SPACE_MAP_ROOT_SIZE: usize = 128; #[derive(Debug)] @@ -23,6 +21,7 @@ pub struct Superblock { pub data_block_size: u32, } +/* pub enum CheckSeverity { Fatal, NonFatal, @@ -48,6 +47,7 @@ struct SuperblockError { severity: CheckSeverity, kind: ErrorType, } +*/ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { let (i, _csum) = le_u32(data)?; From 7243f95380d71b914db340bdbdf02e76029e2023 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 30 Jul 2020 10:17:36 +0100 Subject: [PATCH 12/52] [thin_check (rust)] Drop nr threads down to 4 We get very little benefit from threads atm. Need to improve IO handling first. --- src/thin/check.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index 98b343d..90d9703 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -242,7 +242,7 @@ impl NodeVisitor for TopLevelVisitor { w.engine.read_many(&mut blocks)?; // FIXME: with a thread pool we need to return errors another way. - let nr_workers = 16; + let nr_workers = 4; let pool = ThreadPool::new(nr_workers); let mut n = 0; From fcfcc60b89919afd7a1e852150b6be341ce40b26 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 31 Jul 2020 11:04:12 +0100 Subject: [PATCH 13/52] [functional-tests] Move thin_check functional tests to Rust. They'll be run as part of 'cargo test' now. --- Cargo.lock | 40 ++ Cargo.toml | 2 + functional-tests/thin-functional-tests.scm | 80 --- tests/common/mod.rs | 11 + tests/common/xml_generator.rs | 539 +++++++++++++++++++++ tests/thin_check.rs | 173 +++++++ tests/thin_shrink.rs | 529 +------------------- 7 files changed, 776 insertions(+), 598 deletions(-) create mode 100644 tests/common/mod.rs create mode 100644 tests/common/xml_generator.rs create mode 100644 tests/thin_check.rs diff --git a/Cargo.lock b/Cargo.lock index a41568a..9ec72e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,6 +107,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "duct" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90a9c3a25aafbd538c7d40a53f83c4487ee8216c12d1c8ef2c01eb2f6ea1553" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + [[package]] name = "env_logger" version = "0.7.1" @@ -260,6 +272,12 @@ dependencies = [ "libc", ] +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + [[package]] name = "lazy_static" version = "1.4.0" @@ -369,6 +387,16 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +[[package]] +name = "os_pipe" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb233f06c2307e1f5ce2ecad9f8121cffbbee2c95428f44ea85222e460d0d213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "pin-project" version = "0.4.23" @@ -543,6 +571,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "shared_child" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cebcf3a403e4deafaf34dc882c4a1b6a648b43e5670aa2e4bb985914eaeb2d2" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "slab" version = "0.4.2" @@ -603,10 +641,12 @@ dependencies = [ "byteorder", "clap", "crc32c", + "duct", "fixedbitset", "flate2", "futures", "io-uring", + "json", "libc", "nix", "nom", diff --git a/Cargo.toml b/Cargo.toml index e4c1fd1..dbb8470 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ anyhow = "1.0" byteorder = "1.3" clap = "2.33" crc32c = "0.4" +duct = "0.13" fixedbitset = "0.3" futures = "0.3" flate2 = "1.0" @@ -27,5 +28,6 @@ threadpool = "1.8" thiserror = "1.0" [dev-dependencies] +json = "0.12" quickcheck = "0.9" quickcheck_macros = "0.9" diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm index d5e5d00..5202d94 100644 --- a/functional-tests/thin-functional-tests.scm +++ b/functional-tests/thin-functional-tests.scm @@ -67,86 +67,6 @@ ;; to run. (define (register-thin-tests) #t) - ;;;----------------------------------------------------------- - ;;; thin_check scenarios - ;;;----------------------------------------------------------- - - (define-scenario (thin-check v) - "thin_check -V" - (run-ok-rcv (stdout _) (thin-check "-V") - (assert-equal tools-version stdout))) - - (define-scenario (thin-check version) - "thin_check --version" - (run-ok-rcv (stdout _) (thin-check "--version") - (assert-equal tools-version stdout))) - - (define-scenario (thin-check h) - "print help (-h)" - (run-ok-rcv (stdout _) (thin-check "-h") - (assert-equal thin-check-help stdout))) - - (define-scenario (thin-check help) - "print help (--help)" - (run-ok-rcv (stdout _) (thin-check "--help") - (assert-equal thin-check-help stdout))) - - (define-scenario (thin-check bad-option) - "Unrecognised option should cause failure" - (run-fail (thin-check "--hedgehogs-only"))) - - (define-scenario (thin-check superblock-only-valid) - "--super-block-only check passes on valid metadata" - (with-valid-metadata (md) - (run-ok (thin-check "--super-block-only" md)))) - - (define-scenario (thin-check superblock-only-invalid) - "--super-block-only check fails with corrupt metadata" - (with-corrupt-metadata (md) - (run-fail (thin-check "--super-block-only" md)))) - - (define-scenario (thin-check skip-mappings-valid) - "--skip-mappings check passes on valid metadata" - (with-valid-metadata (md) - (run-ok (thin-check "--skip-mappings" md)))) - - (define-scenario (thin-check ignore-non-fatal-errors) - "--ignore-non-fatal-errors check passes on valid metadata" - (with-valid-metadata (md) - (run-ok (thin-check "--ignore-non-fatal-errors" md)))) - - (define-scenario (thin-check quiet) - "--quiet should give no output" - (with-valid-metadata (md) - (run-ok-rcv (stdout stderr) (thin-check "--quiet" md) - (assert-eof stdout) - (assert-eof stderr)))) - - (define-scenario (thin-check clear-needs-check-flag) - "Accepts --clear-needs-check-flag" - (with-valid-metadata (md) - (run-ok (thin-check "--clear-needs-check-flag" md)))) - - (define-scenario (thin-check tiny-metadata) - "Prints helpful message in case tiny metadata given" - (with-temp-file-sized ((md "thin.bin" 1024)) - (run-fail-rcv (_ stderr) (thin-check md) - (assert-starts-with "Metadata device/file too small. Is this binary metadata?" stderr)))) - - (define-scenario (thin-check spot-accidental-xml-data) - "Prints helpful message if XML metadata given" - (with-thin-xml (xml) - (system (fmt #f "man bash >> " xml)) - (run-fail-rcv (_ stderr) (thin-check xml) - (assert-matches ".*This looks like XML. thin_check only checks the binary metadata format." stderr)))) - - (define-scenario (thin-check info-fields) - "Outputs info fields" - (with-valid-metadata (md) - (run-ok-rcv (stdout stderr) (thin-check md) - (assert-matches ".*TRANSACTION_ID=[0-9]+.*" stdout) - (assert-matches ".*METADATA_FREE_BLOCKS=[0-9]+.*" stdout)))) - ;;;----------------------------------------------------------- ;;; thin_restore scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..325d697 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,11 @@ +use std::path::{Path, PathBuf}; + +pub mod xml_generator; + +pub fn mk_path(dir: &Path, file: &str) -> PathBuf { + let mut p = PathBuf::new(); + p.push(dir); + p.push(PathBuf::from(file)); + p +} + diff --git a/tests/common/xml_generator.rs b/tests/common/xml_generator.rs new file mode 100644 index 0000000..eb388ab --- /dev/null +++ b/tests/common/xml_generator.rs @@ -0,0 +1,539 @@ +use anyhow::{anyhow, Result}; +use rand::prelude::*; +use std::collections::VecDeque; +use std::fs::OpenOptions; +use std::ops::Range; +use std::path::Path; +use thinp::thin::xml; + +//------------------------------------------ + +pub trait XmlGen { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>; +} + +pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> { + let xml_out = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(path)?; + let mut w = xml::XmlWriter::new(xml_out); + + g.generate_xml(&mut w) +} + +fn common_sb(nr_blocks: u64) -> xml::Superblock { + xml::Superblock { + uuid: "".to_string(), + time: 0, + transaction: 0, + flags: None, + version: None, + data_block_size: 32, + nr_data_blocks: nr_blocks, + metadata_snap: None, + } +} + +//------------------------------------------ + +pub struct EmptyPoolS {} + +impl XmlGen for EmptyPoolS { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + v.superblock_b(&common_sb(1024))?; + v.superblock_e()?; + Ok(()) + } +} + +//------------------------------------------ + +pub struct SingleThinS { + pub offset: u64, + pub len: u64, + pub old_nr_data_blocks: u64, + pub new_nr_data_blocks: u64, +} + +impl SingleThinS { + pub fn new(offset: u64, len: u64, old_nr_data_blocks: u64, new_nr_data_blocks: u64) -> Self { + SingleThinS { + offset, + len, + old_nr_data_blocks, + new_nr_data_blocks, + } + } +} + +impl XmlGen for SingleThinS { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + v.superblock_b(&common_sb(self.old_nr_data_blocks))?; + v.device_b(&xml::Device { + dev_id: 0, + mapped_blocks: self.len, + transaction: 0, + creation_time: 0, + snap_time: 0, + })?; + v.map(&xml::Map { + thin_begin: 0, + data_begin: self.offset, + time: 0, + len: self.len, + })?; + v.device_e()?; + v.superblock_e()?; + Ok(()) + } +} + +//------------------------------------------ + +pub struct FragmentedS { + pub nr_thins: u32, + pub thin_size: u64, + pub old_nr_data_blocks: u64, + pub new_nr_data_blocks: u64, +} + +impl FragmentedS { + pub fn new(nr_thins: u32, thin_size: u64) -> Self { + let old_size = (nr_thins as u64) * thin_size; + FragmentedS { + nr_thins, + thin_size, + old_nr_data_blocks: (nr_thins as u64) * thin_size, + new_nr_data_blocks: old_size * 3 / 4, + } + } +} + +#[derive(Clone)] +struct ThinRun { + thin_id: u32, + thin_begin: u64, + len: u64, +} + +#[derive(Clone, Debug, Copy)] +struct MappedRun { + thin_id: u32, + thin_begin: u64, + data_begin: u64, + len: u64, +} + +fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range) -> Vec { + let mut runs = Vec::new(); + let mut b = 0u64; + while b < total_len { + let len = u64::min( + total_len - b, + thread_rng().gen_range(run_len.start, run_len.end), + ); + runs.push(ThinRun { + thin_id: thin_id, + thin_begin: b, + len, + }); + b += len; + } + runs +} + +impl XmlGen for FragmentedS { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + // Allocate each thin fully, in runs between 1 and 16. + let mut runs = Vec::new(); + for thin in 0..self.nr_thins { + runs.append(&mut mk_runs(thin, self.thin_size, 1..17)); + } + + // Shuffle + runs.shuffle(&mut rand::thread_rng()); + + // map across the data + let mut maps = Vec::new(); + let mut b = 0; + for r in &runs { + maps.push(MappedRun { + thin_id: r.thin_id, + thin_begin: r.thin_begin, + data_begin: b, + len: r.len, + }); + b += r.len; + } + + // drop half the mappings, which leaves us free runs + let mut dropped = Vec::new(); + for i in 0..maps.len() { + if i % 2 == 0 { + dropped.push(maps[i].clone()); + } + } + + // Unshuffle. This isn't strictly necc. but makes the xml + // more readable. + use std::cmp::Ordering; + maps.sort_by(|&l, &r| match l.thin_id.cmp(&r.thin_id) { + Ordering::Equal => l.thin_begin.cmp(&r.thin_begin), + o => o, + }); + + // write the xml + v.superblock_b(&common_sb(self.old_nr_data_blocks))?; + for thin in 0..self.nr_thins { + v.device_b(&xml::Device { + dev_id: thin, + mapped_blocks: self.thin_size, + transaction: 0, + creation_time: 0, + snap_time: 0, + })?; + + for m in &dropped { + if m.thin_id != thin { + continue; + } + + v.map(&xml::Map { + thin_begin: m.thin_begin, + data_begin: m.data_begin, + time: 0, + len: m.len, + })?; + } + + v.device_e()?; + } + v.superblock_e()?; + Ok(()) + } +} + +//------------------------------------------ + +struct Allocator { + runs: VecDeque>, +} + +impl Allocator { + fn new_shuffled(total_len: u64, run_len: Range) -> Allocator { + let mut runs = Vec::new(); + + let mut b = 0u64; + while b < total_len { + let len = u64::min( + total_len - b, + thread_rng().gen_range(run_len.start, run_len.end), + ); + runs.push(b..(b + len)); + b += len; + } + + runs.shuffle(&mut thread_rng()); + let runs: VecDeque> = runs.iter().map(|r| r.clone()).collect(); + Allocator { runs } + } + + #[allow(dead_code)] + fn is_empty(&self) -> bool { + self.runs.is_empty() + } + + fn alloc(&mut self, len: u64) -> Result>> { + let mut len = len; + let mut runs = Vec::new(); + + while len > 0 { + let r = self.runs.pop_front(); + + if r.is_none() { + return Err(anyhow!("could not allocate; out of space")); + } + + let r = r.unwrap(); + let rlen = r.end - r.start; + if len < rlen { + runs.push(r.start..(r.start + len)); + + // We need to push something back. + self.runs.push_front((r.start + len)..r.end); + len = 0; + } else { + runs.push(r.start..r.end); + len -= rlen; + } + } + + Ok(runs) + } +} + +// Having explicitly unmapped regions makes it easier to +// apply snapshots. +#[derive(Clone)] +enum Run { + Mapped { data_begin: u64, len: u64 }, + UnMapped { len: u64 }, +} + +impl Run { + #[allow(dead_code)] + fn len(&self) -> u64 { + match self { + Run::Mapped { + data_begin: _data_begin, + len, + } => *len, + Run::UnMapped { len } => *len, + } + } + + fn split(&self, n: u64) -> (Option, Option) { + if n == 0 { + return (None, Some(self.clone())); + } else { + if self.len() <= n { + return (Some(self.clone()), None); + } else { + match self { + Run::Mapped { data_begin, len } => ( + Some(Run::Mapped { + data_begin: *data_begin, + len: n, + }), + Some(Run::Mapped { + data_begin: data_begin + n, + len: len - n, + }), + ), + Run::UnMapped { len } => ( + Some(Run::UnMapped { len: n }), + Some(Run::UnMapped { len: len - n }), + ), + } + } + } + } +} + +#[derive(Clone)] +struct ThinDev { + thin_id: u32, + dev_size: u64, + runs: Vec, +} + +impl ThinDev { + fn emit(&self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + v.device_b(&xml::Device { + dev_id: self.thin_id, + mapped_blocks: self.dev_size, + transaction: 0, + creation_time: 0, + snap_time: 0, + })?; + + let mut b = 0; + for r in &self.runs { + match r { + Run::Mapped { data_begin, len } => { + v.map(&xml::Map { + thin_begin: b, + data_begin: *data_begin, + time: 0, + len: *len, + })?; + b += len; + } + Run::UnMapped { len } => { + b += len; + } + } + } + + v.device_e()?; + Ok(()) + } +} + +#[derive(Clone)] +enum SnapRunType { + Same, + Diff, + Hole, +} + +#[derive(Clone)] +struct SnapRun(SnapRunType, u64); + +fn mk_origin(thin_id: u32, total_len: u64, allocator: &mut Allocator) -> Result { + let mut runs = Vec::new(); + let mut b = 0; + while b < total_len { + let len = u64::min(thread_rng().gen_range(16, 64), total_len - b); + match thread_rng().gen_range(0, 2) { + 0 => { + for data in allocator.alloc(len)? { + assert!(data.end >= data.start); + runs.push(Run::Mapped { + data_begin: data.start, + len: data.end - data.start, + }); + } + } + 1 => { + runs.push(Run::UnMapped { len }); + } + _ => { + return Err(anyhow!("bad value returned from rng")); + } + }; + + b += len; + } + + Ok(ThinDev { + thin_id, + dev_size: total_len, + runs, + }) +} + +fn mk_snap_mapping( + total_len: u64, + run_len: Range, + same_percent: usize, + diff_percent: usize, +) -> Vec { + let mut runs = Vec::new(); + + let mut b = 0u64; + while b < total_len { + let len = u64::min( + total_len - b, + thread_rng().gen_range(run_len.start, run_len.end), + ); + + let n = thread_rng().gen_range(0, 100); + + if n < same_percent { + runs.push(SnapRun(SnapRunType::Same, len)); + } else if n < diff_percent { + runs.push(SnapRun(SnapRunType::Diff, len)); + } else { + runs.push(SnapRun(SnapRunType::Hole, len)); + } + + b += len; + } + + runs +} + +fn split_runs(mut n: u64, runs: &Vec) -> (Vec, Vec) { + let mut before = Vec::new(); + let mut after = Vec::new(); + + for r in runs { + match r.split(n) { + (Some(lhs), None) => { + before.push(lhs); + } + (Some(lhs), Some(rhs)) => { + before.push(lhs); + after.push(rhs); + } + (None, Some(rhs)) => { + after.push(rhs); + } + (None, None) => {} + } + n -= r.len(); + } + + (before, after) +} + +fn apply_snap_runs( + origin: &Vec, + snap: &Vec, + allocator: &mut Allocator, +) -> Result> { + let mut origin = origin.clone(); + let mut runs = Vec::new(); + + for SnapRun(st, slen) in snap { + let (os, rest) = split_runs(*slen, &origin); + match st { + SnapRunType::Same => { + for o in os { + runs.push(o); + } + } + SnapRunType::Diff => { + for data in allocator.alloc(*slen)? { + runs.push(Run::Mapped { + data_begin: data.start, + len: data.end - data.start, + }); + } + } + SnapRunType::Hole => { + runs.push(Run::UnMapped { len: *slen }); + } + } + + origin = rest; + } + + Ok(runs) +} + +// Snapshots share mappings, not neccessarily the entire ranges. +pub struct SnapS { + pub len: u64, + pub nr_snaps: u32, + + // Snaps will differ from the origin by this percentage + pub percent_change: usize, + pub old_nr_data_blocks: u64, + pub new_nr_data_blocks: u64, +} + +impl SnapS { + pub fn new(len: u64, nr_snaps: u32, percent_change: usize) -> Self { + let delta = len * (nr_snaps as u64) * (percent_change as u64) / 100; + let old_nr_data_blocks = len + 3 * delta; + let new_nr_data_blocks = len + 2 * delta; + + SnapS { + len, + nr_snaps, + percent_change, + old_nr_data_blocks, + new_nr_data_blocks, + } + } +} + +impl XmlGen for SnapS { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512); + let origin = mk_origin(0, self.len, &mut allocator)?; + + v.superblock_b(&common_sb(self.old_nr_data_blocks))?; + origin.emit(v)?; + v.superblock_e()?; + + Ok(()) + } +} + +//------------------------------------------ diff --git a/tests/thin_check.rs b/tests/thin_check.rs new file mode 100644 index 0000000..385050d --- /dev/null +++ b/tests/thin_check.rs @@ -0,0 +1,173 @@ +use anyhow::Result; +use duct::{cmd, Expression}; +use std::path::{Path, PathBuf}; +use std::str::from_utf8; +use tempfile::{tempdir, TempDir}; +use thinp::file_utils; +use thinp::version::TOOLS_VERSION; + +mod common; + +use common::mk_path; +use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; + +//------------------------------------------ + +macro_rules! thin_check { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_check", args).stdout_capture().stderr_capture() + } + }; +} + +// Returns stderr, a non zero status must be returned +fn run_fail(command: Expression) -> Result { + let output = command.stderr_capture().unchecked().run()?; + assert!(!output.status.success()); + Ok(from_utf8(&output.stderr[0..]).unwrap().to_string()) +} + +fn mk_valid_md(dir: &TempDir) -> Result { + let xml = mk_path(dir.path(), "meta.xml"); + let md = mk_path(dir.path(), "meta.bin"); + + let mut gen = SingleThinS::new(0, 1024, 2048, 2048); + write_xml(&xml, &mut gen)?; + + let _file = file_utils::create_sized_file(&md, 4096 * 4096); + cmd!("bin/thin_restore", "-i", xml, "-o", &md).run()?; + Ok(md) +} + +fn mk_corrupt_md(dir: &TempDir) -> Result { + let md = mk_path(dir.path(), "meta.bin"); + let _file = file_utils::create_sized_file(&md, 4096 * 4096); + Ok(md) +} + +fn accepts_flag(flag: &str) -> Result<()> { + let dir = tempdir()?; + let md = mk_valid_md(&dir)?; + thin_check!(flag, &md).run()?; + Ok(()) +} + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = thin_check!("-V").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = thin_check!("--version").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +const USAGE: &'static str = "Usage: thin_check [options] {device|file}\nOptions:\n {-q|--quiet}\n {-h|--help}\n {-V|--version}\n {-m|--metadata-snap}\n {--override-mapping-root}\n {--clear-needs-check-flag}\n {--ignore-non-fatal-errors}\n {--skip-mappings}\n {--super-block-only}"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = thin_check!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = thin_check!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn rejects_bad_option() -> Result<()> { + let stderr = run_fail(thin_check!("--hedgehogs-only"))?; + assert!(stderr.contains("unrecognized option \'--hedgehogs-only\'")); + Ok(()) +} + +#[test] +fn accepts_superblock_only() -> Result<()> { + accepts_flag("--super-block-only") +} + +#[test] +fn accepts_skip_mappings() -> Result<()> { + accepts_flag("--skip-mappings") +} + +#[test] +fn accepts_ignore_non_fatal_errors() -> Result<()> { + accepts_flag("--ignore-non-fatal-errors") +} + +#[test] +fn accepts_clear_needs_check_flag() -> Result<()> { + accepts_flag("--clear-needs-check-flag") +} + +#[test] +fn accepts_quiet() -> Result<()> { + let dir = tempdir()?; + let md = mk_valid_md(&dir)?; + + let output = thin_check!("--quiet", &md).run()?; + assert_eq!(output.stdout.len(), 0); + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +#[test] +fn detects_corrupt_superblock_with_superblock_only() -> Result<()> { + let dir = tempdir()?; + let md = mk_corrupt_md(&dir)?; + let output = thin_check!("--super-block-only", &md).unchecked().run()?; + assert!(!output.status.success()); + Ok(()) +} + +#[test] +fn prints_help_message_for_tiny_metadata() -> Result<()> { + let dir = tempdir()?; + let md = mk_path(dir.path(), "meta.bin"); + let _file = file_utils::create_sized_file(&md, 1024); + let stderr = run_fail(thin_check!(&md))?; + assert!(stderr.contains("Metadata device/file too small. Is this binary metadata?")); + Ok(()) +} + +#[test] +fn spot_xml_data() -> Result<()> { + let dir = tempdir()?; + let xml = mk_path(dir.path(), "meta.xml"); + + let mut gen = FragmentedS::new(4, 10240); + write_xml(&xml, &mut gen)?; + + let stderr = run_fail(thin_check!(&xml))?; + eprintln!("{}", stderr); + assert!( + stderr.contains("This looks like XML. thin_check only checks the binary metadata format.") + ); + Ok(()) +} + +#[test] +fn prints_info_fields() -> Result<()> { + let dir = tempdir()?; + let md = mk_valid_md(&dir)?; + let stdout = thin_check!(&md).read()?; + assert!(stdout.contains("TRANSACTION_ID=")); + assert!(stdout.contains("METADATA_FREE_BLOCKS=")); + Ok(()) +} + +//------------------------------------------ diff --git a/tests/thin_shrink.rs b/tests/thin_shrink.rs index 0291340..5686525 100644 --- a/tests/thin_shrink.rs +++ b/tests/thin_shrink.rs @@ -1,16 +1,20 @@ use anyhow::{anyhow, Result}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use rand::prelude::*; -use std::collections::VecDeque; use std::fs::OpenOptions; use std::io::{Cursor, Read, Seek, SeekFrom, Write}; -use std::ops::Range; use std::path::{Path, PathBuf}; use tempfile::tempdir; use thinp::file_utils; use thinp::thin::xml::{self, Visit}; +mod common; +use common::mk_path; +use common::xml_generator::{ + write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen, +}; + //------------------------------------ #[derive(Debug)] @@ -251,25 +255,6 @@ impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> { //------------------------------------ -fn mk_path(dir: &Path, file: &str) -> PathBuf { - let mut p = PathBuf::new(); - p.push(dir); - p.push(PathBuf::from(file)); - p -} - -fn generate_xml(path: &Path, g: &mut dyn Scenario) -> Result<()> { - let xml_out = OpenOptions::new() - .read(false) - .write(true) - .create(true) - .truncate(true) - .open(path)?; - let mut w = xml::XmlWriter::new(xml_out); - - g.generate_xml(&mut w) -} - fn create_data_file(data_path: &Path, xml_path: &Path) -> Result<()> { let input = OpenOptions::new().read(true).write(false).open(xml_path)?; @@ -304,17 +289,19 @@ fn verify(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> { } trait Scenario { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>; fn get_new_nr_blocks(&self) -> u64; } -fn test_shrink(scenario: &mut dyn Scenario) -> Result<()> { +fn test_shrink(scenario: &mut S) -> Result<()> +where + S: Scenario + XmlGen, +{ let dir = tempdir()?; let xml_before = mk_path(dir.path(), "before.xml"); let xml_after = mk_path(dir.path(), "after.xml"); let data_path = mk_path(dir.path(), "metadata.bin"); - generate_xml(&xml_before, scenario)?; + write_xml(&xml_before, scenario)?; create_data_file(&data_path, &xml_before)?; let mut rng = rand::thread_rng(); @@ -332,28 +319,7 @@ fn test_shrink(scenario: &mut dyn Scenario) -> Result<()> { //------------------------------------ -fn common_sb(nr_blocks: u64) -> xml::Superblock { - xml::Superblock { - uuid: "".to_string(), - time: 0, - transaction: 0, - flags: None, - version: None, - data_block_size: 32, - nr_data_blocks: nr_blocks, - metadata_snap: None, - } -} - -struct EmptyPoolS {} - impl Scenario for EmptyPoolS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - v.superblock_b(&common_sb(1024))?; - v.superblock_e()?; - Ok(()) - } - fn get_new_nr_blocks(&self) -> u64 { 512 } @@ -367,45 +333,7 @@ fn shrink_empty_pool() -> Result<()> { //------------------------------------ -struct SingleThinS { - offset: u64, - len: u64, - old_nr_data_blocks: u64, - new_nr_data_blocks: u64, -} - -impl SingleThinS { - fn new(offset: u64, len: u64, old_nr_data_blocks: u64, new_nr_data_blocks: u64) -> Self { - SingleThinS { - offset, - len, - old_nr_data_blocks, - new_nr_data_blocks, - } - } -} - impl Scenario for SingleThinS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - v.superblock_b(&common_sb(self.old_nr_data_blocks))?; - v.device_b(&xml::Device { - dev_id: 0, - mapped_blocks: self.len, - transaction: 0, - creation_time: 0, - snap_time: 0, - })?; - v.map(&xml::Map { - thin_begin: 0, - data_begin: self.offset, - time: 0, - len: self.len, - })?; - v.device_e()?; - v.superblock_e()?; - Ok(()) - } - fn get_new_nr_blocks(&self) -> u64 { self.new_nr_data_blocks } @@ -452,128 +380,7 @@ fn shrink_insufficient_space() -> Result<()> { //------------------------------------ -struct FragmentedS { - nr_thins: u32, - thin_size: u64, - old_nr_data_blocks: u64, - new_nr_data_blocks: u64, -} - -impl FragmentedS { - fn new(nr_thins: u32, thin_size: u64) -> Self { - let old_size = (nr_thins as u64) * thin_size; - FragmentedS { - nr_thins, - thin_size, - old_nr_data_blocks: (nr_thins as u64) * thin_size, - new_nr_data_blocks: old_size * 3 / 4, - } - } -} - -#[derive(Clone)] -struct ThinRun { - thin_id: u32, - thin_begin: u64, - len: u64, -} - -#[derive(Clone, Debug, Copy)] -struct MappedRun { - thin_id: u32, - thin_begin: u64, - data_begin: u64, - len: u64, -} - -fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range) -> Vec { - let mut runs = Vec::new(); - let mut b = 0u64; - while b < total_len { - let len = u64::min( - total_len - b, - thread_rng().gen_range(run_len.start, run_len.end), - ); - runs.push(ThinRun { - thin_id: thin_id, - thin_begin: b, - len, - }); - b += len; - } - runs -} - impl Scenario for FragmentedS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - // Allocate each thin fully, in runs between 1 and 16. - let mut runs = Vec::new(); - for thin in 0..self.nr_thins { - runs.append(&mut mk_runs(thin, self.thin_size, 1..17)); - } - - // Shuffle - runs.shuffle(&mut rand::thread_rng()); - - // map across the data - let mut maps = Vec::new(); - let mut b = 0; - for r in &runs { - maps.push(MappedRun { - thin_id: r.thin_id, - thin_begin: r.thin_begin, - data_begin: b, - len: r.len, - }); - b += r.len; - } - - // drop half the mappings, which leaves us free runs - let mut dropped = Vec::new(); - for i in 0..maps.len() { - if i % 2 == 0 { - dropped.push(maps[i].clone()); - } - } - - // Unshuffle. This isn't strictly necc. but makes the xml - // more readable. - use std::cmp::Ordering; - maps.sort_by(|&l, &r| match l.thin_id.cmp(&r.thin_id) { - Ordering::Equal => l.thin_begin.cmp(&r.thin_begin), - o => o, - }); - - // write the xml - v.superblock_b(&common_sb(self.old_nr_data_blocks))?; - for thin in 0..self.nr_thins { - v.device_b(&xml::Device { - dev_id: thin, - mapped_blocks: self.thin_size, - transaction: 0, - creation_time: 0, - snap_time: 0, - })?; - - for m in &dropped { - if m.thin_id != thin { - continue; - } - - v.map(&xml::Map { - thin_begin: m.thin_begin, - data_begin: m.data_begin, - time: 0, - len: m.len, - })?; - } - - v.device_e()?; - } - v.superblock_e()?; - Ok(()) - } - fn get_new_nr_blocks(&self) -> u64 { self.new_nr_data_blocks } @@ -605,321 +412,7 @@ fn shrink_fragmented_thin_64() -> Result<()> { //------------------------------------ -struct Allocator { - runs: VecDeque>, -} - -impl Allocator { - fn new_shuffled(total_len: u64, run_len: Range) -> Allocator { - let mut runs = Vec::new(); - - let mut b = 0u64; - while b < total_len { - let len = u64::min( - total_len - b, - thread_rng().gen_range(run_len.start, run_len.end), - ); - runs.push(b..(b + len)); - b += len; - } - - runs.shuffle(&mut thread_rng()); - let runs: VecDeque> = runs.iter().map(|r| r.clone()).collect(); - Allocator { runs } - } - - fn is_empty(&self) -> bool { - self.runs.is_empty() - } - - fn alloc(&mut self, len: u64) -> Result>> { - let mut len = len; - let mut runs = Vec::new(); - - while len > 0 { - let r = self.runs.pop_front(); - - if r.is_none() { - return Err(anyhow!("could not allocate; out of space")); - } - - let mut r = r.unwrap(); - let rlen = r.end - r.start; - if len < rlen { - runs.push(r.start..(r.start + len)); - - // We need to push something back. - self.runs.push_front((r.start + len)..r.end); - len = 0; - } else { - runs.push(r.start..r.end); - len -= rlen; - } - } - - Ok(runs) - } -} - -// Having explicitly unmapped regions makes it easier to -// apply snapshots. -#[derive(Clone)] -enum Run { - Mapped { data_begin: u64, len: u64 }, - UnMapped { len: u64 }, -} - -impl Run { - fn len(&self) -> u64 { - match self { - Run::Mapped { - data_begin: _data_begin, - len, - } => *len, - Run::UnMapped { len } => *len, - } - } - - fn split(&self, n: u64) -> (Option, Option) { - if n == 0 { - return (None, Some(self.clone())); - } else { - if self.len() <= n { - return (Some(self.clone()), None); - } else { - match self { - Run::Mapped { data_begin, len } => ( - Some(Run::Mapped { - data_begin: *data_begin, - len: n, - }), - Some(Run::Mapped { - data_begin: data_begin + n, - len: len - n, - }), - ), - Run::UnMapped { len } => ( - Some(Run::UnMapped { len: n }), - Some(Run::UnMapped { len: len - n }), - ), - } - } - } - } -} - -#[derive(Clone)] -struct ThinDev { - thin_id: u32, - dev_size: u64, - runs: Vec, -} - -impl ThinDev { - fn emit(&self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - v.device_b(&xml::Device { - dev_id: self.thin_id, - mapped_blocks: self.dev_size, - transaction: 0, - creation_time: 0, - snap_time: 0, - })?; - - let mut b = 0; - for r in &self.runs { - match r { - Run::Mapped { data_begin, len } => { - v.map(&xml::Map { - thin_begin: b, - data_begin: *data_begin, - time: 0, - len: *len, - })?; - b += len; - } - Run::UnMapped { len } => { - b += len; - } - } - } - - v.device_e()?; - Ok(()) - } -} - -#[derive(Clone)] -enum SnapRunType { - Same, - Diff, - Hole, -} - -#[derive(Clone)] -struct SnapRun(SnapRunType, u64); - -fn mk_origin(thin_id: u32, total_len: u64, allocator: &mut Allocator) -> Result { - let mut runs = Vec::new(); - let mut b = 0; - while b < total_len { - let len = u64::min(thread_rng().gen_range(16, 64), total_len - b); - match thread_rng().gen_range(0, 2) { - 0 => { - for data in allocator.alloc(len)? { - assert!(data.end >= data.start); - runs.push(Run::Mapped { - data_begin: data.start, - len: data.end - data.start, - }); - } - } - 1 => { - runs.push(Run::UnMapped { len }); - } - _ => { - return Err(anyhow!("bad value returned from rng")); - } - }; - - b += len; - } - - Ok(ThinDev { - thin_id, - dev_size: total_len, - runs, - }) -} - -fn mk_snap_mapping( - total_len: u64, - run_len: Range, - same_percent: usize, - diff_percent: usize, -) -> Vec { - let mut runs = Vec::new(); - - let mut b = 0u64; - while b < total_len { - let len = u64::min( - total_len - b, - thread_rng().gen_range(run_len.start, run_len.end), - ); - - let n = thread_rng().gen_range(0, 100); - - if n < same_percent { - runs.push(SnapRun(SnapRunType::Same, len)); - } else if n < diff_percent { - runs.push(SnapRun(SnapRunType::Diff, len)); - } else { - runs.push(SnapRun(SnapRunType::Hole, len)); - } - - b += len; - } - - runs -} - -fn split_runs(mut n: u64, runs: &Vec) -> (Vec, Vec) { - let mut before = Vec::new(); - let mut after = Vec::new(); - - for r in runs { - match r.split(n) { - (Some(lhs), None) => { - before.push(lhs); - } - (Some(lhs), Some(rhs)) => { - before.push(lhs); - after.push(rhs); - } - (None, Some(rhs)) => { - after.push(rhs); - } - (None, None) => {} - } - n -= r.len(); - } - - (before, after) -} - -fn apply_snap_runs( - origin: &Vec, - snap: &Vec, - allocator: &mut Allocator, -) -> Result> { - let mut origin = origin.clone(); - let mut runs = Vec::new(); - - for SnapRun(st, slen) in snap { - let (os, rest) = split_runs(*slen, &origin); - match st { - SnapRunType::Same => { - for o in os { - runs.push(o); - } - } - SnapRunType::Diff => { - for data in allocator.alloc(*slen)? { - runs.push(Run::Mapped { - data_begin: data.start, - len: data.end - data.start, - }); - } - } - SnapRunType::Hole => { - runs.push(Run::UnMapped { len: *slen }); - } - } - - origin = rest; - } - - Ok(runs) -} - -// Snapshots share mappings, not neccessarily the entire ranges. -struct SnapS { - len: u64, - nr_snaps: u32, - - // Snaps will differ from the origin by this percentage - percent_change: usize, - old_nr_data_blocks: u64, - new_nr_data_blocks: u64, -} - -impl SnapS { - fn new(len: u64, nr_snaps: u32, percent_change: usize) -> Self { - let delta = len * (nr_snaps as u64) * (percent_change as u64) / 100; - let old_nr_data_blocks = len + 3 * delta; - let new_nr_data_blocks = len + 2 * delta; - - SnapS { - len, - nr_snaps, - percent_change, - old_nr_data_blocks, - new_nr_data_blocks, - } - } -} - impl Scenario for SnapS { - fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { - let mut allocator = Allocator::new_shuffled(self.old_nr_data_blocks, 64..512); - let origin = mk_origin(0, self.len, &mut allocator)?; - - v.superblock_b(&common_sb(self.old_nr_data_blocks))?; - origin.emit(v)?; - v.superblock_e()?; - - Ok(()) - } - fn get_new_nr_blocks(&self) -> u64 { self.new_nr_data_blocks } From 084a26bf8565a6f52b029c921b2cb7b89ca55aef Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 31 Jul 2020 12:12:40 +0100 Subject: [PATCH 14/52] [functional-tests] Recode thin_restore tests in Rust. Now part of 'cargo test' --- functional-tests/thin-functional-tests.scm | 98 -------------- tests/common/mod.rs | 98 ++++++++++++++ tests/thin_check.rs | 49 +------ tests/thin_restore.rs | 144 +++++++++++++++++++++ 4 files changed, 245 insertions(+), 144 deletions(-) create mode 100644 tests/thin_restore.rs diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm index 5202d94..4887ff7 100644 --- a/functional-tests/thin-functional-tests.scm +++ b/functional-tests/thin-functional-tests.scm @@ -67,104 +67,6 @@ ;; to run. (define (register-thin-tests) #t) - ;;;----------------------------------------------------------- - ;;; thin_restore scenarios - ;;;----------------------------------------------------------- - - (define-scenario (thin-restore print-version-v) - "print help (-V)" - (run-ok-rcv (stdout _) (thin-restore "-V") - (assert-equal tools-version stdout))) - - (define-scenario (thin-restore print-version-long) - "print help (--version)" - (run-ok-rcv (stdout _) (thin-restore "--version") - (assert-equal tools-version stdout))) - - (define-scenario (thin-restore h) - "print help (-h)" - (run-ok-rcv (stdout _) (thin-restore "-h") - (assert-equal thin-restore-help stdout))) - - (define-scenario (thin-restore help) - "print help (-h)" - (run-ok-rcv (stdout _) (thin-restore "--help") - (assert-equal thin-restore-help stdout))) - - (define-scenario (thin-restore no-input-file) - "forget to specify an input file" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-restore "-o" md) - (assert-starts-with "No input file provided." stderr)))) - - (define-scenario (thin-restore missing-input-file) - "the input file can't be found" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-restore "-i no-such-file -o" md) - (assert-superblock-all-zeroes md) - (assert-starts-with "Couldn't stat file" stderr)))) - - (define-scenario (thin-restore garbage-input-file) - "the input file is just zeroes" - (with-empty-metadata (md) - (with-temp-file-sized ((xml "thin.xml" 4096)) - (run-fail-rcv (_ stderr) (thin-restore "-i " xml "-o" md) - (assert-superblock-all-zeroes md))))) - - (define-scenario (thin-restore missing-output-file) - "the output file can't be found" - (with-thin-xml (xml) - (run-fail-rcv (_ stderr) (thin-restore "-i " xml) - (assert-starts-with "No output file provided." stderr)))) - - (define-scenario (thin-restore tiny-output-file) - "Fails if the output file is too small." - (with-temp-file-sized ((md "thin.bin" 4096)) - (with-thin-xml (xml) - (run-fail-rcv (_ stderr) (thin-restore "-i" xml "-o" md) - (assert-starts-with thin-restore-outfile-too-small-text stderr))))) - - (define-scenario (thin-restore q) - "thin_restore accepts -q" - (with-empty-metadata (md) - (with-thin-xml (xml) - (run-ok-rcv (stdout _) (thin-restore "-i" xml "-o" md "-q") - (assert-eof stdout))))) - - (define-scenario (thin-restore quiet) - "thin_restore accepts --quiet" - (with-empty-metadata (md) - (with-thin-xml (xml) - (run-ok-rcv (stdout _) (thin-restore "-i" xml "-o" md "--quiet") - (assert-eof stdout))))) - - (define-scenario (thin-restore override transaction-id) - "thin_restore obeys the --transaction-id override" - (with-empty-metadata (md) - (with-thin-xml (xml) - (run-ok-rcv (stdout stderr) (thin-restore "--transaction-id 2345" "-i" xml "-o" md) - (assert-eof stderr)) - (run-ok-rcv (stdout stderr) (thin-dump md) - (assert-matches ".*transaction=\"2345\"" stdout))))) - - (define-scenario (thin-restore override data-block-size) - "thin_restore obeys the --data-block-size override" - (with-empty-metadata (md) - (with-thin-xml (xml) - (run-ok-rcv (stdout stderr) (thin-restore "--data-block-size 8192" "-i" xml "-o" md) - (assert-eof stderr)) - (run-ok-rcv (stdout stderr) (thin-dump md) - (assert-matches ".*data_block_size=\"8192\"" stdout))))) - - (define-scenario (thin-restore override nr-data-blocks) - "thin_restore obeys the --nr-data-blocks override" - (with-empty-metadata (md) - (with-thin-xml (xml) - (run-ok-rcv (stdout stderr) (thin-restore "--nr-data-blocks 234500" "-i" xml "-o" md) - (assert-eof stderr)) - (run-ok-rcv (stdout stderr) (thin-dump md) - (assert-matches ".*nr_data_blocks=\"234500\"" stdout))))) - ;;;----------------------------------------------------------- ;;; thin_dump scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 325d697..b9fe1f8 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,6 +1,16 @@ +use anyhow::Result; +use duct::{cmd, Expression}; +use std::fs::OpenOptions; use std::path::{Path, PathBuf}; +use std::str::from_utf8; +use tempfile::{tempdir, TempDir}; +use thinp::file_utils; +use std::io::{Read}; pub mod xml_generator; +use crate::common::xml_generator::{write_xml, FragmentedS, SingleThinS}; + +//------------------------------------------ pub fn mk_path(dir: &Path, file: &str) -> PathBuf { let mut p = PathBuf::new(); @@ -9,3 +19,91 @@ pub fn mk_path(dir: &Path, file: &str) -> PathBuf { p } +// FIXME: write a macro to generate these commands +#[macro_export] +macro_rules! thin_check { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_check", args).stdout_capture().stderr_capture() + } + }; +} + +#[macro_export] +macro_rules! thin_restore { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_restore", args).stdout_capture().stderr_capture() + } + }; +} + +#[macro_export] +macro_rules! thin_dump { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_dump", args).stdout_capture().stderr_capture() + } + }; +} + + +// Returns stderr, a non zero status must be returned +pub fn run_fail(command: Expression) -> Result { + let output = command.stderr_capture().unchecked().run()?; + assert!(!output.status.success()); + Ok(from_utf8(&output.stderr[0..]).unwrap().to_string()) +} + +pub fn mk_valid_xml(dir: &TempDir) -> Result { + let xml = mk_path(dir.path(), "meta.xml"); + let mut gen = SingleThinS::new(0, 1024, 2048, 2048); + write_xml(&xml, &mut gen)?; + Ok(xml) +} + +pub fn mk_valid_md(dir: &TempDir) -> Result { + let xml = mk_path(dir.path(), "meta.xml"); + let md = mk_path(dir.path(), "meta.bin"); + + let mut gen = SingleThinS::new(0, 1024, 2048, 2048); + write_xml(&xml, &mut gen)?; + + let _file = file_utils::create_sized_file(&md, 4096 * 4096); + cmd!("bin/thin_restore", "-i", xml, "-o", &md).run()?; + Ok(md) +} + +pub fn mk_zeroed_md(dir: &TempDir) -> Result { + let md = mk_path(dir.path(), "meta.bin"); + let _file = file_utils::create_sized_file(&md, 4096 * 4096); + Ok(md) +} + +pub fn accepts_flag(flag: &str) -> Result<()> { + let dir = tempdir()?; + let md = mk_valid_md(&dir)?; + thin_check!(flag, &md).run()?; + Ok(()) +} + +pub fn superblock_all_zeroes(path: &PathBuf) -> Result { + let mut input = OpenOptions::new().read(true).write(false).open(path)?; + let mut buf = vec![0; 4096]; + input.read_exact(&mut buf[0..])?; + for b in buf { + if b != 0 { + return Ok(false); + } + } + + Ok(true) +} + +//------------------------------------------ diff --git a/tests/thin_check.rs b/tests/thin_check.rs index 385050d..cf59d0f 100644 --- a/tests/thin_check.rs +++ b/tests/thin_check.rs @@ -8,55 +8,11 @@ use thinp::version::TOOLS_VERSION; mod common; -use common::mk_path; +use common::*; use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; //------------------------------------------ -macro_rules! thin_check { - ( $( $arg: expr ),* ) => { - { - use std::ffi::OsString; - let args: &[OsString] = &[$( Into::::into($arg) ),*]; - duct::cmd("bin/thin_check", args).stdout_capture().stderr_capture() - } - }; -} - -// Returns stderr, a non zero status must be returned -fn run_fail(command: Expression) -> Result { - let output = command.stderr_capture().unchecked().run()?; - assert!(!output.status.success()); - Ok(from_utf8(&output.stderr[0..]).unwrap().to_string()) -} - -fn mk_valid_md(dir: &TempDir) -> Result { - let xml = mk_path(dir.path(), "meta.xml"); - let md = mk_path(dir.path(), "meta.bin"); - - let mut gen = SingleThinS::new(0, 1024, 2048, 2048); - write_xml(&xml, &mut gen)?; - - let _file = file_utils::create_sized_file(&md, 4096 * 4096); - cmd!("bin/thin_restore", "-i", xml, "-o", &md).run()?; - Ok(md) -} - -fn mk_corrupt_md(dir: &TempDir) -> Result { - let md = mk_path(dir.path(), "meta.bin"); - let _file = file_utils::create_sized_file(&md, 4096 * 4096); - Ok(md) -} - -fn accepts_flag(flag: &str) -> Result<()> { - let dir = tempdir()?; - let md = mk_valid_md(&dir)?; - thin_check!(flag, &md).run()?; - Ok(()) -} - -//------------------------------------------ - #[test] fn accepts_v() -> Result<()> { let stdout = thin_check!("-V").read()?; @@ -120,6 +76,7 @@ fn accepts_quiet() -> Result<()> { let md = mk_valid_md(&dir)?; let output = thin_check!("--quiet", &md).run()?; + assert!(output.status.success()); assert_eq!(output.stdout.len(), 0); assert_eq!(output.stderr.len(), 0); Ok(()) @@ -128,7 +85,7 @@ fn accepts_quiet() -> Result<()> { #[test] fn detects_corrupt_superblock_with_superblock_only() -> Result<()> { let dir = tempdir()?; - let md = mk_corrupt_md(&dir)?; + let md = mk_zeroed_md(&dir)?; let output = thin_check!("--super-block-only", &md).unchecked().run()?; assert!(!output.status.success()); Ok(()) diff --git a/tests/thin_restore.rs b/tests/thin_restore.rs new file mode 100644 index 0000000..96396bc --- /dev/null +++ b/tests/thin_restore.rs @@ -0,0 +1,144 @@ +use anyhow::Result; +use duct::{cmd, Expression}; +use std::path::{Path, PathBuf}; +use std::str::from_utf8; +use tempfile::{tempdir, TempDir}; +use thinp::file_utils; +use thinp::version::TOOLS_VERSION; + +mod common; + +use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; +use common::*; + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = thin_restore!("-V").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = thin_restore!("--version").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +const USAGE: &'static str = "Usage: thin_restore [options]\nOptions:\n {-h|--help}\n {-i|--input} \n {-o|--output} \n {--transaction-id} \n {--data-block-size} \n {--nr-data-blocks} \n {-q|--quiet}\n {-V|--version}"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = thin_restore!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = thin_restore!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn no_input_file() -> Result<()> { + let dir = tempdir()?; + let md = mk_zeroed_md(&dir)?; + let stderr = run_fail(thin_restore!("-o", &md))?; + assert!(stderr.contains("No input file provided.")); + Ok(()) +} + +#[test] +fn missing_input_file() -> Result<()> { + let dir = tempdir()?; + let md = mk_zeroed_md(&dir)?; + let stderr = run_fail(thin_restore!("-i", "no-such-file", "-o", &md))?; + assert!(superblock_all_zeroes(&md)?); + assert!(stderr.contains("Couldn't stat file")); + Ok(()) +} + +#[test] +fn garbage_input_file() -> Result<()> { + let dir = tempdir()?; + let xml = mk_zeroed_md(&dir)?; + let md = mk_zeroed_md(&dir)?; + let _stderr = run_fail(thin_restore!("-i", &xml, "-o", &md))?; + assert!(superblock_all_zeroes(&md)?); + Ok(()) +} + +#[test] +fn no_output_file() -> Result<()> { + let dir = tempdir()?; + let xml = mk_valid_xml(&dir)?; + let stderr = run_fail(thin_restore!("-i", &xml))?; + assert!(stderr.contains("No output file provided.")); + Ok(()) +} + +#[test] +fn tiny_output_file() -> Result<()> { + let dir = tempdir()?; + let xml = mk_valid_xml(&dir)?; + let md = mk_path(dir.path(), "meta.bin"); + let _file = file_utils::create_sized_file(&md, 4096); + let stderr = run_fail(thin_restore!("-i", &xml, "-o", &md))?; + eprintln!("{}", stderr); + assert!(stderr.contains("Output file too small")); + Ok(()) +} + +fn quiet_flag(flag: &str) -> Result<()> { + let dir = tempdir()?; + let xml = mk_valid_xml(&dir)?; + let md = mk_zeroed_md(&dir)?; + + let output = thin_restore!("-i", &xml, "-o", &md, flag).run()?; + + assert!(output.status.success()); + assert_eq!(output.stdout.len(), 0); + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +#[test] +fn accepts_q() -> Result<()> { + quiet_flag("-q") +} + +#[test] +fn accepts_quiet() -> Result<()> { + quiet_flag("--quiet") +} + +fn override_something(flag: &str, value: &str, pattern: &str) -> Result<()> { + let dir = tempdir()?; + let xml = mk_valid_xml(&dir)?; + let md = mk_zeroed_md(&dir)?; + + thin_restore!("-i", &xml, "-o", &md, flag, value).run()?; + + let output = thin_dump!(&md).run()?; + assert!(from_utf8(&output.stdout)?.contains(pattern)); + Ok(()) +} + +#[test] +fn override_transaction_id() -> Result<()> { + override_something("--transaction-id", "2345", "transaction=\"2345\"") +} + +#[test] +fn override_data_block_size() -> Result<()> { + override_something("--data-block-size", "8192", "data_block_size=\"8192\"") +} + +#[test] +fn override_nr_data_blocks() -> Result<()> { + override_something("--nr-data-blocks", "234500", "nr_data_blocks=\"234500\"") +} From 78db9a24fab14fb845bfe8e2175dc804e53aeb2f Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 31 Jul 2020 14:26:22 +0100 Subject: [PATCH 15/52] [functional-tests (rust)] factor out TestDir --- tests/common/mod.rs | 46 ++++++++++++++++++++++++++++--------------- tests/thin_check.rs | 20 +++++++++---------- tests/thin_restore.rs | 37 +++++++++++++++++----------------- tests/thin_shrink.rs | 12 +++++------ 4 files changed, 64 insertions(+), 51 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index b9fe1f8..52ee3e1 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -12,13 +12,6 @@ use crate::common::xml_generator::{write_xml, FragmentedS, SingleThinS}; //------------------------------------------ -pub fn mk_path(dir: &Path, file: &str) -> PathBuf { - let mut p = PathBuf::new(); - p.push(dir); - p.push(PathBuf::from(file)); - p -} - // FIXME: write a macro to generate these commands #[macro_export] macro_rules! thin_check { @@ -53,6 +46,27 @@ macro_rules! thin_dump { }; } +//------------------------------------------ + +pub struct TestDir { + dir: TempDir, + file_count: usize, +} + +impl TestDir { + pub fn new() -> Result { + let dir = tempdir()?; + Ok(TestDir {dir, file_count: 0}) + } + + pub fn mk_path(&mut self, file: &str) -> PathBuf { + let mut p = PathBuf::new(); + p.push(&self.dir); + p.push(PathBuf::from(format!("{:02}_{}", self.file_count, file))); + self.file_count += 1; + p + } +} // Returns stderr, a non zero status must be returned pub fn run_fail(command: Expression) -> Result { @@ -61,16 +75,16 @@ pub fn run_fail(command: Expression) -> Result { Ok(from_utf8(&output.stderr[0..]).unwrap().to_string()) } -pub fn mk_valid_xml(dir: &TempDir) -> Result { - let xml = mk_path(dir.path(), "meta.xml"); +pub fn mk_valid_xml(td: &mut TestDir) -> Result { + let xml = td.mk_path("meta.xml"); let mut gen = SingleThinS::new(0, 1024, 2048, 2048); write_xml(&xml, &mut gen)?; Ok(xml) } -pub fn mk_valid_md(dir: &TempDir) -> Result { - let xml = mk_path(dir.path(), "meta.xml"); - let md = mk_path(dir.path(), "meta.bin"); +pub fn mk_valid_md(td: &mut TestDir) -> Result { + let xml = td.mk_path("meta.xml"); + let md = td.mk_path("meta.bin"); let mut gen = SingleThinS::new(0, 1024, 2048, 2048); write_xml(&xml, &mut gen)?; @@ -80,15 +94,15 @@ pub fn mk_valid_md(dir: &TempDir) -> Result { Ok(md) } -pub fn mk_zeroed_md(dir: &TempDir) -> Result { - let md = mk_path(dir.path(), "meta.bin"); +pub fn mk_zeroed_md(td: &mut TestDir) -> Result { + let md = td.mk_path("meta.bin"); let _file = file_utils::create_sized_file(&md, 4096 * 4096); Ok(md) } pub fn accepts_flag(flag: &str) -> Result<()> { - let dir = tempdir()?; - let md = mk_valid_md(&dir)?; + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; thin_check!(flag, &md).run()?; Ok(()) } diff --git a/tests/thin_check.rs b/tests/thin_check.rs index cf59d0f..42caba6 100644 --- a/tests/thin_check.rs +++ b/tests/thin_check.rs @@ -72,8 +72,8 @@ fn accepts_clear_needs_check_flag() -> Result<()> { #[test] fn accepts_quiet() -> Result<()> { - let dir = tempdir()?; - let md = mk_valid_md(&dir)?; + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; let output = thin_check!("--quiet", &md).run()?; assert!(output.status.success()); @@ -84,8 +84,8 @@ fn accepts_quiet() -> Result<()> { #[test] fn detects_corrupt_superblock_with_superblock_only() -> Result<()> { - let dir = tempdir()?; - let md = mk_zeroed_md(&dir)?; + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; let output = thin_check!("--super-block-only", &md).unchecked().run()?; assert!(!output.status.success()); Ok(()) @@ -93,8 +93,8 @@ fn detects_corrupt_superblock_with_superblock_only() -> Result<()> { #[test] fn prints_help_message_for_tiny_metadata() -> Result<()> { - let dir = tempdir()?; - let md = mk_path(dir.path(), "meta.bin"); + let mut td = TestDir::new()?; + let md = td.mk_path("meta.bin"); let _file = file_utils::create_sized_file(&md, 1024); let stderr = run_fail(thin_check!(&md))?; assert!(stderr.contains("Metadata device/file too small. Is this binary metadata?")); @@ -103,8 +103,8 @@ fn prints_help_message_for_tiny_metadata() -> Result<()> { #[test] fn spot_xml_data() -> Result<()> { - let dir = tempdir()?; - let xml = mk_path(dir.path(), "meta.xml"); + let mut td = TestDir::new()?; + let xml = td.mk_path("meta.xml"); let mut gen = FragmentedS::new(4, 10240); write_xml(&xml, &mut gen)?; @@ -119,8 +119,8 @@ fn spot_xml_data() -> Result<()> { #[test] fn prints_info_fields() -> Result<()> { - let dir = tempdir()?; - let md = mk_valid_md(&dir)?; + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; let stdout = thin_check!(&md).read()?; assert!(stdout.contains("TRANSACTION_ID=")); assert!(stdout.contains("METADATA_FREE_BLOCKS=")); diff --git a/tests/thin_restore.rs b/tests/thin_restore.rs index 96396bc..e8861bc 100644 --- a/tests/thin_restore.rs +++ b/tests/thin_restore.rs @@ -45,8 +45,8 @@ fn accepts_help() -> Result<()> { #[test] fn no_input_file() -> Result<()> { - let dir = tempdir()?; - let md = mk_zeroed_md(&dir)?; + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; let stderr = run_fail(thin_restore!("-o", &md))?; assert!(stderr.contains("No input file provided.")); Ok(()) @@ -54,8 +54,8 @@ fn no_input_file() -> Result<()> { #[test] fn missing_input_file() -> Result<()> { - let dir = tempdir()?; - let md = mk_zeroed_md(&dir)?; + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; let stderr = run_fail(thin_restore!("-i", "no-such-file", "-o", &md))?; assert!(superblock_all_zeroes(&md)?); assert!(stderr.contains("Couldn't stat file")); @@ -64,9 +64,9 @@ fn missing_input_file() -> Result<()> { #[test] fn garbage_input_file() -> Result<()> { - let dir = tempdir()?; - let xml = mk_zeroed_md(&dir)?; - let md = mk_zeroed_md(&dir)?; + let mut td = TestDir::new()?; + let xml = mk_zeroed_md(&mut td)?; + let md = mk_zeroed_md(&mut td)?; let _stderr = run_fail(thin_restore!("-i", &xml, "-o", &md))?; assert!(superblock_all_zeroes(&md)?); Ok(()) @@ -74,8 +74,8 @@ fn garbage_input_file() -> Result<()> { #[test] fn no_output_file() -> Result<()> { - let dir = tempdir()?; - let xml = mk_valid_xml(&dir)?; + let mut td = TestDir::new()?; + let xml = mk_valid_xml(&mut td)?; let stderr = run_fail(thin_restore!("-i", &xml))?; assert!(stderr.contains("No output file provided.")); Ok(()) @@ -83,20 +83,19 @@ fn no_output_file() -> Result<()> { #[test] fn tiny_output_file() -> Result<()> { - let dir = tempdir()?; - let xml = mk_valid_xml(&dir)?; - let md = mk_path(dir.path(), "meta.bin"); + let mut td = TestDir::new()?; + let xml = mk_valid_xml(&mut td)?; + let md = td.mk_path("meta.bin"); let _file = file_utils::create_sized_file(&md, 4096); let stderr = run_fail(thin_restore!("-i", &xml, "-o", &md))?; - eprintln!("{}", stderr); assert!(stderr.contains("Output file too small")); Ok(()) } fn quiet_flag(flag: &str) -> Result<()> { - let dir = tempdir()?; - let xml = mk_valid_xml(&dir)?; - let md = mk_zeroed_md(&dir)?; + let mut td = TestDir::new()?; + let xml = mk_valid_xml(&mut td)?; + let md = mk_zeroed_md(&mut td)?; let output = thin_restore!("-i", &xml, "-o", &md, flag).run()?; @@ -117,9 +116,9 @@ fn accepts_quiet() -> Result<()> { } fn override_something(flag: &str, value: &str, pattern: &str) -> Result<()> { - let dir = tempdir()?; - let xml = mk_valid_xml(&dir)?; - let md = mk_zeroed_md(&dir)?; + let mut td = TestDir::new()?; + let xml = mk_valid_xml(&mut td)?; + let md = mk_zeroed_md(&mut td)?; thin_restore!("-i", &xml, "-o", &md, flag, value).run()?; diff --git a/tests/thin_shrink.rs b/tests/thin_shrink.rs index 5686525..8efae51 100644 --- a/tests/thin_shrink.rs +++ b/tests/thin_shrink.rs @@ -10,9 +10,9 @@ use thinp::file_utils; use thinp::thin::xml::{self, Visit}; mod common; -use common::mk_path; +use common::*; use common::xml_generator::{ - write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen, + write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen }; //------------------------------------ @@ -296,10 +296,10 @@ fn test_shrink(scenario: &mut S) -> Result<()> where S: Scenario + XmlGen, { - let dir = tempdir()?; - let xml_before = mk_path(dir.path(), "before.xml"); - let xml_after = mk_path(dir.path(), "after.xml"); - let data_path = mk_path(dir.path(), "metadata.bin"); + let mut td = TestDir::new()?; + let xml_before = td.mk_path("before.xml"); + let xml_after = td.mk_path("after.xml"); + let data_path = td.mk_path("metadata.bin"); write_xml(&xml_before, scenario)?; create_data_file(&data_path, &xml_before)?; From baf1fe325f73f55c1d39c85f6bf49506a3566e4f Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 31 Jul 2020 16:31:10 +0100 Subject: [PATCH 16/52] [functional-tests] Move thin_dump tests to Rust. cargo test --- functional-tests/thin-functional-tests.scm | 72 ----------- tests/common/mod.rs | 18 ++- tests/thin_check.rs | 4 - tests/thin_dump.rs | 136 +++++++++++++++++++++ tests/thin_restore.rs | 3 - tests/thin_shrink.rs | 3 +- 6 files changed, 150 insertions(+), 86 deletions(-) create mode 100644 tests/thin_dump.rs diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm index 4887ff7..78ab791 100644 --- a/functional-tests/thin-functional-tests.scm +++ b/functional-tests/thin-functional-tests.scm @@ -67,78 +67,6 @@ ;; to run. (define (register-thin-tests) #t) - ;;;----------------------------------------------------------- - ;;; thin_dump scenarios - ;;;----------------------------------------------------------- - - (define-scenario (thin-dump small-input-file) - "Fails with small input file" - (with-temp-file-sized ((md "thin.bin" 512)) - (run-fail (thin-dump md)))) - - (define-scenario (thin-dump restore-is-noop) - "thin_dump followed by thin_restore is a noop." - (with-valid-metadata (md) - (run-ok-rcv (d1-stdout _) (thin-dump md) - (with-temp-file-containing ((xml "thin.xml" d1-stdout)) - (run-ok (thin-restore "-i" xml "-o" md)) - (run-ok-rcv (d2-stdout _) (thin-dump md) - (assert-equal d1-stdout d2-stdout)))))) - - (define-scenario (thin-dump no-stderr) - "thin_dump of clean data does not output error messages to stderr" - (with-valid-metadata (md) - (run-ok-rcv (stdout stderr) (thin-dump md) - (assert-eof stderr)))) - - (define-scenario (thin-dump override transaction-id) - "thin_dump obeys the --transaction-id override" - (with-valid-metadata (md) - (run-ok-rcv (stdout stderr) (thin-dump "--transaction-id 2345" md) - (assert-eof stderr) - (assert-matches ".*transaction=\"2345\"" stdout)))) - - (define-scenario (thin-dump override data-block-size) - "thin_dump obeys the --data-block-size override" - (with-valid-metadata (md) - (run-ok-rcv (stdout stderr) (thin-dump "--data-block-size 8192" md) - (assert-eof stderr) - (assert-matches ".*data_block_size=\"8192\"" stdout)))) - - (define-scenario (thin-dump override nr-data-blocks) - "thin_dump obeys the --nr-data-blocks override" - (with-valid-metadata (md) - (run-ok-rcv (stdout stderr) (thin-dump "--nr-data-blocks 234500" md) - (assert-eof stderr) - (assert-matches ".*nr_data_blocks=\"234500\"" stdout)))) - - (define-scenario (thin-dump repair-superblock succeeds) - "thin_dump can restore a missing superblock" - (with-valid-metadata (md) - (run-ok-rcv (expected-xml stderr) (thin-dump "--transaction-id=5" "--data-block-size=128" "--nr-data-blocks=4096000" md) - (damage-superblock md) - (run-ok-rcv (repaired-xml stderr) (thin-dump "--repair" "--transaction-id=5" "--data-block-size=128" "--nr-data-blocks=4096000" md) - (assert-eof stderr) - (assert-equal expected-xml repaired-xml))))) - - (define-scenario (thin-dump repair-superblock missing-transaction-id) - "--transaction-id is mandatory if the superblock is damaged" - (with-damaged-superblock (md) - (run-fail-rcv (_ stderr) (thin-dump "--repair" "--data-block-size=128" "--nr-data-blocks=4096000" md) - (assert-matches ".*transaction id.*" stderr)))) - - (define-scenario (thin-dump repair-superblock missing-data-block-size) - "--data-block-size is mandatory if the superblock is damaged" - (with-damaged-superblock (md) - (run-fail-rcv (_ stderr) (thin-dump "--repair" "--transaction-id=5" "--nr-data-blocks=4096000" md) - (assert-matches ".*data block size.*" stderr)))) - - (define-scenario (thin-dump repair-superblock missing-nr-data-blocks) - "--nr-data-blocks is mandatory if the superblock is damaged" - (with-damaged-superblock (md) - (run-fail-rcv (_ stderr) (thin-dump "--repair" "--transaction-id=5" "--data-block-size=128" md) - (assert-matches ".*nr data blocks.*" stderr)))) - ;;;----------------------------------------------------------- ;;; thin_rmap scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 52ee3e1..37a3348 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,14 +1,14 @@ use anyhow::Result; use duct::{cmd, Expression}; use std::fs::OpenOptions; -use std::path::{Path, PathBuf}; +use std::io::{Read, Write}; +use std::path::{Display, PathBuf}; use std::str::from_utf8; use tempfile::{tempdir, TempDir}; use thinp::file_utils; -use std::io::{Read}; pub mod xml_generator; -use crate::common::xml_generator::{write_xml, FragmentedS, SingleThinS}; +use crate::common::xml_generator::{write_xml, SingleThinS}; //------------------------------------------ @@ -56,7 +56,7 @@ pub struct TestDir { impl TestDir { pub fn new() -> Result { let dir = tempdir()?; - Ok(TestDir {dir, file_count: 0}) + Ok(TestDir { dir, file_count: 0 }) } pub fn mk_path(&mut self, file: &str) -> PathBuf { @@ -96,6 +96,7 @@ pub fn mk_valid_md(td: &mut TestDir) -> Result { pub fn mk_zeroed_md(td: &mut TestDir) -> Result { let md = td.mk_path("meta.bin"); + eprintln!("path = {:?}", md); let _file = file_utils::create_sized_file(&md, 4096 * 4096); Ok(md) } @@ -116,8 +117,15 @@ pub fn superblock_all_zeroes(path: &PathBuf) -> Result { return Ok(false); } } - + Ok(true) } +pub fn damage_superblock(path: &PathBuf) -> Result<()> { + let mut output = OpenOptions::new().read(false).write(true).open(path)?; + let buf = [0u8; 512]; + output.write_all(&buf)?; + Ok(()) +} + //------------------------------------------ diff --git a/tests/thin_check.rs b/tests/thin_check.rs index 42caba6..52c7696 100644 --- a/tests/thin_check.rs +++ b/tests/thin_check.rs @@ -1,8 +1,4 @@ use anyhow::Result; -use duct::{cmd, Expression}; -use std::path::{Path, PathBuf}; -use std::str::from_utf8; -use tempfile::{tempdir, TempDir}; use thinp::file_utils; use thinp::version::TOOLS_VERSION; diff --git a/tests/thin_dump.rs b/tests/thin_dump.rs new file mode 100644 index 0000000..f71c840 --- /dev/null +++ b/tests/thin_dump.rs @@ -0,0 +1,136 @@ +use anyhow::Result; +use thinp::file_utils; +use std::fs::OpenOptions; +use std::io::{Write}; +use std::str::from_utf8; + +mod common; + +use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; +use common::*; + +//------------------------------------------ + +#[test] +fn small_input_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = td.mk_path("meta.bin"); + file_utils::create_sized_file(&md, 512)?; + let _stderr = run_fail(thin_dump!(&md))?; + Ok(()) +} + +#[test] +fn dump_restore_cycle() -> Result<()> { + let mut td = TestDir::new()?; + + let md = mk_valid_md(&mut td)?; + let output = thin_dump!(&md).run()?; + + let xml = td.mk_path("meta.xml"); + let mut file = OpenOptions::new().read(false).write(true).create(true).open(&xml)?; + file.write_all(&output.stdout[0..])?; + drop(file); + + let md2 = mk_zeroed_md(&mut td)?; + thin_restore!("-i", &xml, "-o", &md2).run()?; + + let output2 = thin_dump!(&md2).run()?; + assert_eq!(output.stdout, output2.stdout); + + Ok(()) +} + +#[test] +fn no_stderr() -> Result<()> { + let mut td = TestDir::new()?; + + let md = mk_valid_md(&mut td)?; + let output = thin_dump!(&md).run()?; + + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +fn override_something(flag: &str, value: &str, pattern: &str) -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + let output = thin_dump!(&md, flag, value).run()?; + + assert_eq!(output.stderr.len(), 0); + assert!(from_utf8(&output.stdout[0..])?.contains(pattern)); + Ok(()) +} + +#[test] +fn override_transaction_id() -> Result<()> { + override_something("--transaction-id", "2345", "transaction=\"2345\"") +} + +#[test] +fn override_data_block_size() -> Result<()> { + override_something("--data-block-size", "8192", "data_block_size=\"8192\"") +} + +#[test] +fn override_nr_data_blocks() -> Result<()> { + override_something("--nr-data-blocks", "234500", "nr_data_blocks=\"234500\"") +} + +#[test] +fn repair_superblock() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + let before = thin_dump!("--transaction-id=5", "--data-block-size=128", "--nr-data-blocks=4096000", &md).run()?; + damage_superblock(&md)?; + + let after = thin_dump!("--repair", "--transaction-id=5", "--data-block-size=128", "--nr-data-blocks=4096000", &md).run()?; + assert_eq!(after.stderr.len(), 0); + assert_eq!(before.stdout, after.stdout); + + Ok(()) +} + +#[test] +fn missing_transaction_id() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + damage_superblock(&md)?; + let stderr = run_fail(thin_dump!("--repair", "--data-block-size=128", "--nr-data-blocks=4096000", &md))?; + assert!(stderr.contains("transaction id")); + Ok(()) +} + +#[test] +fn missing_data_block_size() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + damage_superblock(&md)?; + let stderr = run_fail(thin_dump!("--repair", "--transaction-id=5", "--nr-data-blocks=4096000", &md))?; + assert!(stderr.contains("data block size")); + Ok(()) +} + +#[test] +fn missing_nr_data_blocks() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + damage_superblock(&md)?; + let stderr = run_fail(thin_dump!("--repair", "--transaction-id=5", "--data-block-size=128", &md))?; + assert!(stderr.contains("nr data blocks")); + Ok(()) +} + + +// (define-scenario (thin-dump repair-superblock missing-data-block-size) +// "--data-block-size is mandatory if the superblock is damaged" +// (with-damaged-superblock (md) +// (run-fail-rcv (_ stderr) (thin-dump "--repair" "--transaction-id=5" "--nr-data-blocks=4096000" md) +// (assert-matches ".*data block size.*" stderr)))) +// +// (define-scenario (thin-dump repair-superblock missing-nr-data-blocks) +// "--nr-data-blocks is mandatory if the superblock is damaged" +// (with-damaged-superblock (md) +// (run-fail-rcv (_ stderr) (thin-dump "--repair" "--transaction-id=5" "--data-block-size=128" md) +// (assert-matches ".*nr data blocks.*" stderr)))) +// diff --git a/tests/thin_restore.rs b/tests/thin_restore.rs index e8861bc..daca8e9 100644 --- a/tests/thin_restore.rs +++ b/tests/thin_restore.rs @@ -1,8 +1,5 @@ use anyhow::Result; -use duct::{cmd, Expression}; -use std::path::{Path, PathBuf}; use std::str::from_utf8; -use tempfile::{tempdir, TempDir}; use thinp::file_utils; use thinp::version::TOOLS_VERSION; diff --git a/tests/thin_shrink.rs b/tests/thin_shrink.rs index 8efae51..3397d20 100644 --- a/tests/thin_shrink.rs +++ b/tests/thin_shrink.rs @@ -3,8 +3,7 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use rand::prelude::*; use std::fs::OpenOptions; use std::io::{Cursor, Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; -use tempfile::tempdir; +use std::path::{Path}; use thinp::file_utils; use thinp::thin::xml::{self, Visit}; From ad29fe65fa8162fae2d972d77468432b920d01ce Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 31 Jul 2020 16:34:04 +0100 Subject: [PATCH 17/52] [functional-tests] Use thin_restore macro --- tests/common/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 37a3348..cf23480 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -90,7 +90,7 @@ pub fn mk_valid_md(td: &mut TestDir) -> Result { write_xml(&xml, &mut gen)?; let _file = file_utils::create_sized_file(&md, 4096 * 4096); - cmd!("bin/thin_restore", "-i", xml, "-o", &md).run()?; + thin_restore!("-i", xml, "-o", &md).run()?; Ok(md) } From 9552cb4817ed94fd0220982b38814d463bdce903 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 3 Aug 2020 10:47:03 +0100 Subject: [PATCH 18/52] [functional-tests] Port thin_rmap tests to Rust. cargo test --- functional-tests/thin-functional-tests.scm | 52 ------------ tests/common/mod.rs | 13 ++- tests/thin_check.rs | 2 +- tests/thin_dump.rs | 14 ---- tests/thin_rmap.rs | 97 ++++++++++++++++++++++ 5 files changed, 110 insertions(+), 68 deletions(-) create mode 100644 tests/thin_rmap.rs diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm index 78ab791..ee3372f 100644 --- a/functional-tests/thin-functional-tests.scm +++ b/functional-tests/thin-functional-tests.scm @@ -67,58 +67,6 @@ ;; to run. (define (register-thin-tests) #t) - ;;;----------------------------------------------------------- - ;;; thin_rmap scenarios - ;;;----------------------------------------------------------- - - (define-scenario (thin-rmap v) - "thin_rmap accepts -V" - (run-ok-rcv (stdout _) (thin-rmap "-V") - (assert-equal tools-version stdout))) - - (define-scenario (thin-rmap version) - "thin_rmap accepts --version" - (run-ok-rcv (stdout _) (thin-rmap "--version") - (assert-equal tools-version stdout))) - - (define-scenario (thin-rmap h) - "thin_rmap accepts -h" - (run-ok-rcv (stdout _) (thin-rmap "-h") - (assert-equal thin-rmap-help stdout))) - - (define-scenario (thin-rmap help) - "thin_rmap accepts --help" - (run-ok-rcv (stdout _) (thin-rmap "--help") - (assert-equal thin-rmap-help stdout))) - - (define-scenario (thin-rmap unrecognised-flag) - "thin_rmap complains with bad flags." - (run-fail (thin-rmap "--unleash-the-hedgehogs"))) - - (define-scenario (thin-rmap valid-region-format-should-pass) - "thin_rmap with a valid region format should pass." - (with-valid-metadata (md) - (run-ok - (thin-rmap "--region 23..7890" md)))) - - (define-scenario (thin-rmap invalid-region-should-fail) - "thin_rmap with an invalid region format should fail." - (for-each (lambda (pattern) - (with-valid-metadata (md) - (run-fail (thin-rmap "--region" pattern md)))) - '("23,7890" "23..six" "found..7890" "89..88" "89..89" "89.." "" "89...99"))) - - (define-scenario (thin-rmap multiple-regions-should-pass) - "thin_rmap should handle multiple regions." - (with-valid-metadata (md) - (run-ok (thin-rmap "--region 1..23 --region 45..78" md)))) - - (define-scenario (thin-rmap handles-junk-input) - "Fail gracefully if given nonsense" - (with-thin-xml (xml) - (run-fail-rcv (_ stderr) (thin-rmap "--region 0..-1" xml) - #t))) - ;;;----------------------------------------------------------- ;;; thin_delta scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/mod.rs b/tests/common/mod.rs index cf23480..416f302 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -46,6 +46,17 @@ macro_rules! thin_dump { }; } +#[macro_export] +macro_rules! thin_rmap { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_rmap", args).stdout_capture().stderr_capture() + } + }; +} + //------------------------------------------ pub struct TestDir { @@ -86,7 +97,7 @@ pub fn mk_valid_md(td: &mut TestDir) -> Result { let xml = td.mk_path("meta.xml"); let md = td.mk_path("meta.bin"); - let mut gen = SingleThinS::new(0, 1024, 2048, 2048); + let mut gen = SingleThinS::new(0, 1024, 20480, 20480); write_xml(&xml, &mut gen)?; let _file = file_utils::create_sized_file(&md, 4096 * 4096); diff --git a/tests/thin_check.rs b/tests/thin_check.rs index 52c7696..d18bdf2 100644 --- a/tests/thin_check.rs +++ b/tests/thin_check.rs @@ -23,7 +23,7 @@ fn accepts_version() -> Result<()> { Ok(()) } -const USAGE: &'static str = "Usage: thin_check [options] {device|file}\nOptions:\n {-q|--quiet}\n {-h|--help}\n {-V|--version}\n {-m|--metadata-snap}\n {--override-mapping-root}\n {--clear-needs-check-flag}\n {--ignore-non-fatal-errors}\n {--skip-mappings}\n {--super-block-only}"; +const USAGE: &str = "Usage: thin_check [options] {device|file}\nOptions:\n {-q|--quiet}\n {-h|--help}\n {-V|--version}\n {-m|--metadata-snap}\n {--override-mapping-root}\n {--clear-needs-check-flag}\n {--ignore-non-fatal-errors}\n {--skip-mappings}\n {--super-block-only}"; #[test] fn accepts_h() -> Result<()> { diff --git a/tests/thin_dump.rs b/tests/thin_dump.rs index f71c840..d4edaad 100644 --- a/tests/thin_dump.rs +++ b/tests/thin_dump.rs @@ -120,17 +120,3 @@ fn missing_nr_data_blocks() -> Result<()> { assert!(stderr.contains("nr data blocks")); Ok(()) } - - -// (define-scenario (thin-dump repair-superblock missing-data-block-size) -// "--data-block-size is mandatory if the superblock is damaged" -// (with-damaged-superblock (md) -// (run-fail-rcv (_ stderr) (thin-dump "--repair" "--transaction-id=5" "--nr-data-blocks=4096000" md) -// (assert-matches ".*data block size.*" stderr)))) -// -// (define-scenario (thin-dump repair-superblock missing-nr-data-blocks) -// "--nr-data-blocks is mandatory if the superblock is damaged" -// (with-damaged-superblock (md) -// (run-fail-rcv (_ stderr) (thin-dump "--repair" "--transaction-id=5" "--data-block-size=128" md) -// (assert-matches ".*nr data blocks.*" stderr)))) -// diff --git a/tests/thin_rmap.rs b/tests/thin_rmap.rs new file mode 100644 index 0000000..bdf4946 --- /dev/null +++ b/tests/thin_rmap.rs @@ -0,0 +1,97 @@ +use anyhow::Result; +use thinp::file_utils; +use std::fs::OpenOptions; +use std::io::{Write}; +use std::str::from_utf8; +use thinp::version::TOOLS_VERSION; + +mod common; + +use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; +use common::*; + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = thin_rmap!("-V").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = thin_rmap!("--version").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +const USAGE: &str = "Usage: thin_rmap [options] {device|file}\nOptions:\n {-h|--help}\n {-V|--version}\n {--region }*\nWhere:\n is of the form ..\n for example 5..45 denotes blocks 5 to 44 inclusive, but not block 45"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = thin_rmap!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = thin_rmap!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn rejects_bad_option() -> Result<()> { + let stderr = run_fail(thin_rmap!("--hedgehogs-only"))?; + assert!(stderr.contains("unrecognized option \'--hedgehogs-only\'")); + Ok(()) +} + +#[test] +fn valid_region_format_should_pass() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + let output = thin_rmap!("--region", "23..7890", &md).unchecked().run()?; + eprintln!("stdout: {:?}", output.stdout); + eprintln!("stderr: {:?}", output.stderr); + assert!(output.status.success()); + Ok(()) +} + +#[test] +fn invalid_regions_should_fail() -> Result<()> { + let invalid_regions = ["23,7890", "23..six", "found..7890", "89..88", "89..89", "89..", "", "89...99"]; + for r in &invalid_regions { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + run_fail(thin_rmap!(r, &md))?; + } + Ok(()) +} + +#[test] +fn multiple_regions_should_pass() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + thin_rmap!("--region", "1..23", "--region", "45..78", &md).run()?; + Ok(()) +} + +#[test] +fn junk_input() -> Result<()> { + let mut td = TestDir::new()?; + let xml = mk_valid_xml(&mut td)?; + let stderr = run_fail(thin_rmap!("--region", "0..-1", &xml))?; + Ok(()) +} + +// (define-scenario (thin-rmap handles-junk-input) +// "Fail gracefully if given nonsense" +// (with-thin-xml (xml) +// (run-fail-rcv (_ stderr) (thin-rmap "--region 0..-1" xml) +// #t))) + + +//------------------------------------------ From 39822a71650112db53e1c42bb7a066680c559d4b Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 3 Aug 2020 10:59:19 +0100 Subject: [PATCH 19/52] [functional-tests (rust)] squash warnings --- tests/common/mod.rs | 6 ++++-- tests/thin_check.rs | 2 +- tests/thin_dump.rs | 2 -- tests/thin_restore.rs | 2 -- tests/thin_rmap.rs | 8 +------- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 416f302..23fa38d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,8 +1,10 @@ +#![allow(dead_code)] + use anyhow::Result; -use duct::{cmd, Expression}; +use duct::{Expression}; use std::fs::OpenOptions; use std::io::{Read, Write}; -use std::path::{Display, PathBuf}; +use std::path::{PathBuf}; use std::str::from_utf8; use tempfile::{tempdir, TempDir}; use thinp::file_utils; diff --git a/tests/thin_check.rs b/tests/thin_check.rs index d18bdf2..a3aa22e 100644 --- a/tests/thin_check.rs +++ b/tests/thin_check.rs @@ -5,7 +5,7 @@ use thinp::version::TOOLS_VERSION; mod common; use common::*; -use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; +use common::xml_generator::{write_xml, FragmentedS}; //------------------------------------------ diff --git a/tests/thin_dump.rs b/tests/thin_dump.rs index d4edaad..c754eff 100644 --- a/tests/thin_dump.rs +++ b/tests/thin_dump.rs @@ -5,8 +5,6 @@ use std::io::{Write}; use std::str::from_utf8; mod common; - -use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; use common::*; //------------------------------------------ diff --git a/tests/thin_restore.rs b/tests/thin_restore.rs index daca8e9..819b292 100644 --- a/tests/thin_restore.rs +++ b/tests/thin_restore.rs @@ -4,8 +4,6 @@ use thinp::file_utils; use thinp::version::TOOLS_VERSION; mod common; - -use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; use common::*; //------------------------------------------ diff --git a/tests/thin_rmap.rs b/tests/thin_rmap.rs index bdf4946..187bf9b 100644 --- a/tests/thin_rmap.rs +++ b/tests/thin_rmap.rs @@ -1,13 +1,7 @@ use anyhow::Result; -use thinp::file_utils; -use std::fs::OpenOptions; -use std::io::{Write}; -use std::str::from_utf8; use thinp::version::TOOLS_VERSION; mod common; - -use common::xml_generator::{write_xml, FragmentedS, SingleThinS}; use common::*; //------------------------------------------ @@ -83,7 +77,7 @@ fn multiple_regions_should_pass() -> Result<()> { fn junk_input() -> Result<()> { let mut td = TestDir::new()?; let xml = mk_valid_xml(&mut td)?; - let stderr = run_fail(thin_rmap!("--region", "0..-1", &xml))?; + run_fail(thin_rmap!("--region", "0..-1", &xml))?; Ok(()) } From cdf19b2454ee46ba07eaff1b93a144f4ff962467 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 3 Aug 2020 11:00:09 +0100 Subject: [PATCH 20/52] [functional-tests (rust)] remove comment --- tests/thin_rmap.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/thin_rmap.rs b/tests/thin_rmap.rs index 187bf9b..e6255e6 100644 --- a/tests/thin_rmap.rs +++ b/tests/thin_rmap.rs @@ -81,11 +81,4 @@ fn junk_input() -> Result<()> { Ok(()) } -// (define-scenario (thin-rmap handles-junk-input) -// "Fail gracefully if given nonsense" -// (with-thin-xml (xml) -// (run-fail-rcv (_ stderr) (thin-rmap "--region 0..-1" xml) -// #t))) - - //------------------------------------------ From cc2582b8b19d0927797205938ce4e27ad310261c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 3 Aug 2020 12:37:32 +0100 Subject: [PATCH 21/52] [thin_check (rust)] factor out pdata/btree.rs --- src/lib.rs | 3 +- src/pdata/btree.rs | 187 +++++++++++++++++++++++++++++++++++++++++++++ src/pdata/mod.rs | 2 + src/thin/check.rs | 187 +-------------------------------------------- 4 files changed, 194 insertions(+), 185 deletions(-) create mode 100644 src/pdata/btree.rs create mode 100644 src/pdata/mod.rs diff --git a/src/lib.rs b/src/lib.rs index b7b2082..6d52650 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,9 +16,10 @@ extern crate quickcheck; extern crate quickcheck_macros; pub mod block_manager; +pub mod checksum; pub mod file_utils; pub mod pack; +pub mod pdata; pub mod shrink; pub mod thin; pub mod version; -pub mod checksum; diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs new file mode 100644 index 0000000..c19d89b --- /dev/null +++ b/src/pdata/btree.rs @@ -0,0 +1,187 @@ +use anyhow::{anyhow, Result}; +use fixedbitset::FixedBitSet; +use nom::{number::complete::*, IResult}; +use std::sync::{Arc, Mutex}; + +use crate::block_manager::*; +use crate::checksum; + +//------------------------------------------ + +pub trait ValueType { + type Value; + fn unpack(data: &[u8]) -> IResult<&[u8], Self::Value>; +} + +pub struct NodeHeader { + is_leaf: bool, + block: u64, + nr_entries: u32, + max_entries: u32, + value_size: u32, +} + +#[allow(dead_code)] +const INTERNAL_NODE: u32 = 1; +const LEAF_NODE: u32 = 2; + +pub fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> { + let (i, _csum) = le_u32(data)?; + let (i, flags) = le_u32(i)?; + let (i, block) = le_u64(i)?; + let (i, nr_entries) = le_u32(i)?; + let (i, max_entries) = le_u32(i)?; + let (i, value_size) = le_u32(i)?; + let (i, _padding) = le_u32(i)?; + + Ok(( + i, + NodeHeader { + is_leaf: flags == LEAF_NODE, + block, + nr_entries, + max_entries, + value_size, + }, + )) +} + +pub enum Node { + Internal { + header: NodeHeader, + keys: Vec, + values: Vec, + }, + Leaf { + header: NodeHeader, + keys: Vec, + values: Vec, + }, +} + +pub fn unpack_node_(data: &[u8]) -> IResult<&[u8], Node> { + use nom::multi::count; + + let (i, header) = unpack_node_header(data)?; + let (i, keys) = count(le_u64, header.nr_entries as usize)(i)?; + let nr_free = header.max_entries - header.nr_entries; + let (i, _padding) = count(le_u64, nr_free as usize)(i)?; + + if header.is_leaf { + let (i, values) = count(V::unpack, header.nr_entries as usize)(i)?; + Ok(( + i, + Node::Leaf { + header, + keys, + values, + }, + )) + } else { + let (i, values) = count(le_u64, header.nr_entries as usize)(i)?; + Ok(( + i, + Node::Internal { + header, + keys, + values, + }, + )) + } +} + +pub fn unpack_node(data: &[u8]) -> Result> { + if let Ok((_i, node)) = unpack_node_(data) { + Ok(node) + } else { + Err(anyhow!("couldn't unpack btree node")) + } +} + +//------------------------------------------ + +pub struct ValueU64; + +impl ValueType for ValueU64 { + type Value = u64; + fn unpack(i: &[u8]) -> IResult<&[u8], u64> { + le_u64(i) + } +} + +//------------------------------------------ + +pub trait NodeVisitor { + fn visit<'a>(&mut self, w: &BTreeWalker, b: &Block, node: &Node) -> Result<()>; +} + +#[derive(Clone)] +pub struct BTreeWalker { + pub engine: Arc, + pub seen: Arc>, +} + +impl BTreeWalker { + pub fn new(engine: AsyncIoEngine) -> BTreeWalker { + let nr_blocks = engine.get_nr_blocks() as usize; + let r: BTreeWalker = BTreeWalker { + engine: Arc::new(engine), + seen: Arc::new(Mutex::new(FixedBitSet::with_capacity(nr_blocks))), + }; + r + } + + pub fn walk_nodes(&mut self, visitor: &mut NV, bs: &Vec) -> Result<()> + where + NV: NodeVisitor, + V: ValueType, + { + let mut blocks = Vec::new(); + let seen = self.seen.lock().unwrap(); + for b in bs { + if !seen[*b as usize] { + blocks.push(Block::new(*b)); + } + } + drop(seen); + + self.engine.read_many(&mut blocks)?; + + for b in blocks { + self.walk_node(visitor, &b)?; + } + + Ok(()) + } + + pub fn walk_node(&mut self, visitor: &mut NV, b: &Block) -> Result<()> + where + NV: NodeVisitor, + V: ValueType, + { + let mut seen = self.seen.lock().unwrap(); + seen.insert(b.loc as usize); + drop(seen); + + let bt = checksum::metadata_block_type(b.get_data()); + if bt != checksum::BT::NODE { + return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); + } + + let node = unpack_node::(&b.get_data())?; + visitor.visit(self, &b, &node)?; + + if let Node::Internal { + header: _h, + keys: _k, + values, + } = node + { + self.walk_nodes(visitor, &values)?; + } + + Ok(()) + } +} + +//------------------------------------------ diff --git a/src/pdata/mod.rs b/src/pdata/mod.rs new file mode 100644 index 0000000..83204ba --- /dev/null +++ b/src/pdata/mod.rs @@ -0,0 +1,2 @@ +pub mod btree; + diff --git a/src/thin/check.rs b/src/thin/check.rs index 90d9703..46bcfcc 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -7,188 +6,8 @@ use std::time::Instant; use threadpool::ThreadPool; use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; -use crate::checksum; use crate::thin::superblock::*; - -//------------------------------------------ - -trait ValueType { - type Value; - fn unpack(data: &[u8]) -> IResult<&[u8], Self::Value>; -} - -#[allow(dead_code)] -struct NodeHeader { - is_leaf: bool, - block: u64, - nr_entries: u32, - max_entries: u32, - value_size: u32, -} - -#[allow(dead_code)] -const INTERNAL_NODE: u32 = 1; -const LEAF_NODE: u32 = 2; - -fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> { - let (i, _csum) = le_u32(data)?; - let (i, flags) = le_u32(i)?; - let (i, block) = le_u64(i)?; - let (i, nr_entries) = le_u32(i)?; - let (i, max_entries) = le_u32(i)?; - let (i, value_size) = le_u32(i)?; - let (i, _padding) = le_u32(i)?; - - Ok(( - i, - NodeHeader { - is_leaf: flags == LEAF_NODE, - block, - nr_entries, - max_entries, - value_size, - }, - )) -} - -enum Node { - Internal { - header: NodeHeader, - keys: Vec, - values: Vec, - }, - Leaf { - header: NodeHeader, - keys: Vec, - values: Vec, - }, -} - -fn unpack_node_(data: &[u8]) -> IResult<&[u8], Node> { - use nom::multi::count; - - let (i, header) = unpack_node_header(data)?; - let (i, keys) = count(le_u64, header.nr_entries as usize)(i)?; - - let nr_free = header.max_entries - header.nr_entries; - let (i, _padding) = count(le_u64, nr_free as usize)(i)?; - - if header.is_leaf { - let (i, values) = count(V::unpack, header.nr_entries as usize)(i)?; - Ok(( - i, - Node::Leaf { - header, - keys, - values, - }, - )) - } else { - let (i, values) = count(le_u64, header.nr_entries as usize)(i)?; - Ok(( - i, - Node::Internal { - header, - keys, - values, - }, - )) - } -} - -fn unpack_node(data: &[u8]) -> Result> { - if let Ok((_i, node)) = unpack_node_(data) { - Ok(node) - } else { - Err(anyhow!("couldn't unpack btree node")) - } -} - -//------------------------------------------ - -struct ValueU64; - -impl ValueType for ValueU64 { - type Value = u64; - fn unpack(i: &[u8]) -> IResult<&[u8], u64> { - le_u64(i) - } -} - -//------------------------------------------ - -trait NodeVisitor { - fn visit<'a>(&mut self, w: &BTreeWalker, b: &Block, node: &Node) -> Result<()>; -} - -#[derive(Clone)] -struct BTreeWalker { - engine: Arc, - seen: Arc>, -} - -impl BTreeWalker { - fn new(engine: AsyncIoEngine) -> BTreeWalker { - let nr_blocks = engine.get_nr_blocks() as usize; - let r: BTreeWalker = BTreeWalker { - engine: Arc::new(engine), - seen: Arc::new(Mutex::new(FixedBitSet::with_capacity(nr_blocks))), - }; - r - } - - fn walk_nodes(&mut self, visitor: &mut NV, bs: &Vec) -> Result<()> - where - NV: NodeVisitor, - V: ValueType, - { - let mut blocks = Vec::new(); - let seen = self.seen.lock().unwrap(); - for b in bs { - if !seen[*b as usize] { - blocks.push(Block::new(*b)); - } - } - drop(seen); - - self.engine.read_many(&mut blocks)?; - - for b in blocks { - self.walk_node(visitor, &b)?; - } - - Ok(()) - } - - fn walk_node(&mut self, visitor: &mut NV, b: &Block) -> Result<()> - where - NV: NodeVisitor, - V: ValueType, - { - let mut seen = self.seen.lock().unwrap(); - seen.insert(b.loc as usize); - drop(seen); - - let bt = checksum::metadata_block_type(b.get_data()); - if bt != checksum::BT::NODE { - return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); - } - - let node = unpack_node::(&b.get_data())?; - visitor.visit(self, &b, &node)?; - - if let Node::Internal { - header: _h, - keys: _k, - values, - } = node - { - self.walk_nodes(visitor, &values)?; - } - - Ok(()) - } -} +use crate::pdata::btree::{ValueType, Node, BTreeWalker, NodeVisitor, ValueU64}; //------------------------------------------ @@ -241,7 +60,7 @@ impl NodeVisitor for TopLevelVisitor { w.engine.read_many(&mut blocks)?; - // FIXME: with a thread pool we need to return errors another way. + // FIXME: with a thread pool we need to return errors another way. let nr_workers = 4; let pool = ThreadPool::new(nr_workers); @@ -249,7 +68,7 @@ impl NodeVisitor for TopLevelVisitor { for b in blocks { let thin_id = thin_ids[n]; n += 1; - + let mut w = w.clone(); pool.execute(move || { let mut v = BottomLevelVisitor {}; From 1368227a71ebbf8a52560f3e27d9f71c106ad9a5 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 3 Aug 2020 15:04:59 +0100 Subject: [PATCH 22/52] [thin_check (rust)] add btree node checks --- src/pdata/btree.rs | 141 ++++++++++++++++++++++++++++++++++----------- src/thin/check.rs | 15 +++-- 2 files changed, 116 insertions(+), 40 deletions(-) diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index c19d89b..90d9733 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -10,9 +10,13 @@ use crate::checksum; pub trait ValueType { type Value; + // The size of the value when on disk. + fn disk_size() -> u32; fn unpack(data: &[u8]) -> IResult<&[u8], Self::Value>; } +const NODE_HEADER_SIZE: usize = 32; + pub struct NodeHeader { is_leaf: bool, block: u64, @@ -59,42 +63,90 @@ pub enum Node { }, } -pub fn unpack_node_(data: &[u8]) -> IResult<&[u8], Node> { - use nom::multi::count; +pub fn node_err(msg: String) -> Result { + let msg = format!("btree node error: {}", msg); + Err(anyhow!(msg)) +} - let (i, header) = unpack_node_header(data)?; - let (i, keys) = count(le_u64, header.nr_entries as usize)(i)?; - let nr_free = header.max_entries - header.nr_entries; - let (i, _padding) = count(le_u64, nr_free as usize)(i)?; - - if header.is_leaf { - let (i, values) = count(V::unpack, header.nr_entries as usize)(i)?; - Ok(( - i, - Node::Leaf { - header, - keys, - values, - }, - )) +pub fn to_any<'a, V>(r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> { + if let Ok((i, v)) = r { + Ok((i, v)) } else { - let (i, values) = count(le_u64, header.nr_entries as usize)(i)?; - Ok(( - i, - Node::Internal { - header, - keys, - values, - }, - )) + Err(anyhow!("btree node error: parse error")) } } -pub fn unpack_node(data: &[u8]) -> Result> { - if let Ok((_i, node)) = unpack_node_(data) { - Ok(node) +pub fn unpack_node( + data: &[u8], + ignore_non_fatal: bool, + is_root: bool, +) -> Result> { + use nom::multi::count; + + let (i, header) = to_any(unpack_node_header(data))?; + + // FIXME: lift checks to own fn + if header.value_size != V::disk_size() { + return node_err(format!( + "value_size mismatch: expected {}, was {}", + V::disk_size(), + header.value_size + )); + } + + let elt_size = V::disk_size() + 8; + if elt_size as usize * header.max_entries as usize + NODE_HEADER_SIZE > BLOCK_SIZE { + return node_err(format!("max_entries is too large ({})", header.max_entries)); + } + + if header.nr_entries > header.max_entries { + return node_err(format!("nr_entries > max_entries")); + } + + if !ignore_non_fatal { + if header.max_entries % 3 != 0 { + return node_err(format!("max_entries is not divisible by 3")); + } + + if !is_root { + let min = header.max_entries / 3; + if header.nr_entries < min { + return node_err(format!("too few entries")); + } + } + } + + let (i, keys) = to_any(count(le_u64, header.nr_entries as usize)(i))?; + + let mut last = None; + for k in &keys { + if let Some(l) = last { + if k <= l { + return node_err(format!("keys out of order")); + } + } + + last = Some(k); + } + + let nr_free = header.max_entries - header.nr_entries; + let (i, _padding) = to_any(count(le_u64, nr_free as usize)(i))?; + + if header.is_leaf { + let (_i, values) = to_any(count(V::unpack, header.nr_entries as usize)(i))?; + + Ok(Node::Leaf { + header, + keys, + values, + }) } else { - Err(anyhow!("couldn't unpack btree node")) + let (_i, values) = to_any(count(le_u64, header.nr_entries as usize)(i))?; + Ok(Node::Internal { + header, + keys, + values, + }) } } @@ -104,6 +156,11 @@ pub struct ValueU64; impl ValueType for ValueU64 { type Value = u64; + + fn disk_size() -> u32 { + 8 + } + fn unpack(i: &[u8]) -> IResult<&[u8], u64> { le_u64(i) } @@ -119,19 +176,21 @@ pub trait NodeVisitor { pub struct BTreeWalker { pub engine: Arc, pub seen: Arc>, + ignore_non_fatal: bool, } impl BTreeWalker { - pub fn new(engine: AsyncIoEngine) -> BTreeWalker { + pub fn new(engine: AsyncIoEngine, ignore_non_fatal: bool) -> BTreeWalker { let nr_blocks = engine.get_nr_blocks() as usize; let r: BTreeWalker = BTreeWalker { engine: Arc::new(engine), seen: Arc::new(Mutex::new(FixedBitSet::with_capacity(nr_blocks))), + ignore_non_fatal, }; r } - pub fn walk_nodes(&mut self, visitor: &mut NV, bs: &Vec) -> Result<()> + fn walk_nodes(&mut self, visitor: &mut NV, bs: &Vec) -> Result<()> where NV: NodeVisitor, V: ValueType, @@ -148,13 +207,13 @@ impl BTreeWalker { self.engine.read_many(&mut blocks)?; for b in blocks { - self.walk_node(visitor, &b)?; + self.walk_node(visitor, &b, false)?; } Ok(()) } - pub fn walk_node(&mut self, visitor: &mut NV, b: &Block) -> Result<()> + fn walk_node(&mut self, visitor: &mut NV, b: &Block, is_root: bool) -> Result<()> where NV: NodeVisitor, V: ValueType, @@ -168,7 +227,7 @@ impl BTreeWalker { return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); } - let node = unpack_node::(&b.get_data())?; + let node = unpack_node::(&b.get_data(), self.ignore_non_fatal, is_root)?; visitor.visit(self, &b, &node)?; if let Node::Internal { @@ -182,6 +241,18 @@ impl BTreeWalker { Ok(()) } + + pub fn walk( + &mut self, + visitor: &mut NV, + root: &Block, + ) -> Result<()> + where + NV: NodeVisitor, + V: ValueType, + { + self.walk_node(visitor, &root, true) + } } //------------------------------------------ diff --git a/src/thin/check.rs b/src/thin/check.rs index 46bcfcc..4946c30 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -6,8 +6,8 @@ use std::time::Instant; use threadpool::ThreadPool; use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; +use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, ValueType, ValueU64}; use crate::thin::superblock::*; -use crate::pdata::btree::{ValueType, Node, BTreeWalker, NodeVisitor, ValueU64}; //------------------------------------------ @@ -21,6 +21,11 @@ struct ValueBlockTime; impl ValueType for ValueBlockTime { type Value = BlockTime; + + fn disk_size() -> u32 { + 8 + } + fn unpack(i: &[u8]) -> IResult<&[u8], BlockTime> { let (i, n) = le_u64(i)?; let block = n >> 24; @@ -72,8 +77,8 @@ impl NodeVisitor for TopLevelVisitor { let mut w = w.clone(); pool.execute(move || { let mut v = BottomLevelVisitor {}; - w.walk_node(&mut v, &b).expect("walk failed"); // FIXME: return error - eprintln!("checked thin_dev {}", thin_id); + let result = w.walk(&mut v, &b).expect("walk failed"); // FIXME: return error + eprintln!("checked thin_dev {} -> {:?}", thin_id, result); }); } @@ -105,9 +110,9 @@ pub fn check(dev: &Path) -> Result<()> { let mut root = Block::new(sb.mapping_root); engine.read(&mut root)?; - let mut w = BTreeWalker::new(engine); let mut visitor = TopLevelVisitor {}; - let _result = w.walk_node(&mut visitor, &root)?; + let mut w = BTreeWalker::new(engine, false); + let _result = w.walk(&mut visitor, &root)?; println!("read mapping tree in {} ms", now.elapsed().as_millis()); Ok(()) From f56ea2d031ef81cc461450f37b3251953a49d3a0 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 3 Aug 2020 16:22:08 +0100 Subject: [PATCH 23/52] [thin_check (rust)] walk devices tree. --- src/pdata/btree.rs | 40 ++++++++-------- src/thin/check.rs | 104 ++++++++++++++++++++++++++++++++--------- src/thin/superblock.rs | 2 +- 3 files changed, 105 insertions(+), 41 deletions(-) diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 90d9733..1069eb3 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -9,10 +9,11 @@ use crate::checksum; //------------------------------------------ pub trait ValueType { - type Value; // The size of the value when on disk. fn disk_size() -> u32; - fn unpack(data: &[u8]) -> IResult<&[u8], Self::Value>; + fn unpack(data: &[u8]) -> IResult<&[u8], Self> + where + Self: std::marker::Sized; } const NODE_HEADER_SIZE: usize = 32; @@ -59,7 +60,7 @@ pub enum Node { Leaf { header: NodeHeader, keys: Vec, - values: Vec, + values: Vec, }, } @@ -85,8 +86,7 @@ pub fn unpack_node( let (i, header) = to_any(unpack_node_header(data))?; - // FIXME: lift checks to own fn - if header.value_size != V::disk_size() { + if header.is_leaf && header.value_size != V::disk_size() { return node_err(format!( "value_size mismatch: expected {}, was {}", V::disk_size(), @@ -94,7 +94,7 @@ pub fn unpack_node( )); } - let elt_size = V::disk_size() + 8; + let elt_size = header.value_size + 8; if elt_size as usize * header.max_entries as usize + NODE_HEADER_SIZE > BLOCK_SIZE { return node_err(format!("max_entries is too large ({})", header.max_entries)); } @@ -152,11 +152,7 @@ pub fn unpack_node( //------------------------------------------ -pub struct ValueU64; - -impl ValueType for ValueU64 { - type Value = u64; - +impl ValueType for u64 { fn disk_size() -> u32 { 8 } @@ -180,10 +176,10 @@ pub struct BTreeWalker { } impl BTreeWalker { - pub fn new(engine: AsyncIoEngine, ignore_non_fatal: bool) -> BTreeWalker { + pub fn new(engine: Arc, ignore_non_fatal: bool) -> BTreeWalker { let nr_blocks = engine.get_nr_blocks() as usize; let r: BTreeWalker = BTreeWalker { - engine: Arc::new(engine), + engine: engine, seen: Arc::new(Mutex::new(FixedBitSet::with_capacity(nr_blocks))), ignore_non_fatal, }; @@ -242,16 +238,22 @@ impl BTreeWalker { Ok(()) } - pub fn walk( - &mut self, - visitor: &mut NV, - root: &Block, - ) -> Result<()> + pub fn walk_b(&mut self, visitor: &mut NV, root: &Block) -> Result<()> where NV: NodeVisitor, V: ValueType, { - self.walk_node(visitor, &root, true) + self.walk_node(visitor, &root, true) + } + + pub fn walk(&mut self, visitor: &mut NV, root: u64) -> Result<()> + where + NV: NodeVisitor, + V: ValueType, + { + let mut root = Block::new(root); + self.engine.read(&mut root)?; + self.walk_node(visitor, &root, true) } } diff --git a/src/thin/check.rs b/src/thin/check.rs index 4946c30..3717a73 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,12 +1,13 @@ use anyhow::{anyhow, Result}; use nom::{number::complete::*, IResult}; +use std::collections::HashMap; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::Instant; use threadpool::ThreadPool; use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; -use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, ValueType, ValueU64}; +use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, ValueType}; use crate::thin::superblock::*; //------------------------------------------ @@ -17,11 +18,7 @@ struct BlockTime { time: u32, } -struct ValueBlockTime; - -impl ValueType for ValueBlockTime { - type Value = BlockTime; - +impl ValueType for BlockTime { fn disk_size() -> u32 { 8 } @@ -43,8 +40,8 @@ impl ValueType for ValueBlockTime { struct TopLevelVisitor {} -impl NodeVisitor for TopLevelVisitor { - fn visit(&mut self, w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { +impl NodeVisitor for TopLevelVisitor { + fn visit(&mut self, w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { if let Node::Leaf { header: _h, keys, @@ -77,7 +74,7 @@ impl NodeVisitor for TopLevelVisitor { let mut w = w.clone(); pool.execute(move || { let mut v = BottomLevelVisitor {}; - let result = w.walk(&mut v, &b).expect("walk failed"); // FIXME: return error + let result = w.walk_b(&mut v, &b).expect("walk failed"); // FIXME: return error eprintln!("checked thin_dev {} -> {:?}", thin_id, result); }); } @@ -91,8 +88,67 @@ impl NodeVisitor for TopLevelVisitor { struct BottomLevelVisitor {} -impl NodeVisitor for BottomLevelVisitor { - fn visit(&mut self, _w: &BTreeWalker, _b: &Block, _node: &Node) -> Result<()> { +impl NodeVisitor for BottomLevelVisitor { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, _node: &Node) -> Result<()> { + Ok(()) + } +} + +//------------------------------------------ + +#[derive(Clone)] +struct DeviceDetail { + mapped_blocks: u64, + transaction_id: u64, + creation_time: u32, + snapshotted_time: u32, +} + +impl ValueType for DeviceDetail { + fn disk_size() -> u32 { + 24 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], DeviceDetail> { + let (i, mapped_blocks) = le_u64(i)?; + let (i, transaction_id) = le_u64(i)?; + let (i, creation_time) = le_u32(i)?; + let (i, snapshotted_time) = le_u32(i)?; + + Ok(( + i, + DeviceDetail { + mapped_blocks, + transaction_id, + creation_time, + snapshotted_time, + }, + )) + } +} + +struct DeviceVisitor { + devs: HashMap, +} + +impl DeviceVisitor { + pub fn new() -> DeviceVisitor { + DeviceVisitor { + devs: HashMap::new(), + } + } +} + +impl NodeVisitor for DeviceVisitor { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf {header: _h, keys, values} = node { + for n in 0..keys.len() { + let k = keys[n] as u32; + let v = values[n].clone(); + self.devs.insert(k, v.clone()); + } + } + Ok(()) } } @@ -101,19 +157,25 @@ impl NodeVisitor for BottomLevelVisitor { pub fn check(dev: &Path) -> Result<()> { //let mut engine = SyncIoEngine::new(dev)?; - let mut engine = AsyncIoEngine::new(dev, 256)?; + let mut engine = Arc::new(AsyncIoEngine::new(dev, 256)?); let now = Instant::now(); - let sb = read_superblock(&mut engine, SUPERBLOCK_LOCATION)?; + let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; eprintln!("{:?}", sb); - - let mut root = Block::new(sb.mapping_root); - engine.read(&mut root)?; - - let mut visitor = TopLevelVisitor {}; - let mut w = BTreeWalker::new(engine, false); - let _result = w.walk(&mut visitor, &root)?; - println!("read mapping tree in {} ms", now.elapsed().as_millis()); + + { + let mut visitor = DeviceVisitor::new(); + let mut w = BTreeWalker::new(engine.clone(), false); + w.walk(&mut visitor, sb.details_root)?; + println!("found {} devices", visitor.devs.len()); + } + + { + let mut visitor = TopLevelVisitor {}; + let mut w = BTreeWalker::new(engine.clone(), false); + let _result = w.walk(&mut visitor, sb.mapping_root)?; + println!("read mapping tree in {} ms", now.elapsed().as_millis()); + } Ok(()) } diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index e52d7f2..5001eb6 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -85,7 +85,7 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { )) } -pub fn read_superblock(engine: &mut E, loc: u64) -> Result { +pub fn read_superblock(engine: &E, loc: u64) -> Result { let mut b = Block::new(loc); engine.read(&mut b)?; From 4ac428128aaf6f880ed684e7dd4454edf0eeea56 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 4 Aug 2020 11:30:46 +0100 Subject: [PATCH 24/52] [functional-tests (rust)] port thin_repair tests to rust. cargo test --- functional-tests/thin-functional-tests.scm | 91 ------------ tests/common/mod.rs | 11 ++ tests/thin_repair.rs | 158 +++++++++++++++++++++ tests/thin_rmap.rs | 5 +- 4 files changed, 170 insertions(+), 95 deletions(-) create mode 100644 tests/thin_repair.rs diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm index ee3372f..c921b87 100644 --- a/functional-tests/thin-functional-tests.scm +++ b/functional-tests/thin-functional-tests.scm @@ -111,97 +111,6 @@ (run-fail-rcv (_ stderr) (thin-delta "--snap1 45 --snap2 46") (assert-starts-with "No input device provided." stderr))) - ;;;----------------------------------------------------------- - ;;; thin_repair scenarios - ;;;----------------------------------------------------------- - (define-scenario (thin-repair dont-repair-xml) - "Fails gracefully if run on XML rather than metadata" - (with-thin-xml (xml) - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-repair "-i" xml "-o" md) - #t)))) - - (define-scenario (thin-repair missing-input-file) - "the input file can't be found" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-repair "-i no-such-file -o" md) - (assert-superblock-all-zeroes md) - (assert-starts-with "Couldn't stat file" stderr)))) - - (define-scenario (thin-repair garbage-input-file) - "the input file is just zeroes" - (with-empty-metadata (md1) - (with-corrupt-metadata (md2) - (run-fail-rcv (_ stderr) (thin-repair "-i " md1 "-o" md2) - (assert-superblock-all-zeroes md2))))) - - (define-scenario (thin-repair missing-output-file) - "the output file can't be found" - (with-thin-xml (xml) - (run-fail-rcv (_ stderr) (thin-repair "-i " xml) - (assert-starts-with "No output file provided." stderr)))) - - (define-scenario (thin-repair override transaction-id) - "thin_repair obeys the --transaction-id override" - (with-valid-metadata (md1) - (with-empty-metadata (md2) - (run-ok-rcv (stdout stderr) (thin-repair "--transaction-id 2345" "-i" md1 "-o" md2) - (assert-eof stderr)) - (run-ok-rcv (stdout stderr) (thin-dump md2) - (assert-matches ".*transaction=\"2345\"" stdout))))) - - (define-scenario (thin-repair override data-block-size) - "thin_repair obeys the --data-block-size override" - (with-valid-metadata (md1) - (with-empty-metadata (md2) - (run-ok-rcv (stdout stderr) (thin-repair "--data-block-size 8192" "-i" md1 "-o" md2) - (assert-eof stderr)) - (run-ok-rcv (stdout stderr) (thin-dump md2) - (assert-matches ".*data_block_size=\"8192\"" stdout))))) - - (define-scenario (thin-repair override nr-data-blocks) - "thin_repair obeys the --nr-data-blocks override" - (with-valid-metadata (md1) - (with-empty-metadata (md2) - (run-ok-rcv (stdout stderr) (thin-repair "--nr-data-blocks 234500" "-i" md1 "-o" md2) - (assert-eof stderr)) - (run-ok-rcv (stdout stderr) (thin-dump md2) - (assert-matches ".*nr_data_blocks=\"234500\"" stdout))))) - - (define-scenario (thin-repair superblock succeeds) - "thin_repair can restore a missing superblock" - (with-valid-metadata (md1) - (run-ok-rcv (expected-xml stderr) (thin-dump "--transaction-id=5" "--data-block-size=128" "--nr-data-blocks=4096000" md1) - (damage-superblock md1) - (with-empty-metadata (md2) - (run-ok-rcv (_ stderr) (thin-repair "--transaction-id=5" "--data-block-size=128" "--nr-data-blocks=4096000" "-i" md1 "-o" md2) - (assert-eof stderr)) - (run-ok-rcv (repaired-xml stderr) (thin-dump md2) - (assert-eof stderr) - (assert-equal expected-xml repaired-xml)))))) - - (define-scenario (thin-repair superblock missing-transaction-id) - "--transaction-id is mandatory if the superblock is damaged" - (with-damaged-superblock (md1) - (with-empty-metadata (md2) - (run-fail-rcv (_ stderr) (thin-repair "--data-block-size=128" "--nr-data-blocks=4096000" "-i" md1 "-o" md2) - (assert-matches ".*transaction id.*" stderr))))) - - (define-scenario (thin-repair superblock missing-data-block-size) - "--data-block-size is mandatory if the superblock is damaged" - (with-damaged-superblock (md1) - (with-empty-metadata (md2) - (run-fail-rcv (_ stderr) (thin-repair "--transaction-id=5" "--nr-data-blocks=4096000" "-i" md1 "-o" md2) - (assert-matches ".*data block size.*" stderr))))) - - (define-scenario (thin-repair superblock missing-nr-data-blocks) - "--nr-data-blocks is mandatory if the superblock is damaged" - (with-damaged-superblock (md1) - (with-empty-metadata (md2) - (run-fail-rcv (_ stderr) (thin-repair "--transaction-id=5" "--data-block-size=128" "-i" md1 "-o" md2) - (assert-matches ".*nr data blocks.*" stderr))))) - - ;;;----------------------------------------------------------- ;;; thin_metadata_pack scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 23fa38d..bf24132 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -59,6 +59,17 @@ macro_rules! thin_rmap { }; } +#[macro_export] +macro_rules! thin_repair { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_repair", args).stdout_capture().stderr_capture() + } + }; +} + //------------------------------------------ pub struct TestDir { diff --git a/tests/thin_repair.rs b/tests/thin_repair.rs new file mode 100644 index 0000000..9bf9a33 --- /dev/null +++ b/tests/thin_repair.rs @@ -0,0 +1,158 @@ +use anyhow::Result; +use std::str::from_utf8; +use thinp::version::TOOLS_VERSION; + +mod common; +use common::*; + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = thin_repair!("-V").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = thin_repair!("--version").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +const USAGE: &str = "Usage: thin_repair [options] {device|file}\nOptions:\n {-h|--help}\n {-i|--input} \n {-o|--output} \n {--transaction-id} \n {--data-block-size} \n {--nr-data-blocks} \n {-V|--version}"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = thin_repair!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = thin_repair!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn dont_repair_xml() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let xml = mk_valid_xml(&mut td)?; + run_fail(thin_repair!("-i", &xml, "-o", &md))?; + Ok(()) +} + +#[test] +fn missing_input_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_repair!("-i", "no-such-file", "-o", &md))?; + assert!(superblock_all_zeroes(&md)?); + assert!(stderr.contains("Couldn't stat file")); + Ok(()) +} + +#[test] +fn garbage_input_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let md2 = mk_zeroed_md(&mut td)?; + run_fail(thin_repair!("-i", &md, "-o", &md2))?; + assert!(superblock_all_zeroes(&md2)?); + Ok(()) +} + +#[test] +fn missing_output_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + let stderr = run_fail(thin_repair!("-i", &md))?; + assert!(stderr.contains("No output file provided.")); + Ok(()) +} + +fn override_thing(flag: &str, val: &str, pattern: &str) -> Result<()> { + let mut td = TestDir::new()?; + let md1 = mk_valid_md(&mut td)?; + let md2 = mk_zeroed_md(&mut td)?; + let output = thin_repair!(flag, val, "-i", &md1, "-o", &md2).run()?; + assert_eq!(output.stderr.len(), 0); + let output = thin_dump!(&md2).run()?; + assert!(from_utf8(&output.stdout[0..])?.contains(pattern)); + Ok(()) +} + +#[test] +fn override_transaction_id() -> Result<()> { + override_thing("--transaction-id", "2345", "transaction=\"2345\"") +} + +#[test] +fn override_data_block_size() -> Result<()> { + override_thing("--data-block-size", "8192", "data_block_size=\"8192\"") +} + +#[test] +fn override_nr_data_blocks() -> Result<()> { + override_thing("--nr-data-blocks", "234500", "nr_data_blocks=\"234500\"") +} + +#[test] +fn superblock_succeeds() -> Result<()> { + let mut td = TestDir::new()?; + let md1 = mk_valid_md(&mut td)?; + let original = thin_dump!( + "--transaction-id=5", + "--data-block-size=128", + "--nr-data-blocks=4096000", + &md1 + ) + .run()?; + assert_eq!(original.stderr.len(), 0); + damage_superblock(&md1)?; + let md2 = mk_zeroed_md(&mut td)?; + thin_repair!( + "--transaction-id=5", + "--data-block-size=128", + "--nr-data-blocks=4096000", + "-i", + &md1, + "-o", + &md2 + ) + .run()?; + let repaired = thin_dump!(&md2).run()?; + assert_eq!(repaired.stderr.len(), 0); + assert_eq!(original.stdout, repaired.stdout); + Ok(()) +} + +fn missing_thing(flag1: &str, flag2: &str, pattern: &str) -> Result<()> +{ + let mut td = TestDir::new()?; + let md1 = mk_valid_md(&mut td)?; + damage_superblock(&md1)?; + let md2 = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_repair!(flag1, flag2, "-i", &md1, "-o", &md2))?; + assert!(stderr.contains(pattern)); + Ok(()) +} + +#[test] +fn missing_transaction_id() -> Result<()> { + missing_thing("--data-block-size=128", "--nr-data-blocks=4096000", "transaction id") +} + +#[test] +fn missing_data_block_size() -> Result<()> { + missing_thing("--transaction-id=5", "--nr-data-blocks=4096000", "data block size") +} + +#[test] +fn missing_nr_data_blocks() -> Result<()> { + missing_thing("--transaction-id=5", "--data-block-size=128", "nr data blocks") +} diff --git a/tests/thin_rmap.rs b/tests/thin_rmap.rs index e6255e6..e741c7f 100644 --- a/tests/thin_rmap.rs +++ b/tests/thin_rmap.rs @@ -47,10 +47,7 @@ fn rejects_bad_option() -> Result<()> { fn valid_region_format_should_pass() -> Result<()> { let mut td = TestDir::new()?; let md = mk_valid_md(&mut td)?; - let output = thin_rmap!("--region", "23..7890", &md).unchecked().run()?; - eprintln!("stdout: {:?}", output.stdout); - eprintln!("stderr: {:?}", output.stderr); - assert!(output.status.success()); + thin_rmap!("--region", "23..7890", &md).run()?; Ok(()) } From 1d44025584e30a88096ff3856f336b55e498fcd4 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 4 Aug 2020 12:11:36 +0100 Subject: [PATCH 25/52] [thin_check (rust)] Walk the top level and bottom level of the mapping tree separately --- src/pdata/btree.rs | 19 ++++++++ src/thin/check.rs | 118 +++++++++++++++++++++++---------------------- 2 files changed, 80 insertions(+), 57 deletions(-) diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 1069eb3..3b07c63 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -6,6 +6,8 @@ use std::sync::{Arc, Mutex}; use crate::block_manager::*; use crate::checksum; +// FIXME: check that keys are in ascending order between nodes. + //------------------------------------------ pub trait ValueType { @@ -186,6 +188,23 @@ impl BTreeWalker { r } + pub fn new_with_seen( + engine: Arc, + seen: Arc>, + ignore_non_fatal: bool, + ) -> BTreeWalker { + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), engine.get_nr_blocks() as usize); + } + + BTreeWalker { + engine, + seen, + ignore_non_fatal, + } + } + fn walk_nodes(&mut self, visitor: &mut NV, bs: &Vec) -> Result<()> where NV: NodeVisitor, diff --git a/src/thin/check.rs b/src/thin/check.rs index 3717a73..207b69f 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Result}; +use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; use std::collections::HashMap; use std::path::Path; @@ -12,6 +13,31 @@ use crate::thin::superblock::*; //------------------------------------------ +struct TopLevelVisitor<'a> { + roots: &'a mut HashMap, +} + +impl<'a> NodeVisitor for TopLevelVisitor<'a> { + fn visit(&mut self, w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf { + header: _h, + keys, + values, + } = node + { + for n in 0..keys.len() { + let k = keys[n]; + let root = values[n]; + self.roots.insert(k as u32, root); + } + } + + Ok(()) + } +} + +//------------------------------------------ + #[allow(dead_code)] struct BlockTime { block: u64, @@ -38,54 +64,6 @@ impl ValueType for BlockTime { } } -struct TopLevelVisitor {} - -impl NodeVisitor for TopLevelVisitor { - fn visit(&mut self, w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { - if let Node::Leaf { - header: _h, - keys, - values, - } = node - { - let mut blocks = Vec::new(); - let mut thin_ids = Vec::new(); - let seen = w.seen.lock().unwrap(); - for n in 0..keys.len() { - let b = values[n]; - if !seen[b as usize] { - thin_ids.push(keys[n]); - blocks.push(Block::new(b)); - } - } - drop(seen); - - w.engine.read_many(&mut blocks)?; - - // FIXME: with a thread pool we need to return errors another way. - let nr_workers = 4; - let pool = ThreadPool::new(nr_workers); - - let mut n = 0; - for b in blocks { - let thin_id = thin_ids[n]; - n += 1; - - let mut w = w.clone(); - pool.execute(move || { - let mut v = BottomLevelVisitor {}; - let result = w.walk_b(&mut v, &b).expect("walk failed"); // FIXME: return error - eprintln!("checked thin_dev {} -> {:?}", thin_id, result); - }); - } - - pool.join(); - } - - Ok(()) - } -} - struct BottomLevelVisitor {} impl NodeVisitor for BottomLevelVisitor { @@ -141,12 +119,17 @@ impl DeviceVisitor { impl NodeVisitor for DeviceVisitor { fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { - if let Node::Leaf {header: _h, keys, values} = node { - for n in 0..keys.len() { - let k = keys[n] as u32; - let v = values[n].clone(); - self.devs.insert(k, v.clone()); - } + if let Node::Leaf { + header: _h, + keys, + values, + } = node + { + for n in 0..keys.len() { + let k = keys[n] as u32; + let v = values[n].clone(); + self.devs.insert(k, v.clone()); + } } Ok(()) @@ -162,21 +145,42 @@ pub fn check(dev: &Path) -> Result<()> { let now = Instant::now(); let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; eprintln!("{:?}", sb); - + { let mut visitor = DeviceVisitor::new(); let mut w = BTreeWalker::new(engine.clone(), false); w.walk(&mut visitor, sb.details_root)?; println!("found {} devices", visitor.devs.len()); } - + + let mut roots = HashMap::new(); { - let mut visitor = TopLevelVisitor {}; + let mut visitor = TopLevelVisitor { roots: &mut roots }; let mut w = BTreeWalker::new(engine.clone(), false); let _result = w.walk(&mut visitor, sb.mapping_root)?; println!("read mapping tree in {} ms", now.elapsed().as_millis()); } + // FIXME: with a thread pool we need to return errors another way. + { + let nr_workers = 4; + let pool = ThreadPool::new(nr_workers); + let mut seen = Arc::new(Mutex::new(FixedBitSet::with_capacity( + engine.get_nr_blocks() as usize, + ))); + + for (thin_id, root) in roots { + let mut w = BTreeWalker::new_with_seen(engine.clone(), seen.clone(), false); + pool.execute(move || { + let mut v = BottomLevelVisitor {}; + let result = w.walk(&mut v, root).expect("walk failed"); // FIXME: return error + eprintln!("checked thin_dev {} -> {:?}", thin_id, result); + }); + } + + pool.join(); + } + Ok(()) } From 197e4ffbfdf674450a4a582fbbc7121d4679be19 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 5 Aug 2020 08:01:02 +0100 Subject: [PATCH 26/52] [thin_check (rust)] Rename ValueType trait to Unpack --- src/block_manager.rs | 3 ++- src/pdata/btree.rs | 18 +++++++++--------- src/thin/check.rs | 13 ++++++------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/block_manager.rs b/src/block_manager.rs index 5abef18..7aebbaf 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -141,7 +141,8 @@ impl Clone for AsyncIoEngine { fn clone(&self) -> AsyncIoEngine { let inner = self.inner.lock().unwrap(); eprintln!("in clone, queue_len = {}", inner.queue_len); - AsyncIoEngine {inner: Mutex::new(AsyncIoEngine_ { + AsyncIoEngine { + inner: Mutex::new(AsyncIoEngine_ { queue_len: inner.queue_len, ring: IoUring::new(inner.queue_len).expect("couldn't create uring"), nr_blocks: inner.nr_blocks, diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 3b07c63..00c8620 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -10,7 +10,7 @@ use crate::checksum; //------------------------------------------ -pub trait ValueType { +pub trait Unpack { // The size of the value when on disk. fn disk_size() -> u32; fn unpack(data: &[u8]) -> IResult<&[u8], Self> @@ -53,7 +53,7 @@ pub fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> { )) } -pub enum Node { +pub enum Node { Internal { header: NodeHeader, keys: Vec, @@ -79,7 +79,7 @@ pub fn to_any<'a, V>(r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> { } } -pub fn unpack_node( +pub fn unpack_node( data: &[u8], ignore_non_fatal: bool, is_root: bool, @@ -154,7 +154,7 @@ pub fn unpack_node( //------------------------------------------ -impl ValueType for u64 { +impl Unpack for u64 { fn disk_size() -> u32 { 8 } @@ -166,7 +166,7 @@ impl ValueType for u64 { //------------------------------------------ -pub trait NodeVisitor { +pub trait NodeVisitor { fn visit<'a>(&mut self, w: &BTreeWalker, b: &Block, node: &Node) -> Result<()>; } @@ -208,7 +208,7 @@ impl BTreeWalker { fn walk_nodes(&mut self, visitor: &mut NV, bs: &Vec) -> Result<()> where NV: NodeVisitor, - V: ValueType, + V: Unpack, { let mut blocks = Vec::new(); let seen = self.seen.lock().unwrap(); @@ -231,7 +231,7 @@ impl BTreeWalker { fn walk_node(&mut self, visitor: &mut NV, b: &Block, is_root: bool) -> Result<()> where NV: NodeVisitor, - V: ValueType, + V: Unpack, { let mut seen = self.seen.lock().unwrap(); seen.insert(b.loc as usize); @@ -260,7 +260,7 @@ impl BTreeWalker { pub fn walk_b(&mut self, visitor: &mut NV, root: &Block) -> Result<()> where NV: NodeVisitor, - V: ValueType, + V: Unpack, { self.walk_node(visitor, &root, true) } @@ -268,7 +268,7 @@ impl BTreeWalker { pub fn walk(&mut self, visitor: &mut NV, root: u64) -> Result<()> where NV: NodeVisitor, - V: ValueType, + V: Unpack, { let mut root = Block::new(root); self.engine.read(&mut root)?; diff --git a/src/thin/check.rs b/src/thin/check.rs index 207b69f..3611248 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -8,7 +8,7 @@ use std::time::Instant; use threadpool::ThreadPool; use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; -use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, ValueType}; +use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, Unpack}; use crate::thin::superblock::*; //------------------------------------------ @@ -18,7 +18,7 @@ struct TopLevelVisitor<'a> { } impl<'a> NodeVisitor for TopLevelVisitor<'a> { - fn visit(&mut self, w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { if let Node::Leaf { header: _h, keys, @@ -44,7 +44,7 @@ struct BlockTime { time: u32, } -impl ValueType for BlockTime { +impl Unpack for BlockTime { fn disk_size() -> u32 { 8 } @@ -82,7 +82,7 @@ struct DeviceDetail { snapshotted_time: u32, } -impl ValueType for DeviceDetail { +impl Unpack for DeviceDetail { fn disk_size() -> u32 { 24 } @@ -139,8 +139,7 @@ impl NodeVisitor for DeviceVisitor { //------------------------------------------ pub fn check(dev: &Path) -> Result<()> { - //let mut engine = SyncIoEngine::new(dev)?; - let mut engine = Arc::new(AsyncIoEngine::new(dev, 256)?); + let engine = Arc::new(AsyncIoEngine::new(dev, 256)?); let now = Instant::now(); let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; @@ -165,7 +164,7 @@ pub fn check(dev: &Path) -> Result<()> { { let nr_workers = 4; let pool = ThreadPool::new(nr_workers); - let mut seen = Arc::new(Mutex::new(FixedBitSet::with_capacity( + let seen = Arc::new(Mutex::new(FixedBitSet::with_capacity( engine.get_nr_blocks() as usize, ))); From 23568aaa11d05ddbe4c7dfc2fd53838e827bdf51 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 5 Aug 2020 08:24:52 +0100 Subject: [PATCH 27/52] [functional-tests] port thin_delta tests to rust. cargo test --- functional-tests/thin-functional-tests.scm | 44 -------------- tests/common/mod.rs | 11 ++++ tests/thin_delta.rs | 70 ++++++++++++++++++++++ 3 files changed, 81 insertions(+), 44 deletions(-) create mode 100644 tests/thin_delta.rs diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm index c921b87..920f0ff 100644 --- a/functional-tests/thin-functional-tests.scm +++ b/functional-tests/thin-functional-tests.scm @@ -67,50 +67,6 @@ ;; to run. (define (register-thin-tests) #t) - ;;;----------------------------------------------------------- - ;;; thin_delta scenarios - ;;;----------------------------------------------------------- - (define-scenario (thin-delta v) - "thin_delta accepts -V" - (run-ok-rcv (stdout _) (thin-delta "-V") - (assert-equal tools-version stdout))) - - (define-scenario (thin-delta version) - "thin_delta accepts --version" - (run-ok-rcv (stdout _) (thin-delta "--version") - (assert-equal tools-version stdout))) - - (define-scenario (thin-delta h) - "thin_delta accepts -h" - (run-ok-rcv (stdout _) (thin-delta "-h") - (assert-equal thin-delta-help stdout))) - - (define-scenario (thin-delta help) - "thin_delta accepts --help" - (run-ok-rcv (stdout _) (thin-delta "--help") - (assert-equal thin-delta-help stdout))) - - (define-scenario (thin-delta unrecognised-option) - "Unrecognised option should cause failure" - (with-valid-metadata (md) - (run-fail-rcv (stdout stderr) (thin-delta "--unleash-the-hedgehogs") - (assert-matches ".*thin_delta: unrecognized option '--unleash-the-hedgehogs" stderr)))) - - (define-scenario (thin-delta snap1-unspecified) - "Fails without --snap1 fails" - (run-fail-rcv (_ stderr) (thin-delta "--snap2 45 foo") - (assert-starts-with "--snap1 not specified." stderr))) - - (define-scenario (thin-delta snap2-unspecified) - "Fails without --snap2 fails" - (run-fail-rcv (_ stderr) (thin-delta "--snap1 45 foo") - (assert-starts-with "--snap2 not specified." stderr))) - - (define-scenario (thin-delta device-unspecified) - "Fails if no device given" - (run-fail-rcv (_ stderr) (thin-delta "--snap1 45 --snap2 46") - (assert-starts-with "No input device provided." stderr))) - ;;;----------------------------------------------------------- ;;; thin_metadata_pack scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/mod.rs b/tests/common/mod.rs index bf24132..0c35c95 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -70,6 +70,17 @@ macro_rules! thin_repair { }; } +#[macro_export] +macro_rules! thin_delta { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_delta", args).stdout_capture().stderr_capture() + } + }; +} + //------------------------------------------ pub struct TestDir { diff --git a/tests/thin_delta.rs b/tests/thin_delta.rs new file mode 100644 index 0000000..7c1442b --- /dev/null +++ b/tests/thin_delta.rs @@ -0,0 +1,70 @@ +use anyhow::Result; +use thinp::version::TOOLS_VERSION; + +mod common; +use common::*; + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = thin_delta!("-V").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = thin_delta!("--version").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +const USAGE: &str = "Usage: thin_delta [options] \nOptions:\n {--thin1, --snap1}\n {--thin2, --snap2}\n {-m, --metadata-snap} [block#]\n {--verbose}\n {-h|--help}\n {-V|--version}"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = thin_delta!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = thin_delta!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn rejects_bad_option() -> Result<()> { + let stderr = run_fail(thin_delta!("--hedgehogs-only"))?; + assert!(stderr.contains("unrecognized option \'--hedgehogs-only\'")); + Ok(()) +} + +#[test] +fn snap1_unspecified() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + let stderr = run_fail(thin_delta!("--snap2", "45", &md))?; + assert!(stderr.contains("--snap1 not specified")); + Ok(()) +} + +#[test] +fn snap2_unspecified() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + let stderr = run_fail(thin_delta!("--snap1", "45", &md))?; + assert!(stderr.contains("--snap2 not specified")); + Ok(()) +} + +#[test] +fn dev_unspecified() -> Result<()> { + let stderr = run_fail(thin_delta!("--snap1", "45", "--snap2", "46"))?; + assert!(stderr.contains("No input device provided")); + Ok(()) +} + From d2678fdf27204a47d29de20cfe0a7b2f70f40138 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 5 Aug 2020 09:09:18 +0100 Subject: [PATCH 28/52] [functional-tests] port thin_metadata_pack tests to Rust. cargo test --- functional-tests/thin-functional-tests.scm | 43 ------------ tests/common/mod.rs | 22 +++++++ tests/thin_metadata_pack.rs | 76 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 43 deletions(-) create mode 100644 tests/thin_metadata_pack.rs diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm index 920f0ff..0f2fd09 100644 --- a/functional-tests/thin-functional-tests.scm +++ b/functional-tests/thin-functional-tests.scm @@ -67,49 +67,6 @@ ;; to run. (define (register-thin-tests) #t) - ;;;----------------------------------------------------------- - ;;; thin_metadata_pack scenarios - ;;;----------------------------------------------------------- - - (define-scenario (thin-metadata-pack version) - "accepts --version" - (run-ok-rcv (stdout _) (thin-metadata-pack "--version") - (assert-equal "thin_metadata_pack 0.9.0-rc2" stdout))) - - (define-scenario (thin-metadata-pack h) - "accepts -h" - (run-ok-rcv (stdout _) (thin-metadata-pack "-h") - (assert-equal thin-metadata-pack-help stdout))) - - (define-scenario (thin-metadata-pack help) - "accepts --help" - (run-ok-rcv (stdout _) (thin-metadata-pack "--help") - (assert-equal thin-metadata-pack-help stdout))) - - (define-scenario (thin-metadata-pack unrecognised-option) - "Unrecognised option should cause failure" - (with-valid-metadata (md) - (run-fail-rcv (stdout stderr) (thin-metadata-pack "--unleash-the-hedgehogs") - (assert-starts-with "error: Found argument '--unleash-the-hedgehogs'" stderr)))) - - (define-scenario (thin-metadata-pack missing-input-file) - "the input file wasn't specified" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-metadata-pack "-o " md) - (assert-starts-with "error: The following required arguments were not provided:\n -i " stderr)))) - - (define-scenario (thin-metadata-pack no-such-input-file) - "the input file can't be found" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-metadata-pack "-i no-such-file -o" md) - (assert-starts-with "Couldn't find input file" stderr)))) - - (define-scenario (thin-metadata-pack missing-output-file) - "the output file wasn't specified" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-metadata-pack "-i" md) - (assert-starts-with "error: The following required arguments were not provided:\n -o " stderr)))) - ;;;----------------------------------------------------------- ;;; thin_metadata_unpack scenarios ;;;----------------------------------------------------------- diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 0c35c95..21c5526 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -81,6 +81,28 @@ macro_rules! thin_delta { }; } +#[macro_export] +macro_rules! thin_metadata_pack { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_metadata_pack", args).stdout_capture().stderr_capture() + } + }; +} + +#[macro_export] +macro_rules! thin_metadata_unpack { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/thin_metadata_unpack", args).stdout_capture().stderr_capture() + } + }; +} + //------------------------------------------ pub struct TestDir { diff --git a/tests/thin_metadata_pack.rs b/tests/thin_metadata_pack.rs new file mode 100644 index 0000000..ee9926b --- /dev/null +++ b/tests/thin_metadata_pack.rs @@ -0,0 +1,76 @@ +use anyhow::Result; +use thinp::version::TOOLS_VERSION; + +mod common; +use common::*; + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = thin_metadata_pack!("-V").read()?; + assert!(stdout.contains(TOOLS_VERSION)); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = thin_metadata_pack!("--version").read()?; + assert!(stdout.contains(TOOLS_VERSION)); + Ok(()) +} + +const USAGE: &str = "thin_metadata_pack 0.9.0-rc2\nProduces a compressed file of thin metadata. Only packs metadata blocks that are actually used.\n\nUSAGE:\n thin_metadata_pack -i -o \n\nFLAGS:\n -h, --help Prints help information\n -V, --version Prints version information\n\nOPTIONS:\n -i Specify thinp metadata binary device/file\n -o Specify packed output file"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = thin_metadata_pack!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = thin_metadata_pack!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn rejects_bad_option() -> Result<()> { + let stderr = run_fail(thin_metadata_pack!("--hedgehogs-only"))?; + assert!(stderr.contains("Found argument \'--hedgehogs-only\'")); + Ok(()) +} + +#[test] +fn missing_input_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_metadata_pack!("-o", &md))?; + assert!( + stderr.contains("error: The following required arguments were not provided:\n -i ") + ); + Ok(()) +} + +#[test] +fn no_such_input_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_metadata_pack!("-i", "no-such-file", "-o", &md))?; + assert!(stderr.contains("Couldn't find input file")); + Ok(()) +} + +#[test] +fn missing_output_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_metadata_pack!("-i", &md))?; + assert!(stderr + .contains("error: The following required arguments were not provided:\n -o ")); + Ok(()) +} + +//------------------------------------------ From 904d9b0c84f02b9bd5b4b98c89a566fa3a9b88d4 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 5 Aug 2020 10:00:44 +0100 Subject: [PATCH 29/52] [functional-tests] port thin_metadata_unpack tests to Rust. cargo test --- functional-tests/run-tests.scm | 4 +- functional-tests/test-runner.scm | 3 +- functional-tests/thin-functional-tests.scm | 132 --------------------- tests/thin_metadata_unpack.rs | 99 ++++++++++++++++ 4 files changed, 101 insertions(+), 137 deletions(-) delete mode 100644 functional-tests/thin-functional-tests.scm create mode 100644 tests/thin_metadata_unpack.rs diff --git a/functional-tests/run-tests.scm b/functional-tests/run-tests.scm index 805a3a1..a6cfa11 100644 --- a/functional-tests/run-tests.scm +++ b/functional-tests/run-tests.scm @@ -1,10 +1,8 @@ (import (rnrs) (test-runner) (cache-functional-tests) - (era-functional-tests) - (thin-functional-tests)) + (era-functional-tests)) -(register-thin-tests) (register-cache-tests) (register-era-tests) diff --git a/functional-tests/test-runner.scm b/functional-tests/test-runner.scm index b21f5f9..bad24de 100644 --- a/functional-tests/test-runner.scm +++ b/functional-tests/test-runner.scm @@ -14,8 +14,7 @@ (only (srfi s1 lists) break) (regex) (srfi s8 receive) - (temp-file) - (thin-functional-tests)) + (temp-file)) ;;------------------------------------------------ diff --git a/functional-tests/thin-functional-tests.scm b/functional-tests/thin-functional-tests.scm deleted file mode 100644 index 0f2fd09..0000000 --- a/functional-tests/thin-functional-tests.scm +++ /dev/null @@ -1,132 +0,0 @@ -(library - (thin-functional-tests) - - (export register-thin-tests) - - (import - (chezscheme) - (bcache block-manager) - (disk-units) - (fmt fmt) - (functional-tests) - (process) - (scenario-string-constants) - (temp-file) - (thin xml) - (srfi s8 receive)) - - (define-tool thin-check) - (define-tool thin-delta) - (define-tool thin-dump) - (define-tool thin-restore) - (define-tool thin-rmap) - (define-tool thin-repair) - (define-tool thin-metadata-pack) - (define-tool thin-metadata-unpack) - - (define-syntax with-thin-xml - (syntax-rules () - ((_ (v) b1 b2 ...) - (with-temp-file-containing ((v "thin.xml" (fmt #f (generate-xml 10 1000)))) - b1 b2 ...)))) - - (define-syntax with-valid-metadata - (syntax-rules () - ((_ (md) b1 b2 ...) - (with-temp-file-sized ((md "thin.bin" (meg 4))) - (with-thin-xml (xml) - (run-ok (thin-restore "-i" xml "-o" md)) - b1 b2 ...))))) - - ;;; It would be nice if the metadata was at least similar to valid data. - ;;; Here I'm just using the start of the ls binary as 'random' data. - (define-syntax with-corrupt-metadata - (syntax-rules () - ((_ (md) b1 b2 ...) - (with-temp-file-sized ((md "thin.bin" (meg 4))) - (system (fmt #f "dd if=/usr/bin/ls of=" md " bs=4096 > /dev/null 2>&1")) - b1 b2 ...)))) - - (define-syntax with-empty-metadata - (syntax-rules () - ((_ (md) b1 b2 ...) - (with-temp-file-sized ((md "thin.bin" (meg 4))) - b1 b2 ...)))) - - (define (damage-superblock md) - (system (string-append "dd if=/dev/zero of=" md " bs=4K count=1 conv=notrunc > /dev/null 2>&1"))) - - (define-syntax with-damaged-superblock - (syntax-rules () - ((_ (md) b1 b2 ...) - (with-valid-metadata (md) - (damage-superblock md) - b1 b2 ...)))) - - ;; We have to export something that forces all the initialisation expressions - ;; to run. - (define (register-thin-tests) #t) - - ;;;----------------------------------------------------------- - ;;; thin_metadata_unpack scenarios - ;;;----------------------------------------------------------- - (define-scenario (thin-metadata-unpack version) - "accepts --version" - (run-ok-rcv (stdout _) (thin-metadata-unpack "--version") - (assert-equal "thin_metadata_unpack 0.9.0-rc2" stdout))) - - (define-scenario (thin-metadata-unpack h) - "accepts -h" - (run-ok-rcv (stdout _) (thin-metadata-unpack "-h") - (assert-equal thin-metadata-unpack-help stdout))) - - (define-scenario (thin-metadata-unpack help) - "accepts --help" - (run-ok-rcv (stdout _) (thin-metadata-unpack "--help") - (assert-equal thin-metadata-unpack-help stdout))) - - (define-scenario (thin-metadata-unpack unrecognised-option) - "Unrecognised option should cause failure" - (with-valid-metadata (md) - (run-fail-rcv (stdout stderr) (thin-metadata-unpack "--unleash-the-hedgehogs") - (assert-starts-with "error: Found argument '--unleash-the-hedgehogs'" stderr)))) - - (define-scenario (thin-metadata-unpack missing-input-file) - "the input file wasn't specified" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-metadata-unpack "-o " md) - (assert-starts-with "error: The following required arguments were not provided:\n -i " stderr)))) - - (define-scenario (thin-metadata-unpack no-such-input-file) - "the input file can't be found" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-metadata-unpack "-i no-such-file -o" md) - (assert-starts-with "Couldn't find input file" stderr)))) - - (define-scenario (thin-metadata-unpack missing-output-file) - "the output file wasn't specified" - (with-empty-metadata (md) - (run-fail-rcv (_ stderr) (thin-metadata-unpack "-i" md) - (assert-starts-with "error: The following required arguments were not provided:\n -o " stderr)))) - - (define-scenario (thin-metadata-unpack garbage-input-file) - "the input file is just zeroes" - (with-empty-metadata (bad-pack) - (run-fail-rcv (_ stderr) (thin-metadata-unpack "-i " bad-pack "-o junk") - (assert-starts-with "Not a pack file." stderr)))) - - ;;;----------------------------------------------------------- - ;;; thin_metadata_pack/unpack end to end scenario - ;;;-----------------------------------------------------------) - (define-scenario (thin-metadata-pack end-to-end) - "pack -> unpack recovers metadata" - (let ((pack-file "md.pack")) - (with-valid-metadata (md-in) - (with-empty-metadata (md-out) - (run-ok (thin-metadata-pack "-i" md-in "-o" pack-file)) - (run-ok (thin-metadata-unpack "-i" pack-file "-o" md-out)) - (run-ok-rcv (dump1 _) (thin-dump md-in) - (run-ok-rcv (dump2 _) (thin-dump md-out) - (assert-equal dump1 dump2))))))) - -) diff --git a/tests/thin_metadata_unpack.rs b/tests/thin_metadata_unpack.rs new file mode 100644 index 0000000..057e147 --- /dev/null +++ b/tests/thin_metadata_unpack.rs @@ -0,0 +1,99 @@ +use anyhow::Result; +use thinp::version::TOOLS_VERSION; + +mod common; +use common::*; + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = thin_metadata_unpack!("-V").read()?; + assert!(stdout.contains(TOOLS_VERSION)); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = thin_metadata_unpack!("--version").read()?; + assert!(stdout.contains(TOOLS_VERSION)); + Ok(()) +} + +const USAGE: &str = "thin_metadata_unpack 0.9.0-rc2\nUnpack a compressed file of thin metadata.\n\nUSAGE:\n thin_metadata_unpack -i -o \n\nFLAGS:\n -h, --help Prints help information\n -V, --version Prints version information\n\nOPTIONS:\n -i Specify thinp metadata binary device/file\n -o Specify packed output file"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = thin_metadata_unpack!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = thin_metadata_unpack!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn rejects_bad_option() -> Result<()> { + let stderr = run_fail(thin_metadata_unpack!("--hedgehogs-only"))?; + assert!(stderr.contains("Found argument \'--hedgehogs-only\'")); + Ok(()) +} + +#[test] +fn missing_input_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_metadata_unpack!("-o", &md))?; + assert!( + stderr.contains("error: The following required arguments were not provided:\n -i ") + ); + Ok(()) +} + +#[test] +fn no_such_input_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_metadata_unpack!("-i", "no-such-file", "-o", &md))?; + assert!(stderr.contains("Couldn't find input file")); + Ok(()) +} + +#[test] +fn missing_output_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_metadata_unpack!("-i", &md))?; + assert!(stderr + .contains("error: The following required arguments were not provided:\n -o ")); + Ok(()) +} + +#[test] +fn garbage_input_file() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let stderr = run_fail(thin_metadata_unpack!("-i", &md, "-o", "junk"))?; + assert!(stderr.contains("Not a pack file.")); + Ok(()) +} + +#[test] +fn end_to_end() -> Result<()> { + let mut td = TestDir::new()?; + let md_in = mk_valid_md(&mut td)?; + let md_out = mk_zeroed_md(&mut td)?; + thin_metadata_pack!("-i", &md_in, "-o", "meta.pack").run()?; + thin_metadata_unpack!("-i", "meta.pack", "-o", &md_out).run()?; + + let dump1 = thin_dump!(&md_in).read()?; + let dump2 = thin_dump!(&md_out).read()?; + assert_eq!(dump1, dump2); + Ok(()) +} + +//------------------------------------------ From 4a0582bb5db214620a369ecf3b0cac65f3e6a755 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Thu, 6 Aug 2020 07:51:48 +0100 Subject: [PATCH 30/52] [thin_check (rust)] start decoding the space maps. --- src/block_manager.rs | 2 +- src/pdata/btree.rs | 19 ++++++ src/pdata/mod.rs | 1 + src/pdata/space_map.rs | 135 +++++++++++++++++++++++++++++++++++++++++ src/shrink/toplevel.rs | 1 + src/thin/check.rs | 106 +++++++++++++++++++++++++++++++- src/thin/superblock.rs | 12 ++-- 7 files changed, 266 insertions(+), 10 deletions(-) create mode 100644 src/pdata/space_map.rs diff --git a/src/block_manager.rs b/src/block_manager.rs index 7aebbaf..1d5e061 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -18,7 +18,7 @@ const ALIGN: usize = 4096; #[derive(Debug)] pub struct Block { pub loc: u64, - data: *mut u8, + pub data: *mut u8, } impl Block { diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 00c8620..81802df 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -18,6 +18,15 @@ pub trait Unpack { Self: std::marker::Sized; } +pub fn unpack(data: &[u8]) -> Result { + match U::unpack(data) { + Err(_e) => { + Err(anyhow!("couldn't parse SMRoot")) + }, + Ok((_i, v)) => Ok(v), + } +} + const NODE_HEADER_SIZE: usize = 32; pub struct NodeHeader { @@ -164,6 +173,16 @@ impl Unpack for u64 { } } +impl Unpack for u32 { + fn disk_size() -> u32 { + 4 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], u32> { + le_u32(i) + } +} + //------------------------------------------ pub trait NodeVisitor { diff --git a/src/pdata/mod.rs b/src/pdata/mod.rs index 83204ba..3e36813 100644 --- a/src/pdata/mod.rs +++ b/src/pdata/mod.rs @@ -1,2 +1,3 @@ pub mod btree; +pub mod space_map; diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs new file mode 100644 index 0000000..649964f --- /dev/null +++ b/src/pdata/space_map.rs @@ -0,0 +1,135 @@ +use anyhow::{anyhow, Result}; +use nom::{number::complete::*, IResult}; + +use crate::block_manager::*; +use crate::pdata::btree::Unpack; + +//------------------------------------------ + +#[derive(Debug)] +pub struct SMRoot { + pub nr_blocks: u64, + pub nr_allocated: u64, + pub bitmap_root: u64, + pub ref_count_root: u64, +} + +pub fn unpack_root(data: &[u8]) -> Result { + match SMRoot::unpack(data) { + Err(_e) => { + Err(anyhow!("couldn't parse SMRoot")) + }, + Ok((_i, v)) => Ok(v), + } +} + +impl Unpack for SMRoot { + fn disk_size() -> u32 { + 32 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], SMRoot> { + let (i, nr_blocks) = le_u64(data)?; + let (i, nr_allocated) = le_u64(i)?; + let (i, bitmap_root) = le_u64(i)?; + let (i, ref_count_root) = le_u64(i)?; + + Ok ((i, SMRoot { + nr_blocks, + nr_allocated, + bitmap_root, + ref_count_root, + })) + } +} + +//------------------------------------------ + +#[derive(Clone, Debug)] +pub struct IndexEntry { + pub blocknr: u64, + pub nr_free: u32, + pub none_free_before: u32, +} + +impl Unpack for IndexEntry { + fn disk_size() -> u32 { + 16 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], Self> { + let (i, blocknr) = le_u64(data)?; + let (i, nr_free) = le_u32(i)?; + let (i, none_free_before) = le_u32(i)?; + + Ok((i, IndexEntry {blocknr, nr_free, none_free_before})) + } +} + +//------------------------------------------ + +#[derive(Debug)] +pub struct BitmapHeader { + pub csum: u32, + pub not_used: u32, + pub blocknr: u64, +} + +impl Unpack for BitmapHeader { + fn disk_size() -> u32 { + 16 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], Self> { + let (i, csum) = le_u32(data)?; + let (i, not_used) = le_u32(i)?; + let (i, blocknr) = le_u64(i)?; + + Ok((i, BitmapHeader {csum, not_used, blocknr} )) + } +} + +#[derive(Debug)] +pub enum BitmapEntry { + Small(u8), + Overflow, +} + +#[derive(Debug)] +pub struct Bitmap { + pub header: BitmapHeader, + pub entries: Vec, +} + +impl Unpack for Bitmap { + fn disk_size() -> u32 { + BLOCK_SIZE as u32 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], Self> { + let (mut i, header) = BitmapHeader::unpack(data)?; + + let mut entries = Vec::new(); + let nr_words = (BLOCK_SIZE - BitmapHeader::disk_size() as usize) / 8; + for _w in 0..nr_words { + let (tmp, mut word) = le_u64(i)?; + + for _b in 0..32 { + let val = word & 0x3; + word = word >> 2; + + if val < 3 { + entries.push(BitmapEntry::Small(val as u8)); + } else { + entries.push(BitmapEntry::Overflow); + } + } + + i = tmp; + } + + Ok((i, Bitmap {header, entries})) + } +} + +//------------------------------------------ diff --git a/src/shrink/toplevel.rs b/src/shrink/toplevel.rs index 38ffe82..1a3d8a9 100644 --- a/src/shrink/toplevel.rs +++ b/src/shrink/toplevel.rs @@ -133,6 +133,7 @@ impl xml::MetadataVisitor for Pass2 { } //--------------------------------------- + type BlockRange = std::ops::Range; fn bits_to_ranges(bits: &FixedBitSet) -> Vec { diff --git a/src/thin/check.rs b/src/thin/check.rs index 3611248..6461ad8 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -8,8 +8,10 @@ use std::time::Instant; use threadpool::ThreadPool; use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; -use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, Unpack}; +use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, Unpack, unpack}; +use crate::pdata::space_map::*; use crate::thin::superblock::*; +use crate::checksum; //------------------------------------------ @@ -128,7 +130,7 @@ impl NodeVisitor for DeviceVisitor { for n in 0..keys.len() { let k = keys[n] as u32; let v = values[n].clone(); - self.devs.insert(k, v.clone()); + self.devs.insert(k, v); } } @@ -138,6 +140,63 @@ impl NodeVisitor for DeviceVisitor { //------------------------------------------ +struct IndexVisitor { + entries: Vec, +} + +impl NodeVisitor for IndexVisitor { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf { + header: _h, + keys, + values, + } = node { + for n in 0..keys.len() { + // FIXME: check keys are in incremental order + let v = values[n].clone(); + self.entries.push(v); + } + } + + Ok(()) + } +} + +//------------------------------------------ + +// FIXME: move to btree +struct ValueCollector { + values: Vec<(u64, V)>, +} + +impl ValueCollector { + fn new() -> ValueCollector { + ValueCollector { + values: Vec::new(), + } + } +} + +impl NodeVisitor for ValueCollector { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf { + header: _h, + keys, + values, + } = node { + for n in 0..keys.len() { + let k = keys[n]; + let v = values[n].clone(); + self.values.push((k, v)); + } + } + + Ok(()) + } +} + +//------------------------------------------ + pub fn check(dev: &Path) -> Result<()> { let engine = Arc::new(AsyncIoEngine::new(dev, 256)?); @@ -145,6 +204,7 @@ pub fn check(dev: &Path) -> Result<()> { let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; eprintln!("{:?}", sb); + // device details { let mut visitor = DeviceVisitor::new(); let mut w = BTreeWalker::new(engine.clone(), false); @@ -152,6 +212,8 @@ pub fn check(dev: &Path) -> Result<()> { println!("found {} devices", visitor.devs.len()); } +/* + // mapping top level let mut roots = HashMap::new(); { let mut visitor = TopLevelVisitor { roots: &mut roots }; @@ -160,8 +222,9 @@ pub fn check(dev: &Path) -> Result<()> { println!("read mapping tree in {} ms", now.elapsed().as_millis()); } - // FIXME: with a thread pool we need to return errors another way. + // mapping bottom level { + // FIXME: with a thread pool we need to return errors another way. let nr_workers = 4; let pool = ThreadPool::new(nr_workers); let seen = Arc::new(Mutex::new(FixedBitSet::with_capacity( @@ -179,6 +242,43 @@ pub fn check(dev: &Path) -> Result<()> { pool.join(); } + */ + + // data space map + { + let root = unpack::(&sb.data_sm_root[0..])?; + eprintln!("data root: {:?}", root); + + // overflow btree + let mut overflow: HashMap = HashMap::new(); + { + let mut v: ValueCollector = ValueCollector::new(); + let mut w = BTreeWalker::new(engine.clone(), false); + w.walk(&mut v, root.ref_count_root)?; + + for (k, v) in v.values { + overflow.insert(k, v); + } + } + eprintln!("{} overflow entries", overflow.len()); + + // Bitmaps + let mut v = IndexVisitor {entries: Vec::new()}; + let mut w = BTreeWalker::new(engine.clone(), false); + let _result = w.walk(&mut v, root.bitmap_root); + eprintln!("{} index entries", v.entries.len()); + + for i in v.entries { + let mut b = Block::new(i.blocknr); + engine.read(&mut b)?; + + if checksum::metadata_block_type(&b.get_data()) != checksum::BT::BITMAP { + return Err(anyhow!("Index entry points to block ({}) that isn't a bitmap", b.loc)); + } + + let bitmap = unpack::(b.get_data())?; + } + } Ok(()) } diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index 5001eb6..8c1e738 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -14,8 +14,8 @@ pub struct Superblock { pub time: u32, pub transaction_id: u64, pub metadata_snap: u64, - //data_sm_root: [u8; SPACE_MAP_ROOT_SIZE], - //metadata_sm_root: [u8; SPACE_MAP_ROOT_SIZE], + pub data_sm_root: Vec, + pub metadata_sm_root: Vec, pub mapping_root: u64, pub details_root: u64, pub data_block_size: u32, @@ -59,8 +59,8 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { let (i, time) = le_u32(i)?; let (i, transaction_id) = le_u64(i)?; let (i, metadata_snap) = le_u64(i)?; - let (i, _data_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; - let (i, _metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; + let (i, data_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; + let (i, metadata_sm_root) = take(SPACE_MAP_ROOT_SIZE)(i)?; let (i, mapping_root) = le_u64(i)?; let (i, details_root) = le_u64(i)?; let (i, data_block_size) = le_u32(i)?; @@ -76,8 +76,8 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { time, transaction_id, metadata_snap, - //data_sm_root, - //metadata_sm_root, + data_sm_root: data_sm_root.to_vec(), + metadata_sm_root: metadata_sm_root.to_vec(), mapping_root, details_root, data_block_size, From fa4ea3e2d96eb64970ecddd12453d6cd3af2ec14 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 7 Aug 2020 14:30:00 +0100 Subject: [PATCH 31/52] [functional-tests] port some of the cache_check tests to Rust --- Cargo.lock | 7 + Cargo.toml | 1 + functional-tests/cache-functional-tests.scm | 99 ---------- src/cache/mod.rs | 1 + src/cache/xml.rs | 175 ++++++++++++++++++ src/lib.rs | 1 + src/thin/check.rs | 4 +- tests/cache_check.rs | 129 +++++++++++++ tests/common/cache_xml_generator.rs | 94 ++++++++++ tests/common/mod.rs | 40 ++-- tests/common/test_dir.rs | 27 +++ ...xml_generator.rs => thin_xml_generator.rs} | 0 tests/thin_check.rs | 3 +- tests/thin_delta.rs | 1 + tests/thin_dump.rs | 1 + tests/thin_metadata_pack.rs | 1 + tests/thin_metadata_unpack.rs | 1 + tests/thin_repair.rs | 1 + tests/thin_restore.rs | 1 + tests/thin_rmap.rs | 1 + tests/thin_shrink.rs | 4 +- 21 files changed, 464 insertions(+), 128 deletions(-) create mode 100644 src/cache/mod.rs create mode 100644 src/cache/xml.rs create mode 100644 tests/cache_check.rs create mode 100644 tests/common/cache_xml_generator.rs create mode 100644 tests/common/test_dir.rs rename tests/common/{xml_generator.rs => thin_xml_generator.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index 9ec72e0..a2678fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,6 +53,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + [[package]] name = "bitflags" version = "1.2.1" @@ -638,6 +644,7 @@ name = "thinp" version = "0.1.0" dependencies = [ "anyhow", + "base64", "byteorder", "clap", "crc32c", diff --git a/Cargo.toml b/Cargo.toml index dbb8470..63de8df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ license = "GPL3" [dependencies] anyhow = "1.0" +base64 = "0.12" byteorder = "1.3" clap = "2.33" crc32c = "0.4" diff --git a/functional-tests/cache-functional-tests.scm b/functional-tests/cache-functional-tests.scm index 9b3a203..13dcaee 100644 --- a/functional-tests/cache-functional-tests.scm +++ b/functional-tests/cache-functional-tests.scm @@ -49,105 +49,6 @@ ;; to run. (define (register-cache-tests) #t) - ;;;----------------------------------------------------------- - ;;; cache_check scenarios - ;;;----------------------------------------------------------- - - (define-scenario (cache-check v) - "cache_check -V" - (run-ok-rcv (stdout _) (cache-check "-V") - (assert-equal tools-version stdout))) - - (define-scenario (cache-check version) - "cache_check --version" - (run-ok-rcv (stdout _) (cache-check "--version") - (assert-equal tools-version stdout))) - - (define-scenario (cache-check h) - "cache_check -h" - (run-ok-rcv (stdout _) (cache-check "-h") - (assert-equal cache-check-help stdout))) - - (define-scenario (cache-check help) - "cache_check --help" - (run-ok-rcv (stdout _) (cache-check "--help") - (assert-equal cache-check-help stdout))) - - (define-scenario (cache-check must-specify-metadata) - "Metadata file must be specified" - (run-fail-rcv (_ stderr) (cache-check) - (assert-equal - (string-append "No input file provided.\n" - cache-check-help) - stderr))) - - (define-scenario (cache-check no-such-metadata) - "Metadata file doesn't exist." - (let ((bad-path "/arbitrary/filename")) - (run-fail-rcv (_ stderr) (cache-check bad-path) - (assert-starts-with - (string-append bad-path ": No such file or directory") - stderr)))) - - (define-scenario (cache-check metadata-file-cannot-be-a-directory) - "Metadata file must not be a directory" - (let ((bad-path "/tmp")) - (run-fail-rcv (_ stderr) (cache-check bad-path) - (assert-starts-with - (string-append bad-path ": Not a block device or regular file") - stderr)))) - - (define-scenario (cache-check unreadable-metadata) - "Metadata file exists, but is unreadable." - (with-valid-metadata (md) - (run-ok "chmod" "-r" md) - (run-fail-rcv (_ stderr) (cache-check md) - (assert-starts-with "syscall 'open' failed: Permission denied" stderr)))) - - (define-scenario (cache-check fails-with-corrupt-metadata) - "Fail with corrupt superblock" - (with-corrupt-metadata (md) - (run-fail (cache-check md)))) - - (define-scenario (cache-check failing-q) - "Fail quietly with -q" - (with-corrupt-metadata (md) - (run-fail-rcv (stdout stderr) (cache-check "-q" md) - (assert-eof stdout) - (assert-eof stderr)))) - - (define-scenario (cache-check failing-quiet) - "Fail quietly with --quiet" - (with-corrupt-metadata (md) - (run-fail-rcv (stdout stderr) (cache-check "--quiet" md) - (assert-eof stdout) - (assert-eof stderr)))) - - (define-scenario (cache-check valid-metadata-passes) - "A valid metadata area passes" - (with-valid-metadata (md) - (run-ok (cache-check md)))) - - (define-scenario (cache-check bad-metadata-version) - "Invalid metadata version fails" - (with-cache-xml (xml) - (with-empty-metadata (md) - (cache-restore "-i" xml "-o" md "--debug-override-metadata-version" "12345") - (run-fail (cache-check md))))) - - (define-scenario (cache-check tiny-metadata) - "Prints helpful message in case tiny metadata given" - (with-temp-file-sized ((md "cache.bin" 1024)) - (run-fail-rcv (_ stderr) (cache-check md) - (assert-starts-with "Metadata device/file too small. Is this binary metadata?" stderr)))) - - (define-scenario (cache-check spot-accidental-xml-data) - "Prints helpful message if XML metadata given" - (with-cache-xml (xml) - (system (fmt #f "man bash >> " xml)) - (run-fail-rcv (_ stderr) (cache-check xml) - (assert-matches ".*This looks like XML. cache_check only checks the binary metadata format." stderr)))) - ;;;----------------------------------------------------------- ;;; cache_restore scenarios ;;;----------------------------------------------------------- diff --git a/src/cache/mod.rs b/src/cache/mod.rs new file mode 100644 index 0000000..2910ec6 --- /dev/null +++ b/src/cache/mod.rs @@ -0,0 +1 @@ +pub mod xml; diff --git a/src/cache/xml.rs b/src/cache/xml.rs new file mode 100644 index 0000000..d39b057 --- /dev/null +++ b/src/cache/xml.rs @@ -0,0 +1,175 @@ +use anyhow::Result; +use base64::encode; +use std::{borrow::Cow, fmt::Display, io::Write}; + +use quick_xml::events::attributes::Attribute; +use quick_xml::events::{BytesEnd, BytesStart, Event}; +use quick_xml::Writer; + +//--------------------------------------- + +#[derive(Clone)] +pub struct Superblock { + pub uuid: String, + pub block_size: u64, + pub nr_cache_blocks: u64, + pub policy: String, + pub hint_width: u32, +} + +#[derive(Clone)] +pub struct Map { + pub cblock: u64, + pub oblock: u64, + pub dirty: bool, +} + +#[derive(Clone)] +pub struct Hint { + pub cblock: u64, + pub data: Vec, +} + +#[derive(Clone)] +pub struct Discard { + pub begin: u64, + pub end: u64, +} + +#[derive(Clone)] +pub enum Visit { + Continue, + Stop, +} + +pub trait MetadataVisitor { + fn superblock_b(&mut self, sb: &Superblock) -> Result; + fn superblock_e(&mut self) -> Result; + + fn mappings_b(&mut self) -> Result; + fn mappings_e(&mut self) -> Result; + fn mapping(&mut self, m: &Map) -> Result; + + fn hints_b(&mut self) -> Result; + fn hints_e(&mut self) -> Result; + fn hint(&mut self, h: &Hint) -> Result; + + fn discards_b(&mut self) -> Result; + fn discards_e(&mut self) -> Result; + fn discard(&mut self, d: &Discard) -> Result; + + fn eof(&mut self) -> Result; +} + +pub struct XmlWriter { + w: Writer, +} + +impl XmlWriter { + pub fn new(w: W) -> XmlWriter { + XmlWriter { + w: Writer::new_with_indent(w, 0x20, 2), + } + } +} + +fn mk_attr_<'a, T: Display>(n: T) -> Cow<'a, [u8]> { + let str = format!("{}", n); + Cow::Owned(str.into_bytes()) +} + +fn mk_attr(key: &[u8], value: T) -> Attribute { + Attribute { + key, + value: mk_attr_(value), + } +} + +impl MetadataVisitor for XmlWriter { + fn superblock_b(&mut self, sb: &Superblock) -> Result { + let tag = b"superblock"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"uuid", sb.uuid.clone())); + elem.push_attribute(mk_attr(b"block_size", sb.block_size)); + elem.push_attribute(mk_attr(b"nr_cache_blocks", sb.nr_cache_blocks)); + elem.push_attribute(mk_attr(b"policy", sb.policy.clone())); + elem.push_attribute(mk_attr(b"hint_width", sb.hint_width)); + + self.w.write_event(Event::Start(elem))?; + Ok(Visit::Continue) + } + + fn superblock_e(&mut self) -> Result { + self.w.write_event(Event::End(BytesEnd::borrowed(b"superblock")))?; + Ok(Visit::Continue) + } + + fn mappings_b(&mut self) -> Result { + let tag = b"mappings"; + let elem = BytesStart::owned(tag.to_vec(), tag.len()); + self.w.write_event(Event::Start(elem))?; + Ok(Visit::Continue) + } + + fn mappings_e(&mut self) -> Result { + self.w.write_event(Event::End(BytesEnd::borrowed(b"mappings")))?; + Ok(Visit::Continue) + } + + fn mapping(&mut self, m: &Map) -> Result { + let tag = b"map"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"cache_block", m.cblock)); + elem.push_attribute(mk_attr(b"origin_block", m.oblock)); + elem.push_attribute(mk_attr(b"dirty", m.dirty)); + self.w.write_event(Event::Empty(elem))?; + Ok(Visit::Continue) + } + + fn hints_b(&mut self) -> Result { + let tag = b"hints"; + let elem = BytesStart::owned(tag.to_vec(), tag.len()); + self.w.write_event(Event::Start(elem))?; + Ok(Visit::Continue) + } + + fn hints_e(&mut self) -> Result { + self.w.write_event(Event::End(BytesEnd::borrowed(b"hints")))?; + Ok(Visit::Continue) + } + + fn hint(&mut self, h: &Hint) -> Result { + let tag = b"hint"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"cache_block", h.cblock)); + elem.push_attribute(mk_attr(b"data", encode(&h.data[0..]))); + self.w.write_event(Event::Empty(elem))?; + Ok(Visit::Continue) + } + + fn discards_b(&mut self) -> Result { + let tag = b"discards"; + let elem = BytesStart::owned(tag.to_vec(), tag.len()); + self.w.write_event(Event::Start(elem))?; + Ok(Visit::Continue) + } + + fn discards_e(&mut self) -> Result { + self.w.write_event(Event::End(BytesEnd::borrowed(b"discards")))?; + Ok(Visit::Continue) + } + + fn discard(&mut self, d: &Discard) -> Result { + let tag = b"discard"; + let mut elem = BytesStart::owned(tag.to_vec(), tag.len()); + elem.push_attribute(mk_attr(b"dbegin", d.begin)); + elem.push_attribute(mk_attr(b"dend", d.end)); + self.w.write_event(Event::Empty(elem))?; + Ok(Visit::Continue) + } + + fn eof(&mut self) -> Result { + Ok(Visit::Continue) + } +} + diff --git a/src/lib.rs b/src/lib.rs index 6d52650..ac0780e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ extern crate quickcheck; extern crate quickcheck_macros; pub mod block_manager; +pub mod cache; pub mod checksum; pub mod file_utils; pub mod pack; diff --git a/src/thin/check.rs b/src/thin/check.rs index 6461ad8..a0a92a6 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -212,7 +212,6 @@ pub fn check(dev: &Path) -> Result<()> { println!("found {} devices", visitor.devs.len()); } -/* // mapping top level let mut roots = HashMap::new(); { @@ -242,7 +241,6 @@ pub fn check(dev: &Path) -> Result<()> { pool.join(); } - */ // data space map { @@ -276,7 +274,7 @@ pub fn check(dev: &Path) -> Result<()> { return Err(anyhow!("Index entry points to block ({}) that isn't a bitmap", b.loc)); } - let bitmap = unpack::(b.get_data())?; + let _bitmap = unpack::(b.get_data())?; } } diff --git a/tests/cache_check.rs b/tests/cache_check.rs new file mode 100644 index 0000000..fc79bc5 --- /dev/null +++ b/tests/cache_check.rs @@ -0,0 +1,129 @@ +use anyhow::Result; +use thinp::file_utils; +use thinp::version::TOOLS_VERSION; +use duct::cmd; + +mod common; + +use common::*; +use common::test_dir::*; +use common::cache_xml_generator::{write_xml, XmlGen}; + +//------------------------------------------ + +#[test] +fn accepts_v() -> Result<()> { + let stdout = cache_check!("-V").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +#[test] +fn accepts_version() -> Result<()> { + let stdout = cache_check!("--version").read()?; + assert_eq!(stdout, TOOLS_VERSION); + Ok(()) +} + +const USAGE: &str = "Usage: cache_check [options] {device|file}\nOptions:\n {-q|--quiet}\n {-h|--help}\n {-V|--version}\n {--clear-needs-check-flag}\n {--super-block-only}\n {--skip-mappings}\n {--skip-hints}\n {--skip-discards}"; + +#[test] +fn accepts_h() -> Result<()> { + let stdout = cache_check!("-h").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn accepts_help() -> Result<()> { + let stdout = cache_check!("--help").read()?; + assert_eq!(stdout, USAGE); + Ok(()) +} + +#[test] +fn missing_metadata() -> Result<()> { + let stderr = run_fail(cache_check!())?; + assert!(stderr.contains("No input file provided")); + Ok(()) +} + +#[test] +fn no_such_metadata() -> Result<()> { + let stderr = run_fail(cache_check!("/arbitrary/filename"))?; + assert!(stderr.contains("No such file or directory")); + Ok(()) +} + +#[test] +fn metadata_cannot_be_a_directory() -> Result<()> { + let stderr = run_fail(cache_check!("/tmp"))?; + assert!(stderr.contains("Not a block device or regular file")); + Ok(()) +} + +#[test] +fn unreadable_metadata() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_valid_md(&mut td)?; + cmd!("chmod", "-r", &md).run()?; + let stderr = run_fail(cache_check!(&md))?; + assert!(stderr.contains("syscall 'open' failed: Permission denied")); + Ok(()) +} + +#[test] +fn corrupt_metadata() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + run_fail(cache_check!(&md))?; + Ok(()) +} + +#[test] +fn failing_q() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let output = cache_check!("-q", &md).unchecked().run()?; + assert!(!output.status.success()); + assert_eq!(output.stdout.len(), 0); + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +#[test] +fn failing_quiet() -> Result<()> { + let mut td = TestDir::new()?; + let md = mk_zeroed_md(&mut td)?; + let output = cache_check!("--quiet", &md).unchecked().run()?; + assert!(!output.status.success()); + assert_eq!(output.stdout.len(), 0); + assert_eq!(output.stderr.len(), 0); + Ok(()) +} + +// (define-scenario (cache-check valid-metadata-passes) +// "A valid metadata area passes" +// (with-valid-metadata (md) +// (run-ok (cache-check md)))) +// +// (define-scenario (cache-check bad-metadata-version) +// "Invalid metadata version fails" +// (with-cache-xml (xml) +// (with-empty-metadata (md) +// (cache-restore "-i" xml "-o" md "--debug-override-metadata-version" "12345") +// (run-fail (cache-check md))))) +// +// (define-scenario (cache-check tiny-metadata) +// "Prints helpful message in case tiny metadata given" +// (with-temp-file-sized ((md "cache.bin" 1024)) +// (run-fail-rcv (_ stderr) (cache-check md) +// (assert-starts-with "Metadata device/file too small. Is this binary metadata?" stderr)))) +// +// (define-scenario (cache-check spot-accidental-xml-data) +// "Prints helpful message if XML metadata given" +// (with-cache-xml (xml) +// (system (fmt #f "man bash >> " xml)) +// (run-fail-rcv (_ stderr) (cache-check xml) +// (assert-matches ".*This looks like XML. cache_check only checks the binary metadata format." stderr)))) +// diff --git a/tests/common/cache_xml_generator.rs b/tests/common/cache_xml_generator.rs new file mode 100644 index 0000000..d9e558e --- /dev/null +++ b/tests/common/cache_xml_generator.rs @@ -0,0 +1,94 @@ +use anyhow::{Result}; +use rand::prelude::*; +use std::collections::HashSet; +use std::fs::OpenOptions; +use std::path::Path; +use thinp::cache::xml; + +//------------------------------------------ + +pub trait XmlGen { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>; +} + +pub fn write_xml(path: &Path, g: &mut dyn XmlGen) -> Result<()> { + let xml_out = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(path)?; + let mut w = xml::XmlWriter::new(xml_out); + + g.generate_xml(&mut w) +} + +pub struct CacheGen { + block_size: u64, + nr_cache_blocks: u64, + nr_origin_blocks: u64, + percent_resident: u8, + percent_dirty: u8, +} + +impl CacheGen { + pub fn new( + block_size: u64, + nr_cache_blocks: u64, + nr_origin_blocks: u64, + percent_resident: u8, + percent_dirty: u8, + ) -> Self { + CacheGen { + block_size, + nr_cache_blocks, + nr_origin_blocks, + percent_resident, + percent_dirty, + } + } +} + +impl XmlGen for CacheGen { + fn generate_xml(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + v.superblock_b(&xml::Superblock { + uuid: "".to_string(), + block_size: self.block_size, + nr_cache_blocks: self.nr_cache_blocks, + policy: "smq".to_string(), + hint_width: 4, + })?; + + let mut cblocks = Vec::new(); + for n in 0..self.nr_cache_blocks { + cblocks.push(n); + } + cblocks.shuffle(&mut rand::thread_rng()); + + v.mappings_b()?; + { + let nr_resident = (self.nr_cache_blocks * 100 as u64) / (self.percent_resident as u64); + let mut used = HashSet::new(); + for n in 0..nr_resident { + let mut oblock = 0u64; + while used.contains(&oblock) { + oblock = rand::thread_rng().gen(); + } + + used.insert(oblock); + // FIXME: dirty should vary + v.mapping(&xml::Map { + cblock: cblocks[n as usize], + oblock, + dirty: false, + })?; + } + } + v.mappings_e()?; + + v.superblock_e()?; + Ok(()) + } +} + +//------------------------------------------ diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 21c5526..6580f79 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,11 +6,14 @@ use std::fs::OpenOptions; use std::io::{Read, Write}; use std::path::{PathBuf}; use std::str::from_utf8; -use tempfile::{tempdir, TempDir}; use thinp::file_utils; -pub mod xml_generator; -use crate::common::xml_generator::{write_xml, SingleThinS}; +pub mod thin_xml_generator; +pub mod cache_xml_generator; +pub mod test_dir; + +use crate::common::thin_xml_generator::{write_xml, SingleThinS}; +use test_dir::TestDir; //------------------------------------------ @@ -103,28 +106,19 @@ macro_rules! thin_metadata_unpack { }; } +#[macro_export] +macro_rules! cache_check { + ( $( $arg: expr ),* ) => { + { + use std::ffi::OsString; + let args: &[OsString] = &[$( Into::::into($arg) ),*]; + duct::cmd("bin/cache_check", args).stdout_capture().stderr_capture() + } + }; +} + //------------------------------------------ -pub struct TestDir { - dir: TempDir, - file_count: usize, -} - -impl TestDir { - pub fn new() -> Result { - let dir = tempdir()?; - Ok(TestDir { dir, file_count: 0 }) - } - - pub fn mk_path(&mut self, file: &str) -> PathBuf { - let mut p = PathBuf::new(); - p.push(&self.dir); - p.push(PathBuf::from(format!("{:02}_{}", self.file_count, file))); - self.file_count += 1; - p - } -} - // Returns stderr, a non zero status must be returned pub fn run_fail(command: Expression) -> Result { let output = command.stderr_capture().unchecked().run()?; diff --git a/tests/common/test_dir.rs b/tests/common/test_dir.rs new file mode 100644 index 0000000..91a4b62 --- /dev/null +++ b/tests/common/test_dir.rs @@ -0,0 +1,27 @@ +use anyhow::Result; +use std::path::{PathBuf}; +use tempfile::{tempdir, TempDir}; + +//--------------------------------------- + +pub struct TestDir { + dir: TempDir, + file_count: usize, +} + +impl TestDir { + pub fn new() -> Result { + let dir = tempdir()?; + Ok(TestDir { dir, file_count: 0 }) + } + + pub fn mk_path(&mut self, file: &str) -> PathBuf { + let mut p = PathBuf::new(); + p.push(&self.dir); + p.push(PathBuf::from(format!("{:02}_{}", self.file_count, file))); + self.file_count += 1; + p + } +} + +//--------------------------------------- diff --git a/tests/common/xml_generator.rs b/tests/common/thin_xml_generator.rs similarity index 100% rename from tests/common/xml_generator.rs rename to tests/common/thin_xml_generator.rs diff --git a/tests/thin_check.rs b/tests/thin_check.rs index a3aa22e..5a7c801 100644 --- a/tests/thin_check.rs +++ b/tests/thin_check.rs @@ -5,7 +5,8 @@ use thinp::version::TOOLS_VERSION; mod common; use common::*; -use common::xml_generator::{write_xml, FragmentedS}; +use common::test_dir::*; +use common::thin_xml_generator::{write_xml, FragmentedS}; //------------------------------------------ diff --git a/tests/thin_delta.rs b/tests/thin_delta.rs index 7c1442b..49d92cb 100644 --- a/tests/thin_delta.rs +++ b/tests/thin_delta.rs @@ -3,6 +3,7 @@ use thinp::version::TOOLS_VERSION; mod common; use common::*; +use common::test_dir::*; //------------------------------------------ diff --git a/tests/thin_dump.rs b/tests/thin_dump.rs index c754eff..d2e58a9 100644 --- a/tests/thin_dump.rs +++ b/tests/thin_dump.rs @@ -6,6 +6,7 @@ use std::str::from_utf8; mod common; use common::*; +use common::test_dir::*; //------------------------------------------ diff --git a/tests/thin_metadata_pack.rs b/tests/thin_metadata_pack.rs index ee9926b..2e12228 100644 --- a/tests/thin_metadata_pack.rs +++ b/tests/thin_metadata_pack.rs @@ -3,6 +3,7 @@ use thinp::version::TOOLS_VERSION; mod common; use common::*; +use common::test_dir::*; //------------------------------------------ diff --git a/tests/thin_metadata_unpack.rs b/tests/thin_metadata_unpack.rs index 057e147..b0a540b 100644 --- a/tests/thin_metadata_unpack.rs +++ b/tests/thin_metadata_unpack.rs @@ -3,6 +3,7 @@ use thinp::version::TOOLS_VERSION; mod common; use common::*; +use common::test_dir::*; //------------------------------------------ diff --git a/tests/thin_repair.rs b/tests/thin_repair.rs index 9bf9a33..98ded1f 100644 --- a/tests/thin_repair.rs +++ b/tests/thin_repair.rs @@ -4,6 +4,7 @@ use thinp::version::TOOLS_VERSION; mod common; use common::*; +use common::test_dir::*; //------------------------------------------ diff --git a/tests/thin_restore.rs b/tests/thin_restore.rs index 819b292..9a90273 100644 --- a/tests/thin_restore.rs +++ b/tests/thin_restore.rs @@ -5,6 +5,7 @@ use thinp::version::TOOLS_VERSION; mod common; use common::*; +use common::test_dir::*; //------------------------------------------ diff --git a/tests/thin_rmap.rs b/tests/thin_rmap.rs index e741c7f..1a932a2 100644 --- a/tests/thin_rmap.rs +++ b/tests/thin_rmap.rs @@ -3,6 +3,7 @@ use thinp::version::TOOLS_VERSION; mod common; use common::*; +use common::test_dir::*; //------------------------------------------ diff --git a/tests/thin_shrink.rs b/tests/thin_shrink.rs index 3397d20..0856859 100644 --- a/tests/thin_shrink.rs +++ b/tests/thin_shrink.rs @@ -9,8 +9,8 @@ use thinp::file_utils; use thinp::thin::xml::{self, Visit}; mod common; -use common::*; -use common::xml_generator::{ +use common::test_dir::*; +use common::thin_xml_generator::{ write_xml, EmptyPoolS, FragmentedS, SingleThinS, SnapS, XmlGen }; From 8f76371bb2cd8ba8ca32fbf2291916b58995eb46 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Fri, 7 Aug 2020 15:41:21 +0100 Subject: [PATCH 32/52] [functional-tests] Fix clippy warnings --- src/block_manager.rs | 2 +- src/pack/vm.rs | 6 ++-- src/pdata/btree.rs | 18 +++++----- src/pdata/space_map.rs | 2 +- src/thin/check.rs | 6 ++-- src/thin/xml.rs | 12 ++----- tests/cache_check.rs | 2 -- tests/common/thin_xml_generator.rs | 54 ++++++++++++++---------------- tests/thin_restore.rs | 2 +- 9 files changed, 46 insertions(+), 58 deletions(-) diff --git a/src/block_manager.rs b/src/block_manager.rs index 1d5e061..5fc7589 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -190,7 +190,7 @@ impl IoEngine for AsyncIoEngine { let count = blocks.len(); let fd = types::Target::Fd(inner.input.as_raw_fd()); - for b in blocks.into_iter() { + for b in blocks.iter_mut() { let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32) .offset(b.loc as i64 * BLOCK_SIZE as i64); diff --git a/src/pack/vm.rs b/src/pack/vm.rs index 592c005..4086640 100644 --- a/src/pack/vm.rs +++ b/src/pack/vm.rs @@ -416,7 +416,7 @@ mod tests { } } - fn check_u64s_match(ns: &Vec, bytes: &[u8]) -> bool { + fn check_u64s_match(ns: &[u64], bytes: &[u8]) -> bool { let mut packed = Vec::with_capacity(ns.len() * 8); let mut w = Cursor::new(&mut packed); for n in ns { @@ -425,7 +425,7 @@ mod tests { packed == bytes } - fn check_pack_u64s(ns: &Vec) -> bool { + fn check_pack_u64s(ns: &[u64]) -> bool { println!("packing {:?}", &ns); let mut bs = Vec::with_capacity(4096); @@ -461,7 +461,7 @@ mod tests { check_pack_u64s(&ns) } - fn check_pack_shifted_u64s(ns: &Vec<(u64, u64)>) -> bool { + fn check_pack_shifted_u64s(ns: &[(u64, u64)]) -> bool { let shifted: Vec = ns .iter() .map(|(h, l)| (h << 24) | (l & ((1 << 24) - 1))) diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 81802df..a5c7ba5 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -31,7 +31,6 @@ const NODE_HEADER_SIZE: usize = 32; pub struct NodeHeader { is_leaf: bool, - block: u64, nr_entries: u32, max_entries: u32, value_size: u32, @@ -44,7 +43,7 @@ const LEAF_NODE: u32 = 2; pub fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> { let (i, _csum) = le_u32(data)?; let (i, flags) = le_u32(i)?; - let (i, block) = le_u64(i)?; + let (i, _block) = le_u64(i)?; let (i, nr_entries) = le_u32(i)?; let (i, max_entries) = le_u32(i)?; let (i, value_size) = le_u32(i)?; @@ -54,7 +53,6 @@ pub fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> { i, NodeHeader { is_leaf: flags == LEAF_NODE, - block, nr_entries, max_entries, value_size, @@ -111,18 +109,18 @@ pub fn unpack_node( } if header.nr_entries > header.max_entries { - return node_err(format!("nr_entries > max_entries")); + return node_err("nr_entries > max_entries".to_string()); } if !ignore_non_fatal { if header.max_entries % 3 != 0 { - return node_err(format!("max_entries is not divisible by 3")); + return node_err("max_entries is not divisible by 3".to_string()); } if !is_root { let min = header.max_entries / 3; if header.nr_entries < min { - return node_err(format!("too few entries")); + return node_err("too few entries".to_string()); } } } @@ -133,7 +131,7 @@ pub fn unpack_node( for k in &keys { if let Some(l) = last { if k <= l { - return node_err(format!("keys out of order")); + return node_err("keys out of order".to_string()); } } @@ -186,7 +184,7 @@ impl Unpack for u32 { //------------------------------------------ pub trait NodeVisitor { - fn visit<'a>(&mut self, w: &BTreeWalker, b: &Block, node: &Node) -> Result<()>; + fn visit(&mut self, w: &BTreeWalker, b: &Block, node: &Node) -> Result<()>; } #[derive(Clone)] @@ -200,7 +198,7 @@ impl BTreeWalker { pub fn new(engine: Arc, ignore_non_fatal: bool) -> BTreeWalker { let nr_blocks = engine.get_nr_blocks() as usize; let r: BTreeWalker = BTreeWalker { - engine: engine, + engine, seen: Arc::new(Mutex::new(FixedBitSet::with_capacity(nr_blocks))), ignore_non_fatal, }; @@ -224,7 +222,7 @@ impl BTreeWalker { } } - fn walk_nodes(&mut self, visitor: &mut NV, bs: &Vec) -> Result<()> + fn walk_nodes(&mut self, visitor: &mut NV, bs: &[u64]) -> Result<()> where NV: NodeVisitor, V: Unpack, diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 649964f..7242cbe 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -116,7 +116,7 @@ impl Unpack for Bitmap { for _b in 0..32 { let val = word & 0x3; - word = word >> 2; + word >>= 2; if val < 3 { entries.push(BitmapEntry::Small(val as u8)); diff --git a/src/thin/check.rs b/src/thin/check.rs index a0a92a6..4330ecb 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -148,12 +148,12 @@ impl NodeVisitor for IndexVisitor { fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { if let Node::Leaf { header: _h, - keys, + keys: _k, values, } = node { - for n in 0..keys.len() { + for v in values { // FIXME: check keys are in incremental order - let v = values[n].clone(); + let v = v.clone(); self.entries.push(v); } } diff --git a/src/thin/xml.rs b/src/thin/xml.rs index 7da0fe2..73155d3 100644 --- a/src/thin/xml.rs +++ b/src/thin/xml.rs @@ -359,13 +359,7 @@ where reader.trim_text(true); let mut buf = Vec::new(); - loop { - match handle_event(&mut reader, &mut buf, visitor)? { - Visit::Continue => {} - Visit::Stop => break, - } - } - + while let Visit::Continue = handle_event(&mut reader, &mut buf, visitor)? {} Ok(()) } @@ -380,7 +374,7 @@ impl MetadataVisitor for SBVisitor { self.superblock = Some(sb.clone()); Ok(Visit::Stop) } - + fn superblock_e(&mut self) -> Result { Ok(Visit::Continue) } @@ -405,7 +399,7 @@ pub fn read_superblock(input: R) -> Result where R: Read, { - let mut v = SBVisitor {superblock: None}; + let mut v = SBVisitor { superblock: None }; read(input, &mut v)?; Ok(v.superblock.unwrap()) } diff --git a/tests/cache_check.rs b/tests/cache_check.rs index fc79bc5..3fb9668 100644 --- a/tests/cache_check.rs +++ b/tests/cache_check.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use thinp::file_utils; use thinp::version::TOOLS_VERSION; use duct::cmd; @@ -7,7 +6,6 @@ mod common; use common::*; use common::test_dir::*; -use common::cache_xml_generator::{write_xml, XmlGen}; //------------------------------------------ diff --git a/tests/common/thin_xml_generator.rs b/tests/common/thin_xml_generator.rs index eb388ab..359b1e9 100644 --- a/tests/common/thin_xml_generator.rs +++ b/tests/common/thin_xml_generator.rs @@ -136,7 +136,7 @@ fn mk_runs(thin_id: u32, total_len: u64, run_len: std::ops::Range) -> Vec> = runs.iter().map(|r| r.clone()).collect(); + let runs: VecDeque> = runs.iter().cloned().collect(); Allocator { runs } } @@ -297,27 +297,25 @@ impl Run { fn split(&self, n: u64) -> (Option, Option) { if n == 0 { - return (None, Some(self.clone())); + (None, Some(self.clone())) + } else if self.len() <= n { + (Some(self.clone()), None) } else { - if self.len() <= n { - return (Some(self.clone()), None); - } else { - match self { - Run::Mapped { data_begin, len } => ( - Some(Run::Mapped { - data_begin: *data_begin, - len: n, - }), - Some(Run::Mapped { - data_begin: data_begin + n, - len: len - n, - }), - ), - Run::UnMapped { len } => ( - Some(Run::UnMapped { len: n }), - Some(Run::UnMapped { len: len - n }), - ), - } + match self { + Run::Mapped { data_begin, len } => ( + Some(Run::Mapped { + data_begin: *data_begin, + len: n, + }), + Some(Run::Mapped { + data_begin: data_begin + n, + len: len - n, + }), + ), + Run::UnMapped { len } => ( + Some(Run::UnMapped { len: n }), + Some(Run::UnMapped { len: len - n }), + ), } } } @@ -437,7 +435,7 @@ fn mk_snap_mapping( runs } -fn split_runs(mut n: u64, runs: &Vec) -> (Vec, Vec) { +fn split_runs(mut n: u64, runs: &[Run]) -> (Vec, Vec) { let mut before = Vec::new(); let mut after = Vec::new(); @@ -462,11 +460,11 @@ fn split_runs(mut n: u64, runs: &Vec) -> (Vec, Vec) { } fn apply_snap_runs( - origin: &Vec, - snap: &Vec, + origin: &[Run], + snap: &[SnapRun], allocator: &mut Allocator, ) -> Result> { - let mut origin = origin.clone(); + let mut origin = origin.to_owned(); let mut runs = Vec::new(); for SnapRun(st, slen) in snap { diff --git a/tests/thin_restore.rs b/tests/thin_restore.rs index 9a90273..62f78dd 100644 --- a/tests/thin_restore.rs +++ b/tests/thin_restore.rs @@ -23,7 +23,7 @@ fn accepts_version() -> Result<()> { Ok(()) } -const USAGE: &'static str = "Usage: thin_restore [options]\nOptions:\n {-h|--help}\n {-i|--input} \n {-o|--output} \n {--transaction-id} \n {--data-block-size} \n {--nr-data-blocks} \n {-q|--quiet}\n {-V|--version}"; +const USAGE: &str = "Usage: thin_restore [options]\nOptions:\n {-h|--help}\n {-i|--input} \n {-o|--output} \n {--transaction-id} \n {--data-block-size} \n {--nr-data-blocks} \n {-q|--quiet}\n {-V|--version}"; #[test] fn accepts_h() -> Result<()> { From ec8f7b7fa8bb9d89f19374849638901b06f59f8d Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Sat, 8 Aug 2020 09:54:16 +0100 Subject: [PATCH 33/52] [thin_check (rust)] Keep track of data block ref counts as we walk the mapping tree. --- Cargo.lock | 16 ++--- src/pdata/space_map.rs | 132 ++++++++++++++++++++++++++++++++++++----- src/thin/check.rs | 86 +++++++++++++++++++++++++-- 3 files changed, 206 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2678fd..fdd3687 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,9 +85,9 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "clap" -version = "2.33.1" +version = "2.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" +checksum = "10040cdf04294b565d9e0319955430099ec3813a64c952b86a41200ad714ae48" dependencies = [ "ansi_term", "atty", @@ -305,9 +305,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9" +checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" [[package]] name = "log" @@ -359,9 +359,9 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746" +checksum = "e0396233fb2d5b0ae3f05ff6aba9a09185f7f6e70f87fb01147d545f85364665" dependencies = [ "proc-macro2", "quote", @@ -607,9 +607,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdb98bcb1f9d81d07b536179c269ea15999b5d14ea958196413869445bb5250" +checksum = "e69abc24912995b3038597a7a593be5053eb0fb44f3cc5beec0deb421790c1f4" dependencies = [ "proc-macro2", "quote", diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 7242cbe..a05fb2b 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -1,5 +1,7 @@ use anyhow::{anyhow, Result}; +use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; +use std::collections::HashMap; use crate::block_manager::*; use crate::pdata::btree::Unpack; @@ -16,9 +18,7 @@ pub struct SMRoot { pub fn unpack_root(data: &[u8]) -> Result { match SMRoot::unpack(data) { - Err(_e) => { - Err(anyhow!("couldn't parse SMRoot")) - }, + Err(_e) => Err(anyhow!("couldn't parse SMRoot")), Ok((_i, v)) => Ok(v), } } @@ -27,19 +27,22 @@ impl Unpack for SMRoot { fn disk_size() -> u32 { 32 } - + fn unpack(data: &[u8]) -> IResult<&[u8], SMRoot> { let (i, nr_blocks) = le_u64(data)?; let (i, nr_allocated) = le_u64(i)?; let (i, bitmap_root) = le_u64(i)?; let (i, ref_count_root) = le_u64(i)?; - Ok ((i, SMRoot { - nr_blocks, - nr_allocated, - bitmap_root, - ref_count_root, - })) + Ok(( + i, + SMRoot { + nr_blocks, + nr_allocated, + bitmap_root, + ref_count_root, + }, + )) } } @@ -62,7 +65,14 @@ impl Unpack for IndexEntry { let (i, nr_free) = le_u32(i)?; let (i, none_free_before) = le_u32(i)?; - Ok((i, IndexEntry {blocknr, nr_free, none_free_before})) + Ok(( + i, + IndexEntry { + blocknr, + nr_free, + none_free_before, + }, + )) } } @@ -85,11 +95,18 @@ impl Unpack for BitmapHeader { let (i, not_used) = le_u32(i)?; let (i, blocknr) = le_u64(i)?; - Ok((i, BitmapHeader {csum, not_used, blocknr} )) + Ok(( + i, + BitmapHeader { + csum, + not_used, + blocknr, + }, + )) } } -#[derive(Debug)] +#[derive(Clone, Debug, PartialEq)] pub enum BitmapEntry { Small(u8), Overflow, @@ -109,7 +126,7 @@ impl Unpack for Bitmap { fn unpack(data: &[u8]) -> IResult<&[u8], Self> { let (mut i, header) = BitmapHeader::unpack(data)?; - let mut entries = Vec::new(); + let mut entries = Vec::new(); let nr_words = (BLOCK_SIZE - BitmapHeader::disk_size() as usize) / 8; for _w in 0..nr_words { let (tmp, mut word) = le_u64(i)?; @@ -128,7 +145,92 @@ impl Unpack for Bitmap { i = tmp; } - Ok((i, Bitmap {header, entries})) + Ok((i, Bitmap { header, entries })) + } +} + +//------------------------------------------ + +pub struct CoreSpaceMap { + nr_entries: u64, + bits: FixedBitSet, + overflow: HashMap, +} + +impl CoreSpaceMap { + pub fn new(nr_entries: u64) -> CoreSpaceMap { + let bits = FixedBitSet::with_capacity(nr_entries as usize * 2); + CoreSpaceMap { + nr_entries, + bits, + overflow: HashMap::new(), + } + } + + fn get_bits(&self, b: u64) -> Result { + if b >= self.nr_entries { + return Err(anyhow!("space map index out of bounds")); + } + + let low_bit = self.bits.contains((b * 2) as usize); + let high_bit = self.bits.contains((b * 2 + 1) as usize); + let mut result = 0u32; + + if high_bit { + result += 2; + } + if low_bit { + result += 1; + } + + Ok(result) + } + + fn set_bits(&mut self, b: u64, count: u32) -> Result<()> { + if count > 3 { + return Err(anyhow!("internal error: bits entry should be <= 3")); + } + + self.bits.set((b * 2) as usize, (count & 0x1) != 0); + self.bits.set((b * 2 + 1) as usize, (count & 0x2) != 0); + Ok(()) + } + + pub fn get(&self, b: u64) -> Result { + let result = self.get_bits(b)?; + if result < 3 { + Ok(result) + } else { + match self.overflow.get(&b) { + None => Err(anyhow!( + "internal error: missing overflow entry in space map" + )), + Some(result) => Ok(*result), + } + } + } + + pub fn inc(&mut self, b: u64) -> Result<()> { + let old = self.get(b)?; + + if old < 3 { + // bump up the bits + self.set_bits(b, old + 1)?; + + if old == 2 { + // insert overflow entry + self.overflow.insert(b, 1); + } + } else if let Some(count) = self.overflow.get_mut(&b) { + // increment the overflow + *count += 1; + } else { + return Err(anyhow!( + "internal error: missing overflow entry in space map" + )); + } + + Ok(()) } } diff --git a/src/thin/check.rs b/src/thin/check.rs index 4330ecb..84e7a7f 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; -use std::collections::HashMap; +use std::collections::{HashMap, BTreeMap}; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -66,10 +66,21 @@ impl Unpack for BlockTime { } } -struct BottomLevelVisitor {} +struct BottomLevelVisitor { + data_sm: Arc>, +} impl NodeVisitor for BottomLevelVisitor { - fn visit(&mut self, _w: &BTreeWalker, _b: &Block, _node: &Node) -> Result<()> { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + // FIXME: do other checks + + if let Node::Leaf {header: _h, keys: _k, values} = node { + let mut data_sm = self.data_sm.lock().unwrap(); + for bt in values { + data_sm.inc(bt.block)?; + } + } + Ok(()) } } @@ -197,6 +208,46 @@ impl NodeVisitor for ValueCollector { //------------------------------------------ +struct RangeBuilder { + run: Option<(u64, BitmapEntry)>, + runs: Vec<(u64, BitmapEntry)> +} + +impl RangeBuilder { + fn new() -> RangeBuilder { + RangeBuilder { + run: None, + runs: Vec::new(), + } + } + + fn push(&mut self, e: &BitmapEntry) { + match &self.run { + Some((len, e2)) if *e == *e2 => { + self.run = Some((*len + 1, e2.clone())); + }, + Some((len, e2)) => { + self.runs.push((*len, e2.clone())); + self.run = Some((1, e.clone())); + }, + None => { + self.run = Some((1, e.clone())); + } + } + } + + fn complete(&mut self) { + match &self.run { + Some((len, e)) => { + self.runs.push((*len, e.clone())); + }, + None => {} + } + } +} + +//------------------------------------------ + pub fn check(dev: &Path) -> Result<()> { let engine = Arc::new(AsyncIoEngine::new(dev, 256)?); @@ -230,10 +281,14 @@ pub fn check(dev: &Path) -> Result<()> { engine.get_nr_blocks() as usize, ))); + let root = unpack::(&sb.data_sm_root[0..])?; + let data_sm = Arc::new(Mutex::new(CoreSpaceMap::new(root.nr_blocks))); + for (thin_id, root) in roots { let mut w = BTreeWalker::new_with_seen(engine.clone(), seen.clone(), false); + let data_sm = data_sm.clone(); pool.execute(move || { - let mut v = BottomLevelVisitor {}; + let mut v = BottomLevelVisitor {data_sm}; let result = w.walk(&mut v, root).expect("walk failed"); // FIXME: return error eprintln!("checked thin_dev {} -> {:?}", thin_id, result); }); @@ -266,6 +321,8 @@ pub fn check(dev: &Path) -> Result<()> { let _result = w.walk(&mut v, root.bitmap_root); eprintln!("{} index entries", v.entries.len()); + let mut builder = RangeBuilder::new(); + for i in v.entries { let mut b = Block::new(i.blocknr); engine.read(&mut b)?; @@ -274,7 +331,26 @@ pub fn check(dev: &Path) -> Result<()> { return Err(anyhow!("Index entry points to block ({}) that isn't a bitmap", b.loc)); } - let _bitmap = unpack::(b.get_data())?; + let bitmap = unpack::(b.get_data())?; + for e in bitmap.entries { + builder.push(&e); + } + } + builder.complete(); + eprintln!("{} ranges", builder.runs.len()); + + let mut counts = BTreeMap::new(); + for (len, _v) in builder.runs { + if let Some(c) = counts.get(&len) { + let new_c = *c + 1; + counts.insert(len, new_c); + } else { + counts.insert(len, 1); + } + } + + for (len, c) in counts { + eprintln!("{}: {}", len, c); } } From 7cf239b87851ba932a2ba499a13b04a0ff1c5d6c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Sat, 8 Aug 2020 12:36:13 +0100 Subject: [PATCH 34/52] [thin_check (rust)] speed up CoreSpaceMap --- src/pdata/space_map.rs | 92 +++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index a05fb2b..170cd10 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -1,7 +1,6 @@ use anyhow::{anyhow, Result}; -use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; -use std::collections::HashMap; +use std::collections::BTreeMap; use crate::block_manager::*; use crate::pdata::btree::Unpack; @@ -151,49 +150,49 @@ impl Unpack for Bitmap { //------------------------------------------ +const ENTRIES_PER_WORD: u64 = 32; + pub struct CoreSpaceMap { nr_entries: u64, - bits: FixedBitSet, - overflow: HashMap, + bits: Vec, + overflow: BTreeMap, } impl CoreSpaceMap { pub fn new(nr_entries: u64) -> CoreSpaceMap { - let bits = FixedBitSet::with_capacity(nr_entries as usize * 2); + let nr_words = (nr_entries + ENTRIES_PER_WORD - 1) / ENTRIES_PER_WORD; CoreSpaceMap { nr_entries, - bits, - overflow: HashMap::new(), + bits: vec![0; nr_words as usize], + overflow: BTreeMap::new(), } } - fn get_bits(&self, b: u64) -> Result { + fn check_bounds(&self, b: u64) -> Result<()> { if b >= self.nr_entries { return Err(anyhow!("space map index out of bounds")); } - - let low_bit = self.bits.contains((b * 2) as usize); - let high_bit = self.bits.contains((b * 2 + 1) as usize); - let mut result = 0u32; - - if high_bit { - result += 2; - } - if low_bit { - result += 1; - } - - Ok(result) + Ok(()) } - fn set_bits(&mut self, b: u64, count: u32) -> Result<()> { - if count > 3 { - return Err(anyhow!("internal error: bits entry should be <= 3")); + fn get_index(b: u64) -> (usize, usize) { + ( + (b / ENTRIES_PER_WORD) as usize, + ((b & (ENTRIES_PER_WORD - 1)) as usize) * 2, + ) + } + + fn get_bits(&self, b: u64) -> Result { + self.check_bounds(b)?; + + let result; + let (w, bit) = CoreSpaceMap::get_index(b); + unsafe { + let word = self.bits.get_unchecked(w); + result = (*word >> bit) & 0x3; } - self.bits.set((b * 2) as usize, (count & 0x1) != 0); - self.bits.set((b * 2 + 1) as usize, (count & 0x2) != 0); - Ok(()) + Ok(result as u32) } pub fn get(&self, b: u64) -> Result { @@ -211,23 +210,34 @@ impl CoreSpaceMap { } pub fn inc(&mut self, b: u64) -> Result<()> { - let old = self.get(b)?; + self.check_bounds(b)?; - if old < 3 { - // bump up the bits - self.set_bits(b, old + 1)?; + let (w, bit) = CoreSpaceMap::get_index(b); + let count; - if old == 2 { - // insert overflow entry - self.overflow.insert(b, 1); + unsafe { + let word = self.bits.get_unchecked_mut(w); + count = (*word >> bit) & 0x3; + + if count < 3 { + // bump up the bits + *word = (*word & !(0x3 << bit)) | (((count + 1) as u64) << bit); + + if count == 2 { + // insert overflow entry + self.overflow.insert(b, 1); + } + } + } + + if count >= 3 { + if let Some(count) = self.overflow.get_mut(&b) { + *count = *count + 1; + } else { + return Err(anyhow!( + "internal error: missing overflow entry in space map" + )); } - } else if let Some(count) = self.overflow.get_mut(&b) { - // increment the overflow - *count += 1; - } else { - return Err(anyhow!( - "internal error: missing overflow entry in space map" - )); } Ok(()) From 1e4a038b410ff51fbad7dcbffba6ae9b3832d923 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Sat, 8 Aug 2020 13:29:30 +0100 Subject: [PATCH 35/52] [thin_check (rust)] Reimplement CoreSpaceMap We now use a simple vector of elements that can hold 'nr thin devs'. Much faster. --- src/pdata/space_map.rs | 116 ++++++++++++----------------------------- src/thin/check.rs | 26 +++++++-- 2 files changed, 55 insertions(+), 87 deletions(-) diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 170cd10..6b77aac 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, Result}; use nom::{number::complete::*, IResult}; -use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; use crate::block_manager::*; use crate::pdata::btree::Unpack; @@ -150,97 +150,49 @@ impl Unpack for Bitmap { //------------------------------------------ -const ENTRIES_PER_WORD: u64 = 32; - -pub struct CoreSpaceMap { - nr_entries: u64, - bits: Vec, - overflow: BTreeMap, +pub trait SpaceMap { + fn get(&self, b: u64) -> Result; + fn inc(&mut self, begin: u64, len: u64) -> Result<()>; } -impl CoreSpaceMap { - pub fn new(nr_entries: u64) -> CoreSpaceMap { - let nr_words = (nr_entries + ENTRIES_PER_WORD - 1) / ENTRIES_PER_WORD; +pub struct CoreSpaceMap { + counts: Vec, +} + +impl CoreSpaceMap +where + V: Copy + Default + std::ops::AddAssign + From, +{ + pub fn new(nr_entries: u64) -> CoreSpaceMap { CoreSpaceMap { - nr_entries, - bits: vec![0; nr_words as usize], - overflow: BTreeMap::new(), + counts: vec![V::default(); nr_entries as usize], } } +} - fn check_bounds(&self, b: u64) -> Result<()> { - if b >= self.nr_entries { - return Err(anyhow!("space map index out of bounds")); +impl SpaceMap for CoreSpaceMap +where + V: Copy + Default + std::ops::AddAssign + From + Into, + { + fn get(&self, b: u64) -> Result { + Ok(self.counts[b as usize].into()) + } + + fn inc(&mut self, begin: u64, len: u64) -> Result<()> { + for b in begin..(begin + len) { + self.counts[b as usize] += V::from(1u8); } Ok(()) } +} - fn get_index(b: u64) -> (usize, usize) { - ( - (b / ENTRIES_PER_WORD) as usize, - ((b & (ENTRIES_PER_WORD - 1)) as usize) * 2, - ) - } - - fn get_bits(&self, b: u64) -> Result { - self.check_bounds(b)?; - - let result; - let (w, bit) = CoreSpaceMap::get_index(b); - unsafe { - let word = self.bits.get_unchecked(w); - result = (*word >> bit) & 0x3; - } - - Ok(result as u32) - } - - pub fn get(&self, b: u64) -> Result { - let result = self.get_bits(b)?; - if result < 3 { - Ok(result) - } else { - match self.overflow.get(&b) { - None => Err(anyhow!( - "internal error: missing overflow entry in space map" - )), - Some(result) => Ok(*result), - } - } - } - - pub fn inc(&mut self, b: u64) -> Result<()> { - self.check_bounds(b)?; - - let (w, bit) = CoreSpaceMap::get_index(b); - let count; - - unsafe { - let word = self.bits.get_unchecked_mut(w); - count = (*word >> bit) & 0x3; - - if count < 3 { - // bump up the bits - *word = (*word & !(0x3 << bit)) | (((count + 1) as u64) << bit); - - if count == 2 { - // insert overflow entry - self.overflow.insert(b, 1); - } - } - } - - if count >= 3 { - if let Some(count) = self.overflow.get_mut(&b) { - *count = *count + 1; - } else { - return Err(anyhow!( - "internal error: missing overflow entry in space map" - )); - } - } - - Ok(()) +pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc> { + if max_count <= u8::MAX as u32 { + Arc::new(Mutex::new(CoreSpaceMap::::new(nr_entries))) + } else if max_count <= u16::MAX as u32 { + Arc::new(Mutex::new(CoreSpaceMap::::new(nr_entries))) + } else { + Arc::new(Mutex::new(CoreSpaceMap::::new(nr_entries))) } } diff --git a/src/thin/check.rs b/src/thin/check.rs index 84e7a7f..d3bdce3 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -67,7 +67,7 @@ impl Unpack for BlockTime { } struct BottomLevelVisitor { - data_sm: Arc>, + data_sm: Arc>, } impl NodeVisitor for BottomLevelVisitor { @@ -75,9 +75,23 @@ impl NodeVisitor for BottomLevelVisitor { // FIXME: do other checks if let Node::Leaf {header: _h, keys: _k, values} = node { - let mut data_sm = self.data_sm.lock().unwrap(); - for bt in values { - data_sm.inc(bt.block)?; + if values.len() > 0 { + let mut data_sm = self.data_sm.lock().unwrap(); + + let mut start = values[0].block; + let mut len = 1; + + for n in 1..values.len() { + if values[n].block == start + len { + len += 1; + } else { + data_sm.inc(start, len)?; + start = values[n].block; + len = 1; + } + } + + data_sm.inc(start, len)?; } } @@ -256,10 +270,12 @@ pub fn check(dev: &Path) -> Result<()> { eprintln!("{:?}", sb); // device details + let nr_devs; { let mut visitor = DeviceVisitor::new(); let mut w = BTreeWalker::new(engine.clone(), false); w.walk(&mut visitor, sb.details_root)?; + nr_devs = visitor.devs.len(); println!("found {} devices", visitor.devs.len()); } @@ -282,7 +298,7 @@ pub fn check(dev: &Path) -> Result<()> { ))); let root = unpack::(&sb.data_sm_root[0..])?; - let data_sm = Arc::new(Mutex::new(CoreSpaceMap::new(root.nr_blocks))); + let data_sm = core_sm(root.nr_blocks, nr_devs as u32); for (thin_id, root) in roots { let mut w = BTreeWalker::new_with_seen(engine.clone(), seen.clone(), false); From 4054b1be4cfda8f8594e5ed1647cf19a6efb4e0c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Sat, 8 Aug 2020 14:58:13 +0100 Subject: [PATCH 36/52] [thin_check (rust)] Switch to BTreeMap. It's faster. --- src/thin/check.rs | 74 +++++------------------------------------------ 1 file changed, 8 insertions(+), 66 deletions(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index d3bdce3..87a2ee5 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; -use std::collections::{HashMap, BTreeMap}; +use std::collections::{BTreeMap}; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -16,7 +16,7 @@ use crate::checksum; //------------------------------------------ struct TopLevelVisitor<'a> { - roots: &'a mut HashMap, + roots: &'a mut BTreeMap, } impl<'a> NodeVisitor for TopLevelVisitor<'a> { @@ -133,13 +133,13 @@ impl Unpack for DeviceDetail { } struct DeviceVisitor { - devs: HashMap, + devs: BTreeMap, } impl DeviceVisitor { pub fn new() -> DeviceVisitor { DeviceVisitor { - devs: HashMap::new(), + devs: BTreeMap::new(), } } } @@ -222,46 +222,6 @@ impl NodeVisitor for ValueCollector { //------------------------------------------ -struct RangeBuilder { - run: Option<(u64, BitmapEntry)>, - runs: Vec<(u64, BitmapEntry)> -} - -impl RangeBuilder { - fn new() -> RangeBuilder { - RangeBuilder { - run: None, - runs: Vec::new(), - } - } - - fn push(&mut self, e: &BitmapEntry) { - match &self.run { - Some((len, e2)) if *e == *e2 => { - self.run = Some((*len + 1, e2.clone())); - }, - Some((len, e2)) => { - self.runs.push((*len, e2.clone())); - self.run = Some((1, e.clone())); - }, - None => { - self.run = Some((1, e.clone())); - } - } - } - - fn complete(&mut self) { - match &self.run { - Some((len, e)) => { - self.runs.push((*len, e.clone())); - }, - None => {} - } - } -} - -//------------------------------------------ - pub fn check(dev: &Path) -> Result<()> { let engine = Arc::new(AsyncIoEngine::new(dev, 256)?); @@ -280,7 +240,7 @@ pub fn check(dev: &Path) -> Result<()> { } // mapping top level - let mut roots = HashMap::new(); + let mut roots = BTreeMap::new(); { let mut visitor = TopLevelVisitor { roots: &mut roots }; let mut w = BTreeWalker::new(engine.clone(), false); @@ -319,7 +279,7 @@ pub fn check(dev: &Path) -> Result<()> { eprintln!("data root: {:?}", root); // overflow btree - let mut overflow: HashMap = HashMap::new(); + let mut overflow: BTreeMap = BTreeMap::new(); { let mut v: ValueCollector = ValueCollector::new(); let mut w = BTreeWalker::new(engine.clone(), false); @@ -337,8 +297,6 @@ pub fn check(dev: &Path) -> Result<()> { let _result = w.walk(&mut v, root.bitmap_root); eprintln!("{} index entries", v.entries.len()); - let mut builder = RangeBuilder::new(); - for i in v.entries { let mut b = Block::new(i.blocknr); engine.read(&mut b)?; @@ -348,26 +306,10 @@ pub fn check(dev: &Path) -> Result<()> { } let bitmap = unpack::(b.get_data())?; - for e in bitmap.entries { - builder.push(&e); + for _e in bitmap.entries { + //builder.push(&e); } } - builder.complete(); - eprintln!("{} ranges", builder.runs.len()); - - let mut counts = BTreeMap::new(); - for (len, _v) in builder.runs { - if let Some(c) = counts.get(&len) { - let new_c = *c + 1; - counts.insert(len, new_c); - } else { - counts.insert(len, 1); - } - } - - for (len, c) in counts { - eprintln!("{}: {}", len, c); - } } Ok(()) From fd0c0ffc1d3b74fa8f962a2626bcd0c4b8ccadaf Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Sat, 8 Aug 2020 16:42:32 +0100 Subject: [PATCH 37/52] [thin_check (rust)] data space map now checked. --- src/block_manager.rs | 64 ++++++++++++-------- src/thin/check.rs | 137 +++++++++++++++++++++++++++++++------------ 2 files changed, 140 insertions(+), 61 deletions(-) diff --git a/src/block_manager.rs b/src/block_manager.rs index 5fc7589..fe68bb7 100644 --- a/src/block_manager.rs +++ b/src/block_manager.rs @@ -135,6 +135,37 @@ impl AsyncIoEngine { }), }) } + + fn read_many_(&self, blocks: &mut [Block]) -> Result<()> { + let mut inner = self.inner.lock().unwrap(); + let count = blocks.len(); + let fd = types::Target::Fd(inner.input.as_raw_fd()); + + for b in blocks.iter_mut() { + let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32) + .offset(b.loc as i64 * BLOCK_SIZE as i64); + + unsafe { + let mut queue = inner.ring.submission().available(); + queue + .push(read_e.build().user_data(1)) + .ok() + .expect("queue is full"); + } + } + + inner.ring.submit_and_wait(count)?; + + let cqes = inner.ring.completion().available().collect::>(); + + // FIXME: return proper errors + assert_eq!(cqes.len(), count); + for c in &cqes { + assert_eq!(c.result(), BLOCK_SIZE as i32); + } + + Ok(()) + } } impl Clone for AsyncIoEngine { @@ -186,33 +217,16 @@ impl IoEngine for AsyncIoEngine { } fn read_many(&self, blocks: &mut Vec) -> Result<()> { - let mut inner = self.inner.lock().unwrap(); - let count = blocks.len(); - let fd = types::Target::Fd(inner.input.as_raw_fd()); + let inner = self.inner.lock().unwrap(); + let queue_len = inner.queue_len as usize; + drop(inner); - for b in blocks.iter_mut() { - let read_e = opcode::Read::new(fd, b.data, BLOCK_SIZE as u32) - .offset(b.loc as i64 * BLOCK_SIZE as i64); - - unsafe { - let mut queue = inner.ring.submission().available(); - queue - .push(read_e.build().user_data(1)) - .ok() - .expect("queue is full"); - } + let mut done = 0; + while done != blocks.len() { + let len = usize::min(blocks.len() - done, queue_len); + self.read_many_(&mut blocks[done..(done + len)])?; + done += len; } - - inner.ring.submit_and_wait(count)?; - - let cqes = inner.ring.completion().available().collect::>(); - - // FIXME: return proper errors - assert_eq!(cqes.len(), count); - for c in &cqes { - assert_eq!(c.result(), BLOCK_SIZE as i32); - } - Ok(()) } } diff --git a/src/thin/check.rs b/src/thin/check.rs index 87a2ee5..40a99b0 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,17 +1,17 @@ use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; -use std::collections::{BTreeMap}; +use std::collections::BTreeMap; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::Instant; use threadpool::ThreadPool; use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; -use crate::pdata::btree::{BTreeWalker, Node, NodeVisitor, Unpack, unpack}; +use crate::checksum; +use crate::pdata::btree::{unpack, BTreeWalker, Node, NodeVisitor, Unpack}; use crate::pdata::space_map::*; use crate::thin::superblock::*; -use crate::checksum; //------------------------------------------ @@ -74,21 +74,26 @@ impl NodeVisitor for BottomLevelVisitor { fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { // FIXME: do other checks - if let Node::Leaf {header: _h, keys: _k, values} = node { + if let Node::Leaf { + header: _h, + keys: _k, + values, + } = node + { if values.len() > 0 { let mut data_sm = self.data_sm.lock().unwrap(); let mut start = values[0].block; let mut len = 1; - + for n in 1..values.len() { - if values[n].block == start + len { - len += 1; - } else { - data_sm.inc(start, len)?; - start = values[n].block; - len = 1; - } + if values[n].block == start + len { + len += 1; + } else { + data_sm.inc(start, len)?; + start = values[n].block; + len = 1; + } } data_sm.inc(start, len)?; @@ -175,7 +180,8 @@ impl NodeVisitor for IndexVisitor { header: _h, keys: _k, values, - } = node { + } = node + { for v in values { // FIXME: check keys are in incremental order let v = v.clone(); @@ -196,9 +202,7 @@ struct ValueCollector { impl ValueCollector { fn new() -> ValueCollector { - ValueCollector { - values: Vec::new(), - } + ValueCollector { values: Vec::new() } } } @@ -208,7 +212,8 @@ impl NodeVisitor for ValueCollector { header: _h, keys, values, - } = node { + } = node + { for n in 0..keys.len() { let k = keys[n]; let v = values[n].clone(); @@ -218,12 +223,49 @@ impl NodeVisitor for ValueCollector { Ok(()) } -} +} //------------------------------------------ +struct OverflowChecker<'a> { + data_sm: &'a dyn SpaceMap, +} + +impl<'a> OverflowChecker<'a> { + fn new(data_sm: &'a dyn SpaceMap) -> OverflowChecker<'a> { + OverflowChecker { data_sm } + } +} + +impl<'a> NodeVisitor for OverflowChecker<'a> { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf { + header: _h, + keys, + values, + } = node + { + for n in 0..keys.len() { + let k = keys[n]; + let v = values[n]; + let expected = self.data_sm.get(k)?; + if expected != v { + return Err(anyhow!("Bad reference count for data block {}. Expected {}, but space map contains {}.", + k, expected, v)); + } + } + } + + Ok(()) + } +} + +//------------------------------------------ + +const MAX_CONCURRENT_IO: u32 = 1024; + pub fn check(dev: &Path) -> Result<()> { - let engine = Arc::new(AsyncIoEngine::new(dev, 256)?); + let engine = Arc::new(AsyncIoEngine::new(dev, MAX_CONCURRENT_IO)?); let now = Instant::now(); let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; @@ -249,6 +291,7 @@ pub fn check(dev: &Path) -> Result<()> { } // mapping bottom level + let data_sm; { // FIXME: with a thread pool we need to return errors another way. let nr_workers = 4; @@ -258,13 +301,13 @@ pub fn check(dev: &Path) -> Result<()> { ))); let root = unpack::(&sb.data_sm_root[0..])?; - let data_sm = core_sm(root.nr_blocks, nr_devs as u32); + data_sm = core_sm(root.nr_blocks, nr_devs as u32); for (thin_id, root) in roots { let mut w = BTreeWalker::new_with_seen(engine.clone(), seen.clone(), false); let data_sm = data_sm.clone(); pool.execute(move || { - let mut v = BottomLevelVisitor {data_sm}; + let mut v = BottomLevelVisitor { data_sm }; let result = w.walk(&mut v, root).expect("walk failed"); // FIXME: return error eprintln!("checked thin_dev {} -> {:?}", thin_id, result); }); @@ -275,39 +318,61 @@ pub fn check(dev: &Path) -> Result<()> { // data space map { + let data_sm = data_sm.lock().unwrap(); let root = unpack::(&sb.data_sm_root[0..])?; eprintln!("data root: {:?}", root); // overflow btree - let mut overflow: BTreeMap = BTreeMap::new(); { - let mut v: ValueCollector = ValueCollector::new(); + let mut v = OverflowChecker::new(&*data_sm); let mut w = BTreeWalker::new(engine.clone(), false); w.walk(&mut v, root.ref_count_root)?; - - for (k, v) in v.values { - overflow.insert(k, v); - } } - eprintln!("{} overflow entries", overflow.len()); // Bitmaps - let mut v = IndexVisitor {entries: Vec::new()}; + let mut v = IndexVisitor { + entries: Vec::new(), + }; let mut w = BTreeWalker::new(engine.clone(), false); let _result = w.walk(&mut v, root.bitmap_root); eprintln!("{} index entries", v.entries.len()); - for i in v.entries { - let mut b = Block::new(i.blocknr); - engine.read(&mut b)?; - + let mut blocks = Vec::new(); + for i in &v.entries { + blocks.push(Block::new(i.blocknr)); + } + + engine.read_many(&mut blocks)?; + + let mut blocknr = 0; + for (n, _i) in v.entries.iter().enumerate() { + let b = &blocks[n]; if checksum::metadata_block_type(&b.get_data()) != checksum::BT::BITMAP { - return Err(anyhow!("Index entry points to block ({}) that isn't a bitmap", b.loc)); + return Err(anyhow!( + "Index entry points to block ({}) that isn't a bitmap", + b.loc + )); } let bitmap = unpack::(b.get_data())?; - for _e in bitmap.entries { - //builder.push(&e); + for e in bitmap.entries { + match e { + BitmapEntry::Small(actual) => { + let expected = data_sm.get(blocknr)?; + if actual != expected as u8 { + return Err(anyhow!("Bad reference count for data block {}. Expected {}, but space map contains {}.", + blocknr, expected, actual)); + } + } + BitmapEntry::Overflow => { + let expected = data_sm.get(blocknr)?; + if expected < 3 { + return Err(anyhow!("Bad reference count for data block {}. Expected {}, but space map says it's >= 3.", + blocknr, expected)); + } + } + } + blocknr += 1; } } } From 08e3ea948efeddbc9d495342925172507605af1c Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 08:29:32 +0100 Subject: [PATCH 38/52] [thin_check (rust)] rename block_manager.rs -> io_engine.rs --- src/{block_manager.rs => io_engine.rs} | 0 src/lib.rs | 2 +- src/pdata/btree.rs | 2 +- src/pdata/space_map.rs | 2 +- src/thin/check.rs | 2 +- src/thin/superblock.rs | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename src/{block_manager.rs => io_engine.rs} (100%) diff --git a/src/block_manager.rs b/src/io_engine.rs similarity index 100% rename from src/block_manager.rs rename to src/io_engine.rs diff --git a/src/lib.rs b/src/lib.rs index ac0780e..529b14c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,7 @@ extern crate quickcheck; #[cfg(test)] extern crate quickcheck_macros; -pub mod block_manager; +pub mod io_engine; pub mod cache; pub mod checksum; pub mod file_utils; diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index a5c7ba5..9a61aff 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -3,7 +3,7 @@ use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; use std::sync::{Arc, Mutex}; -use crate::block_manager::*; +use crate::io_engine::*; use crate::checksum; // FIXME: check that keys are in ascending order between nodes. diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 6b77aac..55c7457 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -2,7 +2,7 @@ use anyhow::{anyhow, Result}; use nom::{number::complete::*, IResult}; use std::sync::{Arc, Mutex}; -use crate::block_manager::*; +use crate::io_engine::*; use crate::pdata::btree::Unpack; //------------------------------------------ diff --git a/src/thin/check.rs b/src/thin/check.rs index 40a99b0..3ef76c5 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use threadpool::ThreadPool; -use crate::block_manager::{AsyncIoEngine, Block, IoEngine}; +use crate::io_engine::{AsyncIoEngine, Block, IoEngine}; use crate::checksum; use crate::pdata::btree::{unpack, BTreeWalker, Node, NodeVisitor, Unpack}; use crate::pdata::space_map::*; diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index 8c1e738..bd3e8ba 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -1,4 +1,4 @@ -use crate::block_manager::*; +use crate::io_engine::*; use anyhow::{anyhow, Result}; use nom::{bytes::complete::*, number::complete::*, IResult}; From f0df17af9e06e3754c67441ceebd4a24f76c6bd7 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 08:59:02 +0100 Subject: [PATCH 39/52] [thin_check (rust)] Get SyncIoEngine working again. --- src/io_engine.rs | 24 +++++++++++++----------- src/pdata/btree.rs | 6 +++--- src/thin/check.rs | 5 +++-- src/thin/superblock.rs | 2 +- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/io_engine.rs b/src/io_engine.rs index fe68bb7..5b0a313 100644 --- a/src/io_engine.rs +++ b/src/io_engine.rs @@ -4,7 +4,7 @@ use io_uring::IoUring; use std::alloc::{alloc, dealloc, Layout}; use std::fs::File; use std::fs::OpenOptions; -use std::io; +use std::io::{self, Seek, Read}; use std::os::unix::fs::OpenOptionsExt; use std::os::unix::io::{AsRawFd, RawFd}; use std::path::Path; @@ -60,10 +60,9 @@ fn get_nr_blocks(path: &Path) -> io::Result { //------------------------------------------ -/* pub struct SyncIoEngine { nr_blocks: u64, - input: File, + input: Mutex, } impl SyncIoEngine { @@ -76,7 +75,7 @@ impl SyncIoEngine { Ok(SyncIoEngine { nr_blocks: get_nr_blocks(path)?, - input, + input: Mutex::new(input), }) } } @@ -86,23 +85,26 @@ impl IoEngine for SyncIoEngine { self.nr_blocks } - fn read(&mut self, b: &mut Block) -> Result<()> { - self.input - .seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; - self.input.read_exact(&mut b.get_data())?; + + fn read(&self, b: &mut Block) -> Result<()> { + let mut input = self.input.lock().unwrap(); + input.seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; + input.read_exact(&mut b.get_data())?; Ok(()) } - fn read_many(&mut self, blocks: &mut Vec) -> Result<()> { + fn read_many(&self, blocks: &mut Vec) -> Result<()> { + let mut input = self.input.lock().unwrap(); for b in blocks { - self.read(b); + input.seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; + input.read_exact(&mut b.get_data())?; } Ok(()) } } -*/ + //------------------------------------------ pub struct AsyncIoEngine_ { diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 9a61aff..50bf8f8 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -189,13 +189,13 @@ pub trait NodeVisitor { #[derive(Clone)] pub struct BTreeWalker { - pub engine: Arc, + pub engine: Arc, pub seen: Arc>, ignore_non_fatal: bool, } impl BTreeWalker { - pub fn new(engine: Arc, ignore_non_fatal: bool) -> BTreeWalker { + pub fn new(engine: Arc, ignore_non_fatal: bool) -> BTreeWalker { let nr_blocks = engine.get_nr_blocks() as usize; let r: BTreeWalker = BTreeWalker { engine, @@ -206,7 +206,7 @@ impl BTreeWalker { } pub fn new_with_seen( - engine: Arc, + engine: Arc, seen: Arc>, ignore_non_fatal: bool, ) -> BTreeWalker { diff --git a/src/thin/check.rs b/src/thin/check.rs index 3ef76c5..a93eaaf 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use threadpool::ThreadPool; -use crate::io_engine::{AsyncIoEngine, Block, IoEngine}; +use crate::io_engine::{AsyncIoEngine, SyncIoEngine, Block, IoEngine}; use crate::checksum; use crate::pdata::btree::{unpack, BTreeWalker, Node, NodeVisitor, Unpack}; use crate::pdata::space_map::*; @@ -265,7 +265,8 @@ impl<'a> NodeVisitor for OverflowChecker<'a> { const MAX_CONCURRENT_IO: u32 = 1024; pub fn check(dev: &Path) -> Result<()> { - let engine = Arc::new(AsyncIoEngine::new(dev, MAX_CONCURRENT_IO)?); + //let engine = Arc::new(AsyncIoEngine::new(dev, MAX_CONCURRENT_IO)?); + let engine: Arc = Arc::new(SyncIoEngine::new(dev)?); let now = Instant::now(); let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; diff --git a/src/thin/superblock.rs b/src/thin/superblock.rs index bd3e8ba..3bb2637 100644 --- a/src/thin/superblock.rs +++ b/src/thin/superblock.rs @@ -85,7 +85,7 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> { )) } -pub fn read_superblock(engine: &E, loc: u64) -> Result { +pub fn read_superblock(engine: &dyn IoEngine, loc: u64) -> Result { let mut b = Block::new(loc); engine.read(&mut b)?; From 0f865856ed9d23c569b47e712f59dc691c117684 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 10:44:47 +0100 Subject: [PATCH 40/52] [thin_check (rust)] Improve SyncIoEngine. Now opens the file multiple times so different threads can do io in parallel. --- src/io_engine.rs | 45 ++++++++++++++++++++++++++++++++++++--------- src/thin/check.rs | 7 ++++--- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/io_engine.rs b/src/io_engine.rs index 5b0a313..e89fae7 100644 --- a/src/io_engine.rs +++ b/src/io_engine.rs @@ -4,11 +4,11 @@ use io_uring::IoUring; use std::alloc::{alloc, dealloc, Layout}; use std::fs::File; use std::fs::OpenOptions; -use std::io::{self, Seek, Read}; +use std::io::{self, Read, Seek}; use std::os::unix::fs::OpenOptionsExt; use std::os::unix::io::{AsRawFd, RawFd}; use std::path::Path; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, Condvar}; //------------------------------------------ @@ -62,22 +62,48 @@ fn get_nr_blocks(path: &Path) -> io::Result { pub struct SyncIoEngine { nr_blocks: u64, - input: Mutex, + files: Mutex>, + cvar: Condvar, } impl SyncIoEngine { - pub fn new(path: &Path) -> Result { - let input = OpenOptions::new() + fn open_file(path: &Path) -> Result { + let file = OpenOptions::new() .read(true) .write(false) .custom_flags(libc::O_DIRECT) .open(path)?; + Ok(file) + } + + pub fn new(path: &Path, nr_files: usize) -> Result { + let mut files = Vec::new(); + for _n in 0..nr_files { + files.push(SyncIoEngine::open_file(path)?); + } + Ok(SyncIoEngine { nr_blocks: get_nr_blocks(path)?, - input: Mutex::new(input), + files: Mutex::new(files), + cvar: Condvar::new(), }) } + + fn get(&self) -> File { + let mut files = self.files.lock().unwrap(); + + while files.len() == 0 { + files = self.cvar.wait(files).unwrap(); + } + files.pop().unwrap() + } + + fn put(&self, f: File) { + let mut files = self.files.lock().unwrap(); + files.push(f); + self.cvar.notify_one(); + } } impl IoEngine for SyncIoEngine { @@ -85,21 +111,22 @@ impl IoEngine for SyncIoEngine { self.nr_blocks } - fn read(&self, b: &mut Block) -> Result<()> { - let mut input = self.input.lock().unwrap(); + let mut input = self.get(); input.seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; input.read_exact(&mut b.get_data())?; + self.put(input); Ok(()) } fn read_many(&self, blocks: &mut Vec) -> Result<()> { - let mut input = self.input.lock().unwrap(); + let mut input = self.get(); for b in blocks { input.seek(io::SeekFrom::Start(b.loc * BLOCK_SIZE as u64))?; input.read_exact(&mut b.get_data())?; } + self.put(input); Ok(()) } diff --git a/src/thin/check.rs b/src/thin/check.rs index a93eaaf..508a206 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -265,8 +265,9 @@ impl<'a> NodeVisitor for OverflowChecker<'a> { const MAX_CONCURRENT_IO: u32 = 1024; pub fn check(dev: &Path) -> Result<()> { - //let engine = Arc::new(AsyncIoEngine::new(dev, MAX_CONCURRENT_IO)?); - let engine: Arc = Arc::new(SyncIoEngine::new(dev)?); + let nr_threads = 4; + let engine = Arc::new(AsyncIoEngine::new(dev, MAX_CONCURRENT_IO)?); + //let engine: Arc = Arc::new(SyncIoEngine::new(dev, nr_threads)?); let now = Instant::now(); let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; @@ -295,7 +296,7 @@ pub fn check(dev: &Path) -> Result<()> { let data_sm; { // FIXME: with a thread pool we need to return errors another way. - let nr_workers = 4; + let nr_workers = nr_threads; let pool = ThreadPool::new(nr_workers); let seen = Arc::new(Mutex::new(FixedBitSet::with_capacity( engine.get_nr_blocks() as usize, From 4e4b7ca2b12ba472bc2747487f5ea0c9773936d0 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 11:24:50 +0100 Subject: [PATCH 41/52] [thin_check (rust)] add --sync-io flag Makes it easier to switch between engines --- src/bin/thin_check.rs | 15 ++++++++++++++- src/thin/check.rs | 21 +++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/bin/thin_check.rs b/src/bin/thin_check.rs index 46ae7ef..2c74c46 100644 --- a/src/bin/thin_check.rs +++ b/src/bin/thin_check.rs @@ -5,6 +5,7 @@ use clap::{App, Arg}; use std::path::Path; use std::process; use thinp::file_utils; +use thinp::thin::check::{check, ThinCheckOptions}; use std::process::exit; @@ -56,6 +57,13 @@ fn main() { .help("Specify the input device to check") .required(true) .index(1), + ) + .arg( + Arg::with_name("SYNC_IO") + .help("Force use of synchronous io") + .long("sync-io") + .value_name("SYNC_IO") + .takes_value(false), ); let matches = parser.get_matches(); @@ -66,7 +74,12 @@ fn main() { exit(1); } - if let Err(reason) = thinp::thin::check::check(&input_file) { + let opts = ThinCheckOptions { + dev: &input_file, + async_io: !matches.is_present("SYNC_IO"), + }; + + if let Err(reason) = check(&opts) { println!("Application error: {}", reason); process::exit(1); } diff --git a/src/thin/check.rs b/src/thin/check.rs index 508a206..fb73321 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -264,10 +264,23 @@ impl<'a> NodeVisitor for OverflowChecker<'a> { const MAX_CONCURRENT_IO: u32 = 1024; -pub fn check(dev: &Path) -> Result<()> { - let nr_threads = 4; - let engine = Arc::new(AsyncIoEngine::new(dev, MAX_CONCURRENT_IO)?); - //let engine: Arc = Arc::new(SyncIoEngine::new(dev, nr_threads)?); +pub struct ThinCheckOptions<'a> { + pub dev: &'a Path, + pub async_io: bool, +} + +pub fn check(opts: &ThinCheckOptions) -> Result<()> { + let engine: Arc; + + let nr_threads; + if opts.async_io { + nr_threads = std::cmp::min(4, num_cpus::get()); + engine = Arc::new(AsyncIoEngine::new(opts.dev, MAX_CONCURRENT_IO)?); + } else { + eprintln!("falling back to synchronous io"); + nr_threads = num_cpus::get() * 2; + engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads)?); + } let now = Instant::now(); let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; From b915257e10b8fdf523429b2fceaf5852fdbd0358 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 12:30:12 +0100 Subject: [PATCH 42/52] [thin_check (rust)] Fix race in btree walking. The seen bitset was locked once to test, and separately to insert. --- src/pdata/btree.rs | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 50bf8f8..0e2f8df 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -3,8 +3,8 @@ use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; use std::sync::{Arc, Mutex}; -use crate::io_engine::*; use crate::checksum; +use crate::io_engine::*; // FIXME: check that keys are in ascending order between nodes. @@ -20,9 +20,7 @@ pub trait Unpack { pub fn unpack(data: &[u8]) -> Result { match U::unpack(data) { - Err(_e) => { - Err(anyhow!("couldn't parse SMRoot")) - }, + Err(_e) => Err(anyhow!("couldn't parse SMRoot")), Ok((_i, v)) => Ok(v), } } @@ -222,16 +220,27 @@ impl BTreeWalker { } } + fn is_seen(&self, b: u64) -> bool { + let mut seen = self.seen.lock().unwrap(); + if !seen[b as usize] { + seen.insert(b as usize); + return false; + } + + true + } + fn walk_nodes(&mut self, visitor: &mut NV, bs: &[u64]) -> Result<()> where NV: NodeVisitor, V: Unpack, { let mut blocks = Vec::new(); - let seen = self.seen.lock().unwrap(); + let mut seen = self.seen.lock().unwrap(); for b in bs { if !seen[*b as usize] { blocks.push(Block::new(*b)); + seen.insert(*b as usize); } } drop(seen); @@ -250,10 +259,6 @@ impl BTreeWalker { NV: NodeVisitor, V: Unpack, { - let mut seen = self.seen.lock().unwrap(); - seen.insert(b.loc as usize); - drop(seen); - let bt = checksum::metadata_block_type(b.get_data()); if bt != checksum::BT::NODE { return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt)); @@ -279,7 +284,11 @@ impl BTreeWalker { NV: NodeVisitor, V: Unpack, { - self.walk_node(visitor, &root, true) + if self.is_seen(root.loc) { + Ok(()) + } else { + self.walk_node(visitor, &root, true) + } } pub fn walk(&mut self, visitor: &mut NV, root: u64) -> Result<()> @@ -287,9 +296,13 @@ impl BTreeWalker { NV: NodeVisitor, V: Unpack, { - let mut root = Block::new(root); - self.engine.read(&mut root)?; - self.walk_node(visitor, &root, true) + if self.is_seen(root) { + Ok(()) + } else { + let mut root = Block::new(root); + self.engine.read(&mut root)?; + self.walk_node(visitor, &root, true) + } } } From d5444d2255ced67cbc919489178e20d07058147a Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 12:55:05 +0100 Subject: [PATCH 43/52] [thin_check (rust)] sm bitmap entries were being unpacked incorrectly. --- src/pdata/space_map.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 55c7457..9d46b32 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -134,11 +134,14 @@ impl Unpack for Bitmap { let val = word & 0x3; word >>= 2; - if val < 3 { - entries.push(BitmapEntry::Small(val as u8)); - } else { - entries.push(BitmapEntry::Overflow); - } + // The bits are stored with the high bit at b * 2 + 1, + // and low at b *2. So we have to interpret this val. + entries.push(match val { + 0 => BitmapEntry::Small(0), + 1 => BitmapEntry::Small(2), + 2 => BitmapEntry::Small(1), + _ => BitmapEntry::Overflow, + }); } i = tmp; @@ -173,7 +176,7 @@ where impl SpaceMap for CoreSpaceMap where V: Copy + Default + std::ops::AddAssign + From + Into, - { +{ fn get(&self, b: u64) -> Result { Ok(self.counts[b as usize].into()) } From cbc9c2c72a5e4ce56c764873c430c649cd35b6f7 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 12:56:41 +0100 Subject: [PATCH 44/52] [thin_check (rust)] Improve data_sm handling --- src/thin/check.rs | 79 ++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index fb73321..07a63eb 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -4,11 +4,10 @@ use nom::{number::complete::*, IResult}; use std::collections::BTreeMap; use std::path::Path; use std::sync::{Arc, Mutex}; -use std::time::Instant; use threadpool::ThreadPool; -use crate::io_engine::{AsyncIoEngine, SyncIoEngine, Block, IoEngine}; use crate::checksum; +use crate::io_engine::{AsyncIoEngine, Block, IoEngine, SyncIoEngine}; use crate::pdata::btree::{unpack, BTreeWalker, Node, NodeVisitor, Unpack}; use crate::pdata::space_map::*; use crate::thin::superblock::*; @@ -80,24 +79,27 @@ impl NodeVisitor for BottomLevelVisitor { values, } = node { - if values.len() > 0 { - let mut data_sm = self.data_sm.lock().unwrap(); - - let mut start = values[0].block; - let mut len = 1; - - for n in 1..values.len() { - if values[n].block == start + len { - len += 1; - } else { - data_sm.inc(start, len)?; - start = values[n].block; - len = 1; - } - } - - data_sm.inc(start, len)?; + if values.len() == 0 { + return Ok(()); } + + let mut data_sm = self.data_sm.lock().unwrap(); + + let mut start = values[0].block; + let mut len = 1; + + for n in 1..values.len() { + let block = values[n].block; + if block == start + len { + len += 1; + } else { + data_sm.inc(start, len)?; + start = block; + len = 1; + } + } + + data_sm.inc(start, len)?; } Ok(()) @@ -282,9 +284,8 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads)?); } - let now = Instant::now(); + // superblock let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; - eprintln!("{:?}", sb); // device details let nr_devs; @@ -302,7 +303,6 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let mut visitor = TopLevelVisitor { roots: &mut roots }; let mut w = BTreeWalker::new(engine.clone(), false); let _result = w.walk(&mut visitor, sb.mapping_root)?; - println!("read mapping tree in {} ms", now.elapsed().as_millis()); } // mapping bottom level @@ -323,8 +323,17 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let data_sm = data_sm.clone(); pool.execute(move || { let mut v = BottomLevelVisitor { data_sm }; - let result = w.walk(&mut v, root).expect("walk failed"); // FIXME: return error - eprintln!("checked thin_dev {} -> {:?}", thin_id, result); + + // FIXME: return error + match w.walk(&mut v, root) { + Err(e) => { + eprintln!("walk failed {:?}", e); + std::process::abort(); + } + Ok(result) => { + eprintln!("checked thin_dev {} -> {:?}", thin_id, result); + } + } }); } @@ -333,8 +342,9 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // data space map { - let data_sm = data_sm.lock().unwrap(); + let mut data_sm = data_sm.lock().unwrap(); let root = unpack::(&sb.data_sm_root[0..])?; + let nr_data_blocks = root.nr_blocks; eprintln!("data root: {:?}", root); // overflow btree @@ -359,6 +369,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { engine.read_many(&mut blocks)?; + let mut fail = false; let mut blocknr = 0; for (n, _i) in v.entries.iter().enumerate() { let b = &blocks[n]; @@ -371,25 +382,35 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let bitmap = unpack::(b.get_data())?; for e in bitmap.entries { + if blocknr >= nr_data_blocks { + break; + } + match e { BitmapEntry::Small(actual) => { let expected = data_sm.get(blocknr)?; if actual != expected as u8 { - return Err(anyhow!("Bad reference count for data block {}. Expected {}, but space map contains {}.", - blocknr, expected, actual)); + eprintln!("Bad reference count for data block {}. Expected {}, but space map contains {}.", + blocknr, expected, actual); + fail = true; } } BitmapEntry::Overflow => { let expected = data_sm.get(blocknr)?; if expected < 3 { - return Err(anyhow!("Bad reference count for data block {}. Expected {}, but space map says it's >= 3.", - blocknr, expected)); + eprintln!("Bad reference count for data block {}. Expected {}, but space map says it's >= 3.", + blocknr, expected); + fail = true; } } } blocknr += 1; } } + + if fail { + return Err(anyhow!("Inconsistent data space map")); + } } Ok(()) From e28c602c3d3bfb800abb3d103d70212b7cff8281 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 14:45:35 +0100 Subject: [PATCH 45/52] [thin_check (rust)] factor out btree_to_map() fn --- src/pdata/btree.rs | 42 ++++++++++++++++++++ src/thin/check.rs | 98 +++++++++------------------------------------- 2 files changed, 60 insertions(+), 80 deletions(-) diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 0e2f8df..21b6ce2 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -2,6 +2,7 @@ use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; use std::sync::{Arc, Mutex}; +use std::collections::BTreeMap; use crate::checksum; use crate::io_engine::*; @@ -307,3 +308,44 @@ impl BTreeWalker { } //------------------------------------------ + +struct ValueCollector { + values: BTreeMap, +} + +impl ValueCollector { + fn new() -> ValueCollector { + ValueCollector { values: BTreeMap::new() } + } +} + +impl NodeVisitor for ValueCollector { + fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { + if let Node::Leaf { + header: _h, + keys, + values, + } = node + { + for n in 0..keys.len() { + let k = keys[n]; + let v = values[n].clone(); + self.values.insert(k, v); + } + } + + Ok(()) + } +} + +pub fn btree_to_map(engine: Arc, + ignore_non_fatal: bool, + root: u64) -> Result> { + let mut walker = BTreeWalker::new(engine, ignore_non_fatal); + let mut visitor = ValueCollector::::new(); + + walker.walk(&mut visitor, root)?; + Ok(visitor.values) +} + +//------------------------------------------ diff --git a/src/thin/check.rs b/src/thin/check.rs index 07a63eb..9502270 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -8,7 +8,7 @@ use threadpool::ThreadPool; use crate::checksum; use crate::io_engine::{AsyncIoEngine, Block, IoEngine, SyncIoEngine}; -use crate::pdata::btree::{unpack, BTreeWalker, Node, NodeVisitor, Unpack}; +use crate::pdata::btree::{btree_to_map, unpack, BTreeWalker, Node, NodeVisitor, Unpack}; use crate::pdata::space_map::*; use crate::thin::superblock::*; @@ -139,37 +139,6 @@ impl Unpack for DeviceDetail { } } -struct DeviceVisitor { - devs: BTreeMap, -} - -impl DeviceVisitor { - pub fn new() -> DeviceVisitor { - DeviceVisitor { - devs: BTreeMap::new(), - } - } -} - -impl NodeVisitor for DeviceVisitor { - fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { - if let Node::Leaf { - header: _h, - keys, - values, - } = node - { - for n in 0..keys.len() { - let k = keys[n] as u32; - let v = values[n].clone(); - self.devs.insert(k, v); - } - } - - Ok(()) - } -} - //------------------------------------------ struct IndexVisitor { @@ -197,38 +166,6 @@ impl NodeVisitor for IndexVisitor { //------------------------------------------ -// FIXME: move to btree -struct ValueCollector { - values: Vec<(u64, V)>, -} - -impl ValueCollector { - fn new() -> ValueCollector { - ValueCollector { values: Vec::new() } - } -} - -impl NodeVisitor for ValueCollector { - fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { - if let Node::Leaf { - header: _h, - keys, - values, - } = node - { - for n in 0..keys.len() { - let k = keys[n]; - let v = values[n].clone(); - self.values.push((k, v)); - } - } - - Ok(()) - } -} - -//------------------------------------------ - struct OverflowChecker<'a> { data_sm: &'a dyn SpaceMap, } @@ -288,22 +225,12 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; // device details - let nr_devs; - { - let mut visitor = DeviceVisitor::new(); - let mut w = BTreeWalker::new(engine.clone(), false); - w.walk(&mut visitor, sb.details_root)?; - nr_devs = visitor.devs.len(); - println!("found {} devices", visitor.devs.len()); - } + let devs = btree_to_map::(engine.clone(), false, sb.details_root)?; + let nr_devs = devs.len(); + println!("found {} devices", nr_devs); // mapping top level - let mut roots = BTreeMap::new(); - { - let mut visitor = TopLevelVisitor { roots: &mut roots }; - let mut w = BTreeWalker::new(engine.clone(), false); - let _result = w.walk(&mut visitor, sb.mapping_root)?; - } + let roots = btree_to_map::(engine.clone(), false, sb.mapping_root)?; // mapping bottom level let data_sm; @@ -342,7 +269,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // data space map { - let mut data_sm = data_sm.lock().unwrap(); + let data_sm = data_sm.lock().unwrap(); let root = unpack::(&sb.data_sm_root[0..])?; let nr_data_blocks = root.nr_blocks; eprintln!("data root: {:?}", root); @@ -369,6 +296,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { engine.read_many(&mut blocks)?; + let mut leaks = 0; let mut fail = false; let mut blocknr = 0; for (n, _i) in v.entries.iter().enumerate() { @@ -389,7 +317,10 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { match e { BitmapEntry::Small(actual) => { let expected = data_sm.get(blocknr)?; - if actual != expected as u8 { + if actual == 1 && expected == 0 { + eprintln!("Data block {} leaked.", blocknr); + leaks += 1; + } else if actual != expected as u8 { eprintln!("Bad reference count for data block {}. Expected {}, but space map contains {}.", blocknr, expected, actual); fail = true; @@ -408,6 +339,13 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { } } + if leaks > 0 { + eprintln!( + "{} data blocks have leaked. Use --auto-repair to fix.", + leaks + ); + } + if fail { return Err(anyhow!("Inconsistent data space map")); } From 55ee4bfad83f42b86f4f788c2466a1c1e49fbc56 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 14:56:39 +0100 Subject: [PATCH 46/52] [thin_check (rust)] replace IndexVisitor with a call to btree_to_map --- src/thin/check.rs | 37 ++++--------------------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index 9502270..73094c9 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -141,31 +141,6 @@ impl Unpack for DeviceDetail { //------------------------------------------ -struct IndexVisitor { - entries: Vec, -} - -impl NodeVisitor for IndexVisitor { - fn visit(&mut self, _w: &BTreeWalker, _b: &Block, node: &Node) -> Result<()> { - if let Node::Leaf { - header: _h, - keys: _k, - values, - } = node - { - for v in values { - // FIXME: check keys are in incremental order - let v = v.clone(); - self.entries.push(v); - } - } - - Ok(()) - } -} - -//------------------------------------------ - struct OverflowChecker<'a> { data_sm: &'a dyn SpaceMap, } @@ -282,15 +257,11 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { } // Bitmaps - let mut v = IndexVisitor { - entries: Vec::new(), - }; - let mut w = BTreeWalker::new(engine.clone(), false); - let _result = w.walk(&mut v, root.bitmap_root); - eprintln!("{} index entries", v.entries.len()); + let entries = btree_to_map::(engine.clone(), false, root.bitmap_root)?; + eprintln!("{} index entries", entries.len()); let mut blocks = Vec::new(); - for i in &v.entries { + for (_k, i) in &entries { blocks.push(Block::new(i.blocknr)); } @@ -299,7 +270,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let mut leaks = 0; let mut fail = false; let mut blocknr = 0; - for (n, _i) in v.entries.iter().enumerate() { + for n in 0..entries.len() { let b = &blocks[n]; if checksum::metadata_block_type(&b.get_data()) != checksum::BT::BITMAP { return Err(anyhow!( From 50bde693a144b4e819f8421a939112193d6911ee Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 10 Aug 2020 15:42:10 +0100 Subject: [PATCH 47/52] [thin_check (rust)] Factor out pdata/unpack --- src/pdata/btree.rs | 52 ++++++++---------------------------------- src/pdata/mod.rs | 1 + src/pdata/space_map.rs | 2 +- src/pdata/unpack.rs | 43 ++++++++++++++++++++++++++++++++++ src/thin/check.rs | 11 ++++++--- 5 files changed, 63 insertions(+), 46 deletions(-) create mode 100644 src/pdata/unpack.rs diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 21b6ce2..2706bb6 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -1,31 +1,17 @@ use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; -use std::sync::{Arc, Mutex}; use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; use crate::checksum; use crate::io_engine::*; +use crate::pdata::unpack::*; // FIXME: check that keys are in ascending order between nodes. //------------------------------------------ -pub trait Unpack { - // The size of the value when on disk. - fn disk_size() -> u32; - fn unpack(data: &[u8]) -> IResult<&[u8], Self> - where - Self: std::marker::Sized; -} - -pub fn unpack(data: &[u8]) -> Result { - match U::unpack(data) { - Err(_e) => Err(anyhow!("couldn't parse SMRoot")), - Ok((_i, v)) => Ok(v), - } -} - const NODE_HEADER_SIZE: usize = 32; pub struct NodeHeader { @@ -160,28 +146,6 @@ pub fn unpack_node( //------------------------------------------ -impl Unpack for u64 { - fn disk_size() -> u32 { - 8 - } - - fn unpack(i: &[u8]) -> IResult<&[u8], u64> { - le_u64(i) - } -} - -impl Unpack for u32 { - fn disk_size() -> u32 { - 4 - } - - fn unpack(i: &[u8]) -> IResult<&[u8], u32> { - le_u32(i) - } -} - -//------------------------------------------ - pub trait NodeVisitor { fn visit(&mut self, w: &BTreeWalker, b: &Block, node: &Node) -> Result<()>; } @@ -315,7 +279,9 @@ struct ValueCollector { impl ValueCollector { fn new() -> ValueCollector { - ValueCollector { values: BTreeMap::new() } + ValueCollector { + values: BTreeMap::new(), + } } } @@ -338,9 +304,11 @@ impl NodeVisitor for ValueCollector { } } -pub fn btree_to_map(engine: Arc, - ignore_non_fatal: bool, - root: u64) -> Result> { +pub fn btree_to_map( + engine: Arc, + ignore_non_fatal: bool, + root: u64, +) -> Result> { let mut walker = BTreeWalker::new(engine, ignore_non_fatal); let mut visitor = ValueCollector::::new(); diff --git a/src/pdata/mod.rs b/src/pdata/mod.rs index 3e36813..6e473e2 100644 --- a/src/pdata/mod.rs +++ b/src/pdata/mod.rs @@ -1,3 +1,4 @@ pub mod btree; pub mod space_map; +pub mod unpack; diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 9d46b32..91ac606 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -3,7 +3,7 @@ use nom::{number::complete::*, IResult}; use std::sync::{Arc, Mutex}; use crate::io_engine::*; -use crate::pdata::btree::Unpack; +use crate::pdata::unpack::Unpack; //------------------------------------------ diff --git a/src/pdata/unpack.rs b/src/pdata/unpack.rs new file mode 100644 index 0000000..c2a80b5 --- /dev/null +++ b/src/pdata/unpack.rs @@ -0,0 +1,43 @@ +use anyhow::{anyhow, Result}; +use nom::{number::complete::*, IResult}; + +//------------------------------------------ + +pub trait Unpack { + // The size of the value when on disk. + fn disk_size() -> u32; + fn unpack(data: &[u8]) -> IResult<&[u8], Self> + where + Self: std::marker::Sized; +} + +pub fn unpack(data: &[u8]) -> Result { + match U::unpack(data) { + Err(_e) => Err(anyhow!("couldn't parse SMRoot")), + Ok((_i, v)) => Ok(v), + } +} + +//------------------------------------------ + +impl Unpack for u64 { + fn disk_size() -> u32 { + 8 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], u64> { + le_u64(i) + } +} + +impl Unpack for u32 { + fn disk_size() -> u32 { + 4 + } + + fn unpack(i: &[u8]) -> IResult<&[u8], u32> { + le_u32(i) + } +} + +//------------------------------------------ diff --git a/src/thin/check.rs b/src/thin/check.rs index 73094c9..e7110c2 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -8,7 +8,8 @@ use threadpool::ThreadPool; use crate::checksum; use crate::io_engine::{AsyncIoEngine, Block, IoEngine, SyncIoEngine}; -use crate::pdata::btree::{btree_to_map, unpack, BTreeWalker, Node, NodeVisitor, Unpack}; +use crate::pdata::unpack::*; +use crate::pdata::btree::{btree_to_map, BTreeWalker, Node, NodeVisitor}; use crate::pdata::space_map::*; use crate::thin::superblock::*; @@ -207,7 +208,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // mapping top level let roots = btree_to_map::(engine.clone(), false, sb.mapping_root)?; - // mapping bottom level + // Check the mappings filling in the data_sm as we go. let data_sm; { // FIXME: with a thread pool we need to return errors another way. @@ -242,7 +243,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { pool.join(); } - // data space map + // Check the data space map. { let data_sm = data_sm.lock().unwrap(); let root = unpack::(&sb.data_sm_root[0..])?; @@ -265,6 +266,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { blocks.push(Block::new(i.blocknr)); } + // FIXME: we should do this in batches engine.read_many(&mut blocks)?; let mut leaks = 0; @@ -322,6 +324,9 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { } } + // Check the metadata space map. + + Ok(()) } From 34425521e257c365d692cecc5e125c201ab9a040 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 11 Aug 2020 10:50:43 +0100 Subject: [PATCH 48/52] [thin_check (rust)] change BTreeWalker to use a space map rather than seen bitset --- src/pdata/btree.rs | 58 ++++++++++++++++++++++++------------------ src/pdata/space_map.rs | 48 +++++++++++++++++++++++++++++++++- src/thin/check.rs | 28 +++++++++++++------- 3 files changed, 99 insertions(+), 35 deletions(-) diff --git a/src/pdata/btree.rs b/src/pdata/btree.rs index 2706bb6..d22956d 100644 --- a/src/pdata/btree.rs +++ b/src/pdata/btree.rs @@ -1,11 +1,11 @@ use anyhow::{anyhow, Result}; -use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use crate::checksum; use crate::io_engine::*; +use crate::pdata::space_map::*; use crate::pdata::unpack::*; // FIXME: check that keys are in ascending order between nodes. @@ -153,7 +153,7 @@ pub trait NodeVisitor { #[derive(Clone)] pub struct BTreeWalker { pub engine: Arc, - pub seen: Arc>, + pub sm: Arc>, ignore_non_fatal: bool, } @@ -162,37 +162,35 @@ impl BTreeWalker { let nr_blocks = engine.get_nr_blocks() as usize; let r: BTreeWalker = BTreeWalker { engine, - seen: Arc::new(Mutex::new(FixedBitSet::with_capacity(nr_blocks))), + sm: Arc::new(Mutex::new(RestrictedSpaceMap::new(nr_blocks as u64))), ignore_non_fatal, }; r } - pub fn new_with_seen( + pub fn new_with_sm( engine: Arc, - seen: Arc>, + sm: Arc>, ignore_non_fatal: bool, - ) -> BTreeWalker { + ) -> Result { { - let seen = seen.lock().unwrap(); - assert_eq!(seen.len(), engine.get_nr_blocks() as usize); + let sm = sm.lock().unwrap(); + assert_eq!(sm.get_nr_blocks()?, engine.get_nr_blocks()); } - BTreeWalker { + Ok(BTreeWalker { engine, - seen, + sm, ignore_non_fatal, - } + }) } - fn is_seen(&self, b: u64) -> bool { - let mut seen = self.seen.lock().unwrap(); - if !seen[b as usize] { - seen.insert(b as usize); - return false; - } - - true + // Atomically increments the ref count, and returns the _old_ count. + fn sm_inc(&self, b: u64) -> Result { + let mut sm = self.sm.lock().unwrap(); + let count = sm.get(b)?; + sm.inc(b, 1)?; + Ok(count) } fn walk_nodes(&mut self, visitor: &mut NV, bs: &[u64]) -> Result<()> @@ -201,14 +199,11 @@ impl BTreeWalker { V: Unpack, { let mut blocks = Vec::new(); - let mut seen = self.seen.lock().unwrap(); for b in bs { - if !seen[*b as usize] { + if self.sm_inc(*b)? == 0 { blocks.push(Block::new(*b)); - seen.insert(*b as usize); } } - drop(seen); self.engine.read_many(&mut blocks)?; @@ -249,7 +244,7 @@ impl BTreeWalker { NV: NodeVisitor, V: Unpack, { - if self.is_seen(root.loc) { + if self.sm_inc(root.loc)? > 0 { Ok(()) } else { self.walk_node(visitor, &root, true) @@ -261,7 +256,7 @@ impl BTreeWalker { NV: NodeVisitor, V: Unpack, { - if self.is_seen(root) { + if self.sm_inc(root)? > 0 { Ok(()) } else { let mut root = Block::new(root); @@ -316,4 +311,17 @@ pub fn btree_to_map( Ok(visitor.values) } +pub fn btree_to_map_with_sm( + engine: Arc, + sm: Arc>, + ignore_non_fatal: bool, + root: u64, +) -> Result> { + let mut walker = BTreeWalker::new_with_sm(engine, sm, ignore_non_fatal)?; + let mut visitor = ValueCollector::::new(); + + walker.walk(&mut visitor, root)?; + Ok(visitor.values) +} + //------------------------------------------ diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 91ac606..a111c86 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Result}; +use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; use std::sync::{Arc, Mutex}; @@ -154,10 +155,13 @@ impl Unpack for Bitmap { //------------------------------------------ pub trait SpaceMap { + fn get_nr_blocks(&self) -> Result; fn get(&self, b: u64) -> Result; fn inc(&mut self, begin: u64, len: u64) -> Result<()>; } +//------------------------------------------ + pub struct CoreSpaceMap { counts: Vec, } @@ -177,6 +181,10 @@ impl SpaceMap for CoreSpaceMap where V: Copy + Default + std::ops::AddAssign + From + Into, { + fn get_nr_blocks(&self) -> Result { + Ok(self.counts.len() as u64) + } + fn get(&self, b: u64) -> Result { Ok(self.counts[b as usize].into()) } @@ -189,7 +197,7 @@ where } } -pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc> { +pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc> { if max_count <= u8::MAX as u32 { Arc::new(Mutex::new(CoreSpaceMap::::new(nr_entries))) } else if max_count <= u16::MAX as u32 { @@ -200,3 +208,41 @@ pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc RestrictedSpaceMap { + RestrictedSpaceMap { + counts: FixedBitSet::with_capacity(nr_entries as usize), + } + } +} + +impl SpaceMap for RestrictedSpaceMap { + fn get_nr_blocks(&self) -> Result { + Ok(self.counts.len() as u64) + } + + fn get(&self, b: u64) -> Result { + if self.counts.contains(b as usize) { + Ok(1) + } else { + Ok(0) + } + } + + fn inc(&mut self, begin: u64, len: u64) -> Result<()> { + for b in begin..(begin + len) { + self.counts.insert(b as usize); + } + Ok(()) + } +} + +//------------------------------------------ diff --git a/src/thin/check.rs b/src/thin/check.rs index e7110c2..5d94ed9 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -use fixedbitset::FixedBitSet; use nom::{number::complete::*, IResult}; use std::collections::BTreeMap; use std::path::Path; @@ -8,9 +7,9 @@ use threadpool::ThreadPool; use crate::checksum; use crate::io_engine::{AsyncIoEngine, Block, IoEngine, SyncIoEngine}; -use crate::pdata::unpack::*; -use crate::pdata::btree::{btree_to_map, BTreeWalker, Node, NodeVisitor}; +use crate::pdata::btree::{btree_to_map, btree_to_map_with_sm, BTreeWalker, Node, NodeVisitor}; use crate::pdata::space_map::*; +use crate::pdata::unpack::*; use crate::thin::superblock::*; //------------------------------------------ @@ -200,11 +199,26 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // superblock let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; - // device details + // Device details. We read this once to get the number of thin devices, and hence the + // maximum metadata ref count. Then create metadata space map, and reread to increment + // the ref counts for that metadata. let devs = btree_to_map::(engine.clone(), false, sb.details_root)?; let nr_devs = devs.len(); + let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32); + let _devs = btree_to_map_with_sm::( + engine.clone(), + metadata_sm.clone(), + false, + sb.details_root, + )?; println!("found {} devices", nr_devs); + // increment superblock + { + let mut sm = metadata_sm.lock().unwrap(); + sm.inc(SUPERBLOCK_LOCATION, 1)?; + } + // mapping top level let roots = btree_to_map::(engine.clone(), false, sb.mapping_root)?; @@ -214,15 +228,12 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // FIXME: with a thread pool we need to return errors another way. let nr_workers = nr_threads; let pool = ThreadPool::new(nr_workers); - let seen = Arc::new(Mutex::new(FixedBitSet::with_capacity( - engine.get_nr_blocks() as usize, - ))); let root = unpack::(&sb.data_sm_root[0..])?; data_sm = core_sm(root.nr_blocks, nr_devs as u32); for (thin_id, root) in roots { - let mut w = BTreeWalker::new_with_seen(engine.clone(), seen.clone(), false); + let mut w = BTreeWalker::new_with_sm(engine.clone(), metadata_sm.clone(), false)?; let data_sm = data_sm.clone(); pool.execute(move || { let mut v = BottomLevelVisitor { data_sm }; @@ -325,7 +336,6 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { } // Check the metadata space map. - Ok(()) } From e65d2dec6fc91a38b0da2eccb2f0aba7eb0f2e14 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Tue, 11 Aug 2020 13:22:14 +0100 Subject: [PATCH 49/52] [thin_check (rust)] Add progress bar --- Cargo.lock | 70 ++++++++++++++++++++++++++ Cargo.toml | 1 + src/pdata/space_map.rs | 32 ++++++++++-- src/thin/check.rs | 112 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 201 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdd3687..e6d5c20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,6 +98,23 @@ dependencies = [ "vec_map", ] +[[package]] +name = "console" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "termios", + "unicode-width", + "winapi", + "winapi-util", +] + [[package]] name = "crc32c" version = "0.4.0" @@ -125,6 +142,12 @@ dependencies = [ "shared_child", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "env_logger" version = "0.7.1" @@ -268,6 +291,18 @@ dependencies = [ "libc", ] +[[package]] +name = "indicatif" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + [[package]] name = "io-uring" version = "0.3.5" @@ -387,6 +422,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" + [[package]] name = "once_cell" version = "1.4.0" @@ -630,6 +671,25 @@ dependencies = [ "winapi", ] +[[package]] +name = "terminal_size" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "termios" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" +dependencies = [ + "libc", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -652,6 +712,7 @@ dependencies = [ "fixedbitset", "flate2", "futures", + "indicatif", "io-uring", "json", "libc", @@ -759,6 +820,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 63de8df..86626a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ fixedbitset = "0.3" futures = "0.3" flate2 = "1.0" io-uring = "0.3" +indicatif = "0.15" libc = "0.2.71" nix = "0.17" nom = "5.1" diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index a111c86..1ec9d34 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -156,6 +156,7 @@ impl Unpack for Bitmap { pub trait SpaceMap { fn get_nr_blocks(&self) -> Result; + fn get_nr_allocated(&self) -> Result; fn get(&self, b: u64) -> Result; fn inc(&mut self, begin: u64, len: u64) -> Result<()>; } @@ -163,6 +164,7 @@ pub trait SpaceMap { //------------------------------------------ pub struct CoreSpaceMap { + nr_allocated: u64, counts: Vec, } @@ -172,6 +174,7 @@ where { pub fn new(nr_entries: u64) -> CoreSpaceMap { CoreSpaceMap { + nr_allocated: 0, counts: vec![V::default(); nr_entries as usize], } } @@ -179,19 +182,29 @@ where impl SpaceMap for CoreSpaceMap where - V: Copy + Default + std::ops::AddAssign + From + Into, + V: Copy + Default + Eq + std::ops::AddAssign + From + Into, { fn get_nr_blocks(&self) -> Result { Ok(self.counts.len() as u64) } - + + fn get_nr_allocated(&self) -> Result { + Ok(self.nr_allocated) + } + fn get(&self, b: u64) -> Result { Ok(self.counts[b as usize].into()) } fn inc(&mut self, begin: u64, len: u64) -> Result<()> { for b in begin..(begin + len) { - self.counts[b as usize] += V::from(1u8); + if self.counts[b as usize] == V::from(0u8) { + // FIXME: can we get a ref to save dereferencing counts twice? + self.nr_allocated += 1; + self.counts[b as usize] = V::from(1u8); + } else { + self.counts[b as usize] += V::from(1u8); + } } Ok(()) } @@ -213,12 +226,14 @@ pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc RestrictedSpaceMap { RestrictedSpaceMap { + nr_allocated: 0, counts: FixedBitSet::with_capacity(nr_entries as usize), } } @@ -228,7 +243,11 @@ impl SpaceMap for RestrictedSpaceMap { fn get_nr_blocks(&self) -> Result { Ok(self.counts.len() as u64) } - + + fn get_nr_allocated(&self) -> Result { + Ok(self.nr_allocated) + } + fn get(&self, b: u64) -> Result { if self.counts.contains(b as usize) { Ok(1) @@ -239,7 +258,10 @@ impl SpaceMap for RestrictedSpaceMap { fn inc(&mut self, begin: u64, len: u64) -> Result<()> { for b in begin..(begin + len) { - self.counts.insert(b as usize); + if !self.counts.contains(b as usize) { + self.nr_allocated += 1; + self.counts.insert(b as usize); + } } Ok(()) } diff --git a/src/thin/check.rs b/src/thin/check.rs index 5d94ed9..448dad7 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -1,8 +1,11 @@ use anyhow::{anyhow, Result}; +use indicatif::{ProgressBar, ProgressStyle}; use nom::{number::complete::*, IResult}; use std::collections::BTreeMap; use std::path::Path; +use std::sync::mpsc::{channel, Receiver, Sender, TryRecvError}; use std::sync::{Arc, Mutex}; +use std::{thread, time}; use threadpool::ThreadPool; use crate::checksum; @@ -176,6 +179,86 @@ impl<'a> NodeVisitor for OverflowChecker<'a> { //------------------------------------------ +enum SpinnerCmd { + Complete, + Abort, + Title(String), +} + +struct Spinner { + tx: Sender, + tid: thread::JoinHandle<()>, +} + +impl Spinner { + fn new(sm: Arc>, total_allocated: u64) -> Result { + let (tx, rx) = channel(); + let tid = thread::spawn(move || spinner_thread(sm, total_allocated, rx)); + Ok(Spinner { tx, tid }) + } + + fn complete(self) -> Result<()> { + self.tx.send(SpinnerCmd::Complete)?; + self.tid.join(); + Ok(()) + } + + fn abort(self) -> Result<()> { + self.tx.send(SpinnerCmd::Abort)?; + self.tid.join(); + Ok(()) + } + + fn set_title(&mut self, txt: &str) -> Result<()> { + self.tx.send(SpinnerCmd::Title(txt.to_string()))?; + Ok(()) + } +} + +fn spinner_thread( + sm: Arc>, + total_allocated: u64, + rx: Receiver, +) { + let interval = time::Duration::from_millis(250); + let bar = ProgressBar::new(total_allocated); + loop { + match rx.try_recv() { + Ok(SpinnerCmd::Complete) => { + bar.finish(); + return; + } + Ok(SpinnerCmd::Abort) => { + return; + } + Ok(SpinnerCmd::Title(txt)) => { + let mut fmt = "Checking thin metadata [{bar:40.cyan/blue}] Remaining {eta}, ".to_string(); + fmt.push_str(&txt); + bar.set_style( + ProgressStyle::default_bar() + .template(&fmt) + .progress_chars("=> "), + ); + } + Err(TryRecvError::Disconnected) => { + return; + } + Err(TryRecvError::Empty) => {} + } + + let sm = sm.lock().unwrap(); + let nr_allocated = sm.get_nr_allocated().unwrap(); + drop(sm); + + bar.set_position(nr_allocated); + bar.tick(); + + thread::sleep(interval); + } +} + +//------------------------------------------ + const MAX_CONCURRENT_IO: u32 = 1024; pub struct ThinCheckOptions<'a> { @@ -191,7 +274,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { nr_threads = std::cmp::min(4, num_cpus::get()); engine = Arc::new(AsyncIoEngine::new(opts.dev, MAX_CONCURRENT_IO)?); } else { - eprintln!("falling back to synchronous io"); + eprintln!("Using synchronous io"); nr_threads = num_cpus::get() * 2; engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads)?); } @@ -199,30 +282,39 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // superblock let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?; + let nr_allocated_metadata; + { + let root = unpack::(&sb.metadata_sm_root[0..])?; + nr_allocated_metadata = root.nr_allocated; + } + // Device details. We read this once to get the number of thin devices, and hence the // maximum metadata ref count. Then create metadata space map, and reread to increment // the ref counts for that metadata. let devs = btree_to_map::(engine.clone(), false, sb.details_root)?; let nr_devs = devs.len(); let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32); + let mut spinner = Spinner::new(metadata_sm.clone(), nr_allocated_metadata)?; + + spinner.set_title("device details tree")?; let _devs = btree_to_map_with_sm::( engine.clone(), metadata_sm.clone(), false, sb.details_root, )?; - println!("found {} devices", nr_devs); // increment superblock { let mut sm = metadata_sm.lock().unwrap(); sm.inc(SUPERBLOCK_LOCATION, 1)?; } - + // mapping top level let roots = btree_to_map::(engine.clone(), false, sb.mapping_root)?; // Check the mappings filling in the data_sm as we go. + spinner.set_title("mapping tree")?; let data_sm; { // FIXME: with a thread pool we need to return errors another way. @@ -232,7 +324,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let root = unpack::(&sb.data_sm_root[0..])?; data_sm = core_sm(root.nr_blocks, nr_devs as u32); - for (thin_id, root) in roots { + for (_thin_id, root) in roots { let mut w = BTreeWalker::new_with_sm(engine.clone(), metadata_sm.clone(), false)?; let data_sm = data_sm.clone(); pool.execute(move || { @@ -244,8 +336,8 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { eprintln!("walk failed {:?}", e); std::process::abort(); } - Ok(result) => { - eprintln!("checked thin_dev {} -> {:?}", thin_id, result); + Ok(_result) => { + //eprintln!("checked thin_dev {} -> {:?}", thin_id, result); } } }); @@ -256,10 +348,10 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // Check the data space map. { + spinner.set_title("data space map")?; let data_sm = data_sm.lock().unwrap(); let root = unpack::(&sb.data_sm_root[0..])?; let nr_data_blocks = root.nr_blocks; - eprintln!("data root: {:?}", root); // overflow btree { @@ -270,7 +362,6 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { // Bitmaps let entries = btree_to_map::(engine.clone(), false, root.bitmap_root)?; - eprintln!("{} index entries", entries.len()); let mut blocks = Vec::new(); for (_k, i) in &entries { @@ -302,7 +393,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { BitmapEntry::Small(actual) => { let expected = data_sm.get(blocknr)?; if actual == 1 && expected == 0 { - eprintln!("Data block {} leaked.", blocknr); + // eprintln!("Data block {} leaked.", blocknr); leaks += 1; } else if actual != expected as u8 { eprintln!("Bad reference count for data block {}. Expected {}, but space map contains {}.", @@ -331,12 +422,15 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { } if fail { + spinner.abort()?; return Err(anyhow!("Inconsistent data space map")); } } // Check the metadata space map. + spinner.set_title("metadata space map")?; + spinner.complete()?; Ok(()) } From 3757e1d947541ffa1e64c54af9992d10b518c387 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 12 Aug 2020 08:02:29 +0100 Subject: [PATCH 50/52] [thin_check (rust)] check metadata space map --- src/pdata/space_map.rs | 25 +++++- src/thin/check.rs | 194 ++++++++++++++++++++++++----------------- 2 files changed, 137 insertions(+), 82 deletions(-) diff --git a/src/pdata/space_map.rs b/src/pdata/space_map.rs index 1ec9d34..7f36aba 100644 --- a/src/pdata/space_map.rs +++ b/src/pdata/space_map.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, Result}; use fixedbitset::FixedBitSet; -use nom::{number::complete::*, IResult}; +use nom::{multi::count, number::complete::*, IResult}; use std::sync::{Arc, Mutex}; use crate::io_engine::*; @@ -78,6 +78,29 @@ impl Unpack for IndexEntry { //------------------------------------------ +const MAX_METADATA_BITMAPS: usize = 255; + +pub struct MetadataIndex { + pub indexes: Vec, +} + +impl Unpack for MetadataIndex { + fn disk_size() -> u32 { + BLOCK_SIZE as u32 + } + + fn unpack(data: &[u8]) -> IResult<&[u8], Self> { + let (i, _csum) = le_u32(data)?; + let (i, _padding) = le_u32(i)?; + let (i, _blocknr) = le_u64(i)?; + let (i, indexes) = count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?; + + Ok((i, MetadataIndex {indexes})) + } +} + +//------------------------------------------ + #[derive(Debug)] pub struct BitmapHeader { pub csum: u32, diff --git a/src/thin/check.rs b/src/thin/check.rs index 448dad7..3fc3a89 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -232,7 +232,7 @@ fn spinner_thread( return; } Ok(SpinnerCmd::Title(txt)) => { - let mut fmt = "Checking thin metadata [{bar:40.cyan/blue}] Remaining {eta}, ".to_string(); + let mut fmt = "Checking thin metadata [{bar:40}] Remaining {eta}, ".to_string(); fmt.push_str(&txt); bar.set_style( ProgressStyle::default_bar() @@ -259,6 +259,87 @@ fn spinner_thread( //------------------------------------------ +fn check_space_map( + engine: Arc, + entries: Vec, + sm: Arc>, + root: SMRoot, +) -> Result<()> { + let sm = sm.lock().unwrap(); + + // overflow btree + { + let mut v = OverflowChecker::new(&*sm); + let mut w = BTreeWalker::new(engine.clone(), false); + w.walk(&mut v, root.ref_count_root)?; + } + + let mut blocks = Vec::new(); + for i in &entries { + blocks.push(Block::new(i.blocknr)); + } + + // FIXME: we should do this in batches + engine.read_many(&mut blocks)?; + + let mut leaks = 0; + let mut fail = false; + let mut blocknr = 0; + for n in 0..entries.len() { + let b = &blocks[n]; + if checksum::metadata_block_type(&b.get_data()) != checksum::BT::BITMAP { + return Err(anyhow!( + "Index entry points to block ({}) that isn't a bitmap", + b.loc + )); + } + + let bitmap = unpack::(b.get_data())?; + for e in bitmap.entries { + if blocknr >= root.nr_blocks { + break; + } + + match e { + BitmapEntry::Small(actual) => { + let expected = sm.get(blocknr)?; + if actual == 1 && expected == 0 { + // eprintln!("Data block {} leaked.", blocknr); + leaks += 1; + } else if actual != expected as u8 { + eprintln!("Bad reference count for data block {}. Expected {}, but space map contains {}.", + blocknr, expected, actual); + fail = true; + } + } + BitmapEntry::Overflow => { + let expected = sm.get(blocknr)?; + if expected < 3 { + eprintln!("Bad reference count for data block {}. Expected {}, but space map says it's >= 3.", + blocknr, expected); + fail = true; + } + } + } + blocknr += 1; + } + } + + if leaks > 0 { + eprintln!( + "{} data blocks have leaked. Use --auto-repair to fix.", + leaks + ); + } + + if fail { + return Err(anyhow!("Inconsistent data space map")); + } + Ok(()) +} + +//------------------------------------------ + const MAX_CONCURRENT_IO: u32 = 1024; pub struct ThinCheckOptions<'a> { @@ -346,89 +427,40 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { pool.join(); } - // Check the data space map. + spinner.set_title("data space map")?; + let root = unpack::(&sb.data_sm_root[0..])?; + let entries = btree_to_map::(engine.clone(), false, root.bitmap_root)?; + let entries: Vec = entries.values().cloned().collect(); + check_space_map(engine.clone(), entries, data_sm.clone(), root)?; + + spinner.set_title("metadata space map")?; + let root = unpack::(&sb.metadata_sm_root[0..])?; + let mut b = Block::new(root.bitmap_root); + engine.read(&mut b)?; + let entries = unpack::(b.get_data())?.indexes; + + // Unused entries will point to block 0 + let entries: Vec = entries + .iter() + .take_while(|e| e.blocknr != 0) + .cloned() + .collect(); + + // We need to increment the ref counts for all the bitmaps, then walk the overflow + // tree to inc the ref counts for those. { - spinner.set_title("data space map")?; - let data_sm = data_sm.lock().unwrap(); - let root = unpack::(&sb.data_sm_root[0..])?; - let nr_data_blocks = root.nr_blocks; - - // overflow btree - { - let mut v = OverflowChecker::new(&*data_sm); - let mut w = BTreeWalker::new(engine.clone(), false); - w.walk(&mut v, root.ref_count_root)?; - } - - // Bitmaps - let entries = btree_to_map::(engine.clone(), false, root.bitmap_root)?; - - let mut blocks = Vec::new(); - for (_k, i) in &entries { - blocks.push(Block::new(i.blocknr)); - } - - // FIXME: we should do this in batches - engine.read_many(&mut blocks)?; - - let mut leaks = 0; - let mut fail = false; - let mut blocknr = 0; - for n in 0..entries.len() { - let b = &blocks[n]; - if checksum::metadata_block_type(&b.get_data()) != checksum::BT::BITMAP { - return Err(anyhow!( - "Index entry points to block ({}) that isn't a bitmap", - b.loc - )); - } - - let bitmap = unpack::(b.get_data())?; - for e in bitmap.entries { - if blocknr >= nr_data_blocks { - break; - } - - match e { - BitmapEntry::Small(actual) => { - let expected = data_sm.get(blocknr)?; - if actual == 1 && expected == 0 { - // eprintln!("Data block {} leaked.", blocknr); - leaks += 1; - } else if actual != expected as u8 { - eprintln!("Bad reference count for data block {}. Expected {}, but space map contains {}.", - blocknr, expected, actual); - fail = true; - } - } - BitmapEntry::Overflow => { - let expected = data_sm.get(blocknr)?; - if expected < 3 { - eprintln!("Bad reference count for data block {}. Expected {}, but space map says it's >= 3.", - blocknr, expected); - fail = true; - } - } - } - blocknr += 1; - } - } - - if leaks > 0 { - eprintln!( - "{} data blocks have leaked. Use --auto-repair to fix.", - leaks - ); - } - - if fail { - spinner.abort()?; - return Err(anyhow!("Inconsistent data space map")); + let mut sm = metadata_sm.lock().unwrap(); + for ie in &entries { + sm.inc(ie.blocknr, 1)?; } } - - // Check the metadata space map. - spinner.set_title("metadata space map")?; + let _counts = btree_to_map_with_sm::( + engine.clone(), + metadata_sm.clone(), + false, + root.ref_count_root, + )?; + check_space_map(engine.clone(), entries, metadata_sm.clone(), root)?; spinner.complete()?; Ok(()) From 544335ae4a3e928f47bdd95534f4d52fab0b30e6 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 12 Aug 2020 09:35:21 +0100 Subject: [PATCH 51/52] [thin_check (rust)] Send all reporting through the Spinner. This means the spinner doesn't overwrite messages. --- src/thin/check.rs | 132 +++++++++++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 42 deletions(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index 3fc3a89..23f8307 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -180,6 +180,7 @@ impl<'a> NodeVisitor for OverflowChecker<'a> { //------------------------------------------ enum SpinnerCmd { + Log(String), Complete, Abort, Title(String), @@ -197,6 +198,11 @@ impl Spinner { Ok(Spinner { tx, tid }) } + fn log>(&mut self, txt: I) -> Result<()> { + self.tx.send(SpinnerCmd::Log(txt.into()))?; + Ok(()) + } + fn complete(self) -> Result<()> { self.tx.send(SpinnerCmd::Complete)?; self.tid.join(); @@ -223,27 +229,34 @@ fn spinner_thread( let interval = time::Duration::from_millis(250); let bar = ProgressBar::new(total_allocated); loop { - match rx.try_recv() { - Ok(SpinnerCmd::Complete) => { - bar.finish(); - return; + loop { + match rx.try_recv() { + Ok(SpinnerCmd::Log(txt)) => { + bar.println(txt); + } + Ok(SpinnerCmd::Complete) => { + bar.finish(); + return; + } + Ok(SpinnerCmd::Abort) => { + return; + } + Ok(SpinnerCmd::Title(txt)) => { + let mut fmt = "Checking thin metadata [{bar:40}] Remaining {eta}, ".to_string(); + fmt.push_str(&txt); + bar.set_style( + ProgressStyle::default_bar() + .template(&fmt) + .progress_chars("=> "), + ); + } + Err(TryRecvError::Disconnected) => { + return; + } + Err(TryRecvError::Empty) => { + break; + } } - Ok(SpinnerCmd::Abort) => { - return; - } - Ok(SpinnerCmd::Title(txt)) => { - let mut fmt = "Checking thin metadata [{bar:40}] Remaining {eta}, ".to_string(); - fmt.push_str(&txt); - bar.set_style( - ProgressStyle::default_bar() - .template(&fmt) - .progress_chars("=> "), - ); - } - Err(TryRecvError::Disconnected) => { - return; - } - Err(TryRecvError::Empty) => {} } let sm = sm.lock().unwrap(); @@ -260,8 +273,11 @@ fn spinner_thread( //------------------------------------------ fn check_space_map( + kind: &str, engine: Arc, + bar: &mut Spinner, entries: Vec, + metadata_sm: Option>>, sm: Arc>, root: SMRoot, ) -> Result<()> { @@ -270,7 +286,12 @@ fn check_space_map( // overflow btree { let mut v = OverflowChecker::new(&*sm); - let mut w = BTreeWalker::new(engine.clone(), false); + let mut w; + if metadata_sm.is_none() { + w = BTreeWalker::new(engine.clone(), false); + } else { + w = BTreeWalker::new_with_sm(engine.clone(), metadata_sm.unwrap().clone(), false)?; + } w.walk(&mut v, root.ref_count_root)?; } @@ -304,19 +325,18 @@ fn check_space_map( BitmapEntry::Small(actual) => { let expected = sm.get(blocknr)?; if actual == 1 && expected == 0 { - // eprintln!("Data block {} leaked.", blocknr); leaks += 1; } else if actual != expected as u8 { - eprintln!("Bad reference count for data block {}. Expected {}, but space map contains {}.", - blocknr, expected, actual); + bar.log(format!("Bad reference count for {} block {}. Expected {}, but space map contains {}.", + kind, blocknr, expected, actual))?; fail = true; } } BitmapEntry::Overflow => { let expected = sm.get(blocknr)?; if expected < 3 { - eprintln!("Bad reference count for data block {}. Expected {}, but space map says it's >= 3.", - blocknr, expected); + bar.log(format!("Bad reference count for {} block {}. Expected {}, but space map says it's >= 3.", + kind, blocknr, expected))?; fail = true; } } @@ -326,10 +346,10 @@ fn check_space_map( } if leaks > 0 { - eprintln!( - "{} data blocks have leaked. Use --auto-repair to fix.", - leaks - ); + bar.log(format!( + "{} {} blocks have leaked. Use --auto-repair to fix.", + leaks, kind + ))?; } if fail { @@ -340,6 +360,16 @@ fn check_space_map( //------------------------------------------ +fn inc_entries(sm: &Arc>, entries: &[IndexEntry]) -> Result<()> { + let mut sm = sm.lock().unwrap(); + for ie in entries { + sm.inc(ie.blocknr, 1)?; + } + Ok(()) +} + +//------------------------------------------ + const MAX_CONCURRENT_IO: u32 = 1024; pub struct ThinCheckOptions<'a> { @@ -355,7 +385,6 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { nr_threads = std::cmp::min(4, num_cpus::get()); engine = Arc::new(AsyncIoEngine::new(opts.dev, MAX_CONCURRENT_IO)?); } else { - eprintln!("Using synchronous io"); nr_threads = num_cpus::get() * 2; engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads)?); } @@ -429,9 +458,25 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { spinner.set_title("data space map")?; let root = unpack::(&sb.data_sm_root[0..])?; - let entries = btree_to_map::(engine.clone(), false, root.bitmap_root)?; + + let entries = btree_to_map_with_sm::( + engine.clone(), + metadata_sm.clone(), + false, + root.bitmap_root, + )?; let entries: Vec = entries.values().cloned().collect(); - check_space_map(engine.clone(), entries, data_sm.clone(), root)?; + inc_entries(&metadata_sm, &entries[0..])?; + + check_space_map( + "data", + engine.clone(), + &mut spinner, + entries, + Some(metadata_sm.clone()), + data_sm.clone(), + root, + )?; spinner.set_title("metadata space map")?; let root = unpack::(&sb.metadata_sm_root[0..])?; @@ -445,22 +490,25 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { .take_while(|e| e.blocknr != 0) .cloned() .collect(); + inc_entries(&metadata_sm, &entries[0..])?; - // We need to increment the ref counts for all the bitmaps, then walk the overflow - // tree to inc the ref counts for those. - { - let mut sm = metadata_sm.lock().unwrap(); - for ie in &entries { - sm.inc(ie.blocknr, 1)?; - } - } let _counts = btree_to_map_with_sm::( engine.clone(), metadata_sm.clone(), false, root.ref_count_root, )?; - check_space_map(engine.clone(), entries, metadata_sm.clone(), root)?; + + // Now the counts should be correct and we can check it. + check_space_map( + "metadata", + engine.clone(), + &mut spinner, + entries, + None, + metadata_sm.clone(), + root, + )?; spinner.complete()?; Ok(()) From afa3f2f04d234783165291a7de2bab745acb9c57 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Wed, 12 Aug 2020 10:25:06 +0100 Subject: [PATCH 52/52] [thin_check (rust)] Rename Spinner -> Reporter --- src/thin/check.rs | 117 +++++++++++++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 38 deletions(-) diff --git a/src/thin/check.rs b/src/thin/check.rs index 23f8307..1ffce8e 100644 --- a/src/thin/check.rs +++ b/src/thin/check.rs @@ -179,69 +179,109 @@ impl<'a> NodeVisitor for OverflowChecker<'a> { //------------------------------------------ -enum SpinnerCmd { +struct ReportOptions {} + +#[derive(Clone)] +enum ReportOutcome { + Success, + NonFatal, + Fatal, +} + +use ReportOutcome::*; + +impl ReportOutcome { + fn combine(lhs: &ReportOutcome, rhs: &ReportOutcome) -> ReportOutcome { + match (lhs, rhs) { + (Success, rhs) => rhs.clone(), + (lhs, Success) => lhs.clone(), + (Fatal, _) => Fatal, + (_, Fatal) => Fatal, + (_, _) => NonFatal, + } + } +} + +enum ReportCmd { Log(String), Complete, - Abort, Title(String), } -struct Spinner { - tx: Sender, +struct Report { + opts: ReportOptions, + outcome: ReportOutcome, + tx: Sender, tid: thread::JoinHandle<()>, } -impl Spinner { - fn new(sm: Arc>, total_allocated: u64) -> Result { +impl Report { + fn new( + opts: ReportOptions, + sm: Arc>, + total_allocated: u64, + ) -> Result { let (tx, rx) = channel(); - let tid = thread::spawn(move || spinner_thread(sm, total_allocated, rx)); - Ok(Spinner { tx, tid }) + let tid = thread::spawn(move || report_thread(sm, total_allocated, rx)); + Ok(Report { + opts, + outcome: ReportOutcome::Success, + tx, + tid, + }) } - fn log>(&mut self, txt: I) -> Result<()> { - self.tx.send(SpinnerCmd::Log(txt.into()))?; + fn info>(&mut self, txt: I) -> Result<()> { + self.tx.send(ReportCmd::Log(txt.into()))?; + Ok(()) + } + + fn add_outcome(&mut self, rhs: ReportOutcome) { + self.outcome = ReportOutcome::combine(&self.outcome, &rhs); + } + + fn non_fatal>(&mut self, txt: I) -> Result<()> { + self.add_outcome(NonFatal); + self.tx.send(ReportCmd::Log(txt.into()))?; + Ok(()) + } + + fn fatal>(&mut self, txt: I) -> Result<()> { + self.add_outcome(Fatal); + self.tx.send(ReportCmd::Log(txt.into()))?; Ok(()) } fn complete(self) -> Result<()> { - self.tx.send(SpinnerCmd::Complete)?; - self.tid.join(); - Ok(()) - } - - fn abort(self) -> Result<()> { - self.tx.send(SpinnerCmd::Abort)?; + self.tx.send(ReportCmd::Complete)?; self.tid.join(); Ok(()) } fn set_title(&mut self, txt: &str) -> Result<()> { - self.tx.send(SpinnerCmd::Title(txt.to_string()))?; + self.tx.send(ReportCmd::Title(txt.to_string()))?; Ok(()) } } -fn spinner_thread( +fn report_thread( sm: Arc>, total_allocated: u64, - rx: Receiver, + rx: Receiver, ) { let interval = time::Duration::from_millis(250); let bar = ProgressBar::new(total_allocated); loop { loop { match rx.try_recv() { - Ok(SpinnerCmd::Log(txt)) => { + Ok(ReportCmd::Log(txt)) => { bar.println(txt); } - Ok(SpinnerCmd::Complete) => { + Ok(ReportCmd::Complete) => { bar.finish(); return; } - Ok(SpinnerCmd::Abort) => { - return; - } - Ok(SpinnerCmd::Title(txt)) => { + Ok(ReportCmd::Title(txt)) => { let mut fmt = "Checking thin metadata [{bar:40}] Remaining {eta}, ".to_string(); fmt.push_str(&txt); bar.set_style( @@ -275,7 +315,7 @@ fn spinner_thread( fn check_space_map( kind: &str, engine: Arc, - bar: &mut Spinner, + bar: &mut Report, entries: Vec, metadata_sm: Option>>, sm: Arc>, @@ -327,7 +367,7 @@ fn check_space_map( if actual == 1 && expected == 0 { leaks += 1; } else if actual != expected as u8 { - bar.log(format!("Bad reference count for {} block {}. Expected {}, but space map contains {}.", + bar.fatal(format!("Bad reference count for {} block {}. Expected {}, but space map contains {}.", kind, blocknr, expected, actual))?; fail = true; } @@ -335,7 +375,7 @@ fn check_space_map( BitmapEntry::Overflow => { let expected = sm.get(blocknr)?; if expected < 3 { - bar.log(format!("Bad reference count for {} block {}. Expected {}, but space map says it's >= 3.", + bar.fatal(format!("Bad reference count for {} block {}. Expected {}, but space map says it's >= 3.", kind, blocknr, expected))?; fail = true; } @@ -346,7 +386,7 @@ fn check_space_map( } if leaks > 0 { - bar.log(format!( + bar.non_fatal(format!( "{} {} blocks have leaked. Use --auto-repair to fix.", leaks, kind ))?; @@ -404,9 +444,10 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let devs = btree_to_map::(engine.clone(), false, sb.details_root)?; let nr_devs = devs.len(); let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32); - let mut spinner = Spinner::new(metadata_sm.clone(), nr_allocated_metadata)?; + let opts = ReportOptions {}; + let mut report = Report::new(opts, metadata_sm.clone(), nr_allocated_metadata)?; - spinner.set_title("device details tree")?; + report.set_title("device details tree")?; let _devs = btree_to_map_with_sm::( engine.clone(), metadata_sm.clone(), @@ -424,7 +465,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { let roots = btree_to_map::(engine.clone(), false, sb.mapping_root)?; // Check the mappings filling in the data_sm as we go. - spinner.set_title("mapping tree")?; + report.set_title("mapping tree")?; let data_sm; { // FIXME: with a thread pool we need to return errors another way. @@ -456,7 +497,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { pool.join(); } - spinner.set_title("data space map")?; + report.set_title("data space map")?; let root = unpack::(&sb.data_sm_root[0..])?; let entries = btree_to_map_with_sm::( @@ -471,14 +512,14 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { check_space_map( "data", engine.clone(), - &mut spinner, + &mut report, entries, Some(metadata_sm.clone()), data_sm.clone(), root, )?; - spinner.set_title("metadata space map")?; + report.set_title("metadata space map")?; let root = unpack::(&sb.metadata_sm_root[0..])?; let mut b = Block::new(root.bitmap_root); engine.read(&mut b)?; @@ -503,14 +544,14 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> { check_space_map( "metadata", engine.clone(), - &mut spinner, + &mut report, entries, None, metadata_sm.clone(), root, )?; - spinner.complete()?; + report.complete()?; Ok(()) }