From 3618da3a124747a870acd25f1151055eb086eb77 Mon Sep 17 00:00:00 2001 From: Joe Thornber Date: Mon, 29 Jun 2020 10:49:40 +0100 Subject: [PATCH] [thin_shrink] Get test framework built This verifies data has been moved to the correct place. --- src/bin/thin_metadata_pack.rs | 11 ++-- src/bin/thin_metadata_unpack.rs | 39 +++++++----- src/bin/thin_shrink.rs | 13 ++-- src/file_utils.rs | 34 +++++++--- src/pack/toplevel.rs | 7 ++- src/shrink/copier.rs | 3 +- src/shrink/toplevel.rs | 11 +++- tests/thin_shrink.rs | 106 +++++++++++++++++++++++++++++--- 8 files changed, 172 insertions(+), 52 deletions(-) diff --git a/src/bin/thin_metadata_pack.rs b/src/bin/thin_metadata_pack.rs index ff3afa7..d4b731f 100644 --- a/src/bin/thin_metadata_pack.rs +++ b/src/bin/thin_metadata_pack.rs @@ -2,6 +2,7 @@ extern crate clap; extern crate thinp; use clap::{App, Arg}; +use std::path::Path; use std::process::exit; use thinp::file_utils; @@ -23,14 +24,14 @@ fn main() { .takes_value(true)); let matches = parser.get_matches(); - let input_file = matches.value_of("INPUT").unwrap(); - let output_file = matches.value_of("OUTPUT").unwrap(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = Path::new(matches.value_of("OUTPUT").unwrap()); - if !file_utils::file_exists(input_file) { - eprintln!("Couldn't find input file '{}'.", &input_file); + if !file_utils::file_exists(&input_file) { + eprintln!("Couldn't find input file '{}'.", &input_file.display()); exit(1); } - + if let Err(reason) = thinp::pack::toplevel::pack(&input_file, &output_file) { println!("Application error: {}\n", reason); exit(1); diff --git a/src/bin/thin_metadata_unpack.rs b/src/bin/thin_metadata_unpack.rs index a87dc8a..ca6403f 100644 --- a/src/bin/thin_metadata_unpack.rs +++ b/src/bin/thin_metadata_unpack.rs @@ -2,6 +2,7 @@ extern crate clap; extern crate thinp; use clap::{App, Arg}; +use std::path::Path; use std::process; use thinp::file_utils; @@ -9,30 +10,34 @@ use std::process::exit; fn main() { let parser = App::new("thin_metadata_unpack") - .version(thinp::version::TOOLS_VERSION) + .version(thinp::version::TOOLS_VERSION) .about("Unpack a compressed file of thin metadata.") - .arg(Arg::with_name("INPUT") - .help("Specify thinp metadata binary device/file") - .required(true) - .short("i") - .value_name("DEV") - .takes_value(true)) - .arg(Arg::with_name("OUTPUT") - .help("Specify packed output file") - .required(true) - .short("o") - .value_name("FILE") - .takes_value(true)); + .arg( + Arg::with_name("INPUT") + .help("Specify thinp metadata binary device/file") + .required(true) + .short("i") + .value_name("DEV") + .takes_value(true), + ) + .arg( + Arg::with_name("OUTPUT") + .help("Specify packed output file") + .required(true) + .short("o") + .value_name("FILE") + .takes_value(true), + ); let matches = parser.get_matches(); - let input_file = matches.value_of("INPUT").unwrap(); - let output_file = matches.value_of("OUTPUT").unwrap(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = Path::new(matches.value_of("OUTPUT").unwrap()); if !file_utils::file_exists(input_file) { - eprintln!("Couldn't find input file '{}'.", &input_file); + eprintln!("Couldn't find input file '{}'.", &input_file.display()); exit(1); } - + if let Err(reason) = thinp::pack::toplevel::unpack(&input_file, &output_file) { println!("Application error: {}", reason); process::exit(1); diff --git a/src/bin/thin_shrink.rs b/src/bin/thin_shrink.rs index 54d053f..2e4e270 100644 --- a/src/bin/thin_shrink.rs +++ b/src/bin/thin_shrink.rs @@ -2,6 +2,7 @@ extern crate clap; extern crate thinp; use clap::{App, Arg}; +use std::path::Path; use std::process::exit; use thinp::file_utils; @@ -53,18 +54,20 @@ fn main() { let matches = parser.get_matches(); // FIXME: check these look like xml - let input_file = matches.value_of("INPUT").unwrap(); - let output_file = matches.value_of("OUTPUT").unwrap(); + let input_file = Path::new(matches.value_of("INPUT").unwrap()); + let output_file = Path::new(matches.value_of("OUTPUT").unwrap()); let size = matches.value_of("SIZE").unwrap().parse::().unwrap(); - let data_file = matches.value_of("DATA").unwrap(); + let data_file = Path::new(matches.value_of("DATA").unwrap()); let do_copy = !matches.is_present("NOCOPY"); if !file_utils::file_exists(input_file) { - eprintln!("Couldn't find input file '{}'.", &input_file); + eprintln!("Couldn't find input file '{}'.", input_file.display()); exit(1); } - if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy) { + if let Err(reason) = + thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy) + { println!("Application error: {}\n", reason); exit(1); } diff --git a/src/file_utils.rs b/src/file_utils.rs index 064e59a..85f0714 100644 --- a/src/file_utils.rs +++ b/src/file_utils.rs @@ -1,9 +1,10 @@ use nix::sys::stat; use nix::sys::stat::{FileStat, SFlag}; -use std::fs::File; +use std::fs::{File, OpenOptions}; use std::io; use std::io::{Seek, Write}; use std::os::unix::io::AsRawFd; +use std::path::Path; use tempfile::tempfile; //--------------------------------------- @@ -17,7 +18,7 @@ pub fn is_file_or_blk(info: FileStat) -> bool { || check_bits(info.st_mode, &stat::SFlag::S_IFREG) } -pub fn file_exists(path: &str) -> bool { +pub fn file_exists(path: &Path) -> bool { match stat::stat(path) { Ok(info) => is_file_or_blk(info), _ => { @@ -39,7 +40,7 @@ pub fn fail(msg: &str) -> io::Result { Err(e) } -fn get_device_size(path: &str) -> io::Result { +fn get_device_size(path: &Path) -> io::Result { let file = File::open(path)?; let fd = file.as_raw_fd(); let mut cap = 0u64; @@ -51,7 +52,7 @@ fn get_device_size(path: &str) -> io::Result { } } -pub fn file_size(path: &str) -> io::Result { +pub fn file_size(path: &Path) -> io::Result { match stat::stat(path) { Ok(info) => { if check_bits(info.st_mode, &SFlag::S_IFREG) { @@ -68,16 +69,31 @@ pub fn file_size(path: &str) -> io::Result { //--------------------------------------- -pub fn temp_file_sized(nr_bytes: u64) -> io::Result { - let mut file = tempfile()?; - +fn set_size(w: &mut W, nr_bytes: u64) -> io::Result<()> { let zeroes: Vec = vec![0; 1]; if nr_bytes > 0 { - file.seek(io::SeekFrom::Start(nr_bytes - 1))?; - file.write_all(&zeroes)?; + w.seek(io::SeekFrom::Start(nr_bytes - 1))?; + w.write_all(&zeroes)?; } + Ok(()) +} + +pub fn temp_file_sized(nr_bytes: u64) -> io::Result { + let mut file = tempfile()?; + set_size(&mut file, nr_bytes)?; + Ok(file) +} + +pub fn create_sized_file(path: &Path, nr_bytes: u64) -> io::Result { + let mut file = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(path)?; + set_size(&mut file, nr_bytes)?; Ok(file) } diff --git a/src/pack/toplevel.rs b/src/pack/toplevel.rs index bd1757d..6179899 100644 --- a/src/pack/toplevel.rs +++ b/src/pack/toplevel.rs @@ -6,6 +6,7 @@ use std::os::unix::fs::OpenOptionsExt; use std::{ error::Error, fs::OpenOptions, + path::Path, io, io::prelude::*, io::Cursor, @@ -67,7 +68,7 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec> { vs } -pub fn pack(input_file: &str, output_file: &str) -> Result<(), Box> { +pub fn pack(input_file: &Path, output_file: &Path) -> Result<(), Box> { let nr_blocks = get_nr_blocks(&input_file)?; let nr_jobs = std::cmp::max(1, std::cmp::min(num_cpus::get() as u64, nr_blocks / 128)); let chunk_vecs = mk_chunk_vecs(nr_blocks, nr_jobs); @@ -192,7 +193,7 @@ where r.read_u64::() } -fn get_nr_blocks(path: &str) -> io::Result { +fn get_nr_blocks(path: &Path) -> io::Result { let len = file_utils::file_size(path)?; Ok(len / (BLOCK_SIZE as u64)) } @@ -306,7 +307,7 @@ where Ok(()) } -pub fn unpack(input_file: &str, output_file: &str) -> Result<(), Box> { +pub fn unpack(input_file: &Path, output_file: &Path) -> Result<(), Box> { let mut input = OpenOptions::new() .read(true) .write(false) diff --git a/src/shrink/copier.rs b/src/shrink/copier.rs index ef04a02..f112e28 100644 --- a/src/shrink/copier.rs +++ b/src/shrink/copier.rs @@ -1,5 +1,6 @@ use anyhow::Result; use std::fs::OpenOptions; +use std::path::Path; use std::io::{Seek, SeekFrom, Write, Read}; //use std::os::unix::fs::OpenOptionsExt; @@ -43,7 +44,7 @@ where Ok(()) } -pub fn copy(path: &str, regions: &[Region]) -> Result<()> { +pub fn copy(path: &Path, regions: &[Region]) -> Result<()> { let mut input = OpenOptions::new() .read(true) .write(true) diff --git a/src/shrink/toplevel.rs b/src/shrink/toplevel.rs index 500a5ef..db08ee2 100644 --- a/src/shrink/toplevel.rs +++ b/src/shrink/toplevel.rs @@ -3,6 +3,7 @@ use fixedbitset::FixedBitSet; use std::fs::OpenOptions; use std::io::Write; use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; use crate::shrink::copier::{self, Region}; use crate::thin::xml::{self, Visit}; @@ -426,7 +427,7 @@ fn build_copy_regions(remaps: &[(BlockRange, BlockRange)], block_size: u64) -> V rs } -fn process_xml(input_path: &str, pass: &mut MV) -> Result<()> { +fn process_xml(input_path: &Path, pass: &mut MV) -> Result<()> { let input = OpenOptions::new() .read(true) .write(false) @@ -437,7 +438,13 @@ fn process_xml(input_path: &str, pass: &mut MV) -> Res Ok(()) } -pub fn shrink(input_path: &str, output_path: &str, data_path: &str, nr_blocks: u64, do_copy: bool) -> Result<()> { +pub fn shrink( + input_path: &Path, + output_path: &Path, + data_path: &Path, + nr_blocks: u64, + do_copy: bool, +) -> Result<()> { let mut pass1 = Pass1::new(nr_blocks); eprint!("Reading xml..."); process_xml(input_path, &mut pass1)?; diff --git a/tests/thin_shrink.rs b/tests/thin_shrink.rs index 82d6c78..96c6296 100644 --- a/tests/thin_shrink.rs +++ b/tests/thin_shrink.rs @@ -3,6 +3,8 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use rand::Rng; use std::fs::OpenOptions; use std::io::{Cursor, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use tempfile::tempdir; use thinp::file_utils; use thinp::thin::xml::{self, Visit}; @@ -22,6 +24,7 @@ struct ThinReadRef { struct ThinWriteRef<'a, W: Write + Seek> { file: &'a mut W, + block_byte: u64, pub data: Vec, } @@ -38,6 +41,7 @@ impl ThinBlock { fn zero_ref<'a, W: Write + Seek>(&self, w: &'a mut W) -> ThinWriteRef<'a, W> { ThinWriteRef { file: w, + block_byte: self.data_block * (self.block_size as u64), data: vec![0; self.block_size], } } @@ -52,6 +56,7 @@ impl ThinBlock { let wr = ThinWriteRef { file: w, + block_byte: self.data_block * (self.block_size as u64), data: vec![0; self.block_size], }; @@ -61,6 +66,10 @@ impl ThinBlock { impl<'a, W: Write + Seek> Drop for ThinWriteRef<'a, W> { fn drop(&mut self) { + // FIXME: We shouldn't panic in a drop function, so any IO + // errors will have to make their way back to the user + // another way (eg, via a flush() method). + self.file.seek(SeekFrom::Start(self.block_byte)).unwrap(); self.file.write_all(&self.data[0..]).unwrap(); } } @@ -232,28 +241,105 @@ impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> { //------------------------------------ -fn create_data_file(xml_path: &str) -> Result { +fn mk_path(dir: &Path, file: &str) -> PathBuf { + let mut p = PathBuf::new(); + p.push(dir); + p.push(PathBuf::from(file)); + p +} + +fn generate_xml(path: &Path, g: &mut G) -> Result<()> +where + G: XmlGenerator, +{ + let xml_out = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(path)?; + let mut w = xml::XmlWriter::new(xml_out); + + g.generate(&mut w) +} + +fn create_data_file(data_path: &Path, xml_path: &Path) -> Result<()> { let input = OpenOptions::new().read(true).write(false).open(xml_path)?; let sb = xml::read_superblock(input)?; let nr_blocks = sb.nr_data_blocks as u64; let block_size = sb.data_block_size as u64 * 512; - let file = file_utils::temp_file_sized(nr_blocks * block_size)?; - Ok(file) + let _file = file_utils::create_sized_file(data_path, nr_blocks * block_size)?; + Ok(()) } -fn main(xml_path: &str) -> Result<()> { - let mut data_file = create_data_file(xml_path)?; - let mut xml_in = OpenOptions::new().read(true).write(false).open(xml_path)?; +fn stamp(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> { + let mut data = OpenOptions::new() + .read(false) + .write(true) + .open(&data_path)?; + let xml = OpenOptions::new().read(true).write(false).open(&xml_path)?; + + let mut stamper = Stamper::new(&mut data, seed); + thin_visit(xml, &mut stamper) +} + +fn verify(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> { + let mut data = OpenOptions::new() + .read(true) + .write(false) + .open(&data_path)?; + let xml = OpenOptions::new().read(true).write(false).open(&xml_path)?; + + let mut verifier = Verifier::new(&mut data, seed); + thin_visit(xml, &mut verifier) +} + +//------------------------------------ + +trait XmlGenerator { + fn generate(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>; +} + +struct EmptyPoolG {} + +impl XmlGenerator for EmptyPoolG { + fn generate(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> { + v.superblock_b(&xml::Superblock { + uuid: "".to_string(), + time: 0, + transaction: 0, + flags: None, + version: None, + data_block_size: 64, + nr_data_blocks: 1024, + metadata_snap: None, + })?; + v.superblock_e()?; + Ok(()) + } +} + +#[test] +fn shrink_empty_pool() -> Result<()> { + let dir = tempdir()?; + let xml_before = mk_path(dir.path(), "before.xml"); + let xml_after = mk_path(dir.path(), "after.xml"); + let data_path = mk_path(dir.path(), "bin"); + + let mut gen = EmptyPoolG {}; + generate_xml(&xml_before, &mut gen)?; + create_data_file(&data_path, &xml_before)?; let mut rng = rand::thread_rng(); let seed = rng.gen::(); - let mut stamper = Stamper::new(&mut data_file, seed); - thin_visit(&mut xml_in, &mut stamper)?; + stamp(&xml_before, &data_path, seed)?; - let mut verifier = Verifier::new(&mut data_file, seed); - thin_visit(&mut xml_in, &mut verifier)?; + let new_nr_blocks = 10; + thinp::shrink::toplevel::shrink(&xml_before, &xml_after, &data_path, new_nr_blocks, true)?; + + verify(&xml_after, &data_path, seed)?; Ok(()) }