[thin_shrink] Get test framework built

This verifies data has been moved to the correct place.
This commit is contained in:
Joe Thornber 2020-06-29 10:49:40 +01:00
parent d03dac8f75
commit 3618da3a12
8 changed files with 172 additions and 52 deletions

View File

@ -2,6 +2,7 @@ extern crate clap;
extern crate thinp; extern crate thinp;
use clap::{App, Arg}; use clap::{App, Arg};
use std::path::Path;
use std::process::exit; use std::process::exit;
use thinp::file_utils; use thinp::file_utils;
@ -23,11 +24,11 @@ fn main() {
.takes_value(true)); .takes_value(true));
let matches = parser.get_matches(); let matches = parser.get_matches();
let input_file = matches.value_of("INPUT").unwrap(); let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = matches.value_of("OUTPUT").unwrap(); let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
if !file_utils::file_exists(input_file) { if !file_utils::file_exists(&input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file); eprintln!("Couldn't find input file '{}'.", &input_file.display());
exit(1); exit(1);
} }

View File

@ -2,6 +2,7 @@ extern crate clap;
extern crate thinp; extern crate thinp;
use clap::{App, Arg}; use clap::{App, Arg};
use std::path::Path;
use std::process; use std::process;
use thinp::file_utils; use thinp::file_utils;
@ -11,25 +12,29 @@ fn main() {
let parser = App::new("thin_metadata_unpack") let parser = App::new("thin_metadata_unpack")
.version(thinp::version::TOOLS_VERSION) .version(thinp::version::TOOLS_VERSION)
.about("Unpack a compressed file of thin metadata.") .about("Unpack a compressed file of thin metadata.")
.arg(Arg::with_name("INPUT") .arg(
Arg::with_name("INPUT")
.help("Specify thinp metadata binary device/file") .help("Specify thinp metadata binary device/file")
.required(true) .required(true)
.short("i") .short("i")
.value_name("DEV") .value_name("DEV")
.takes_value(true)) .takes_value(true),
.arg(Arg::with_name("OUTPUT") )
.arg(
Arg::with_name("OUTPUT")
.help("Specify packed output file") .help("Specify packed output file")
.required(true) .required(true)
.short("o") .short("o")
.value_name("FILE") .value_name("FILE")
.takes_value(true)); .takes_value(true),
);
let matches = parser.get_matches(); let matches = parser.get_matches();
let input_file = matches.value_of("INPUT").unwrap(); let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = matches.value_of("OUTPUT").unwrap(); let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
if !file_utils::file_exists(input_file) { if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file); eprintln!("Couldn't find input file '{}'.", &input_file.display());
exit(1); exit(1);
} }

View File

@ -2,6 +2,7 @@ extern crate clap;
extern crate thinp; extern crate thinp;
use clap::{App, Arg}; use clap::{App, Arg};
use std::path::Path;
use std::process::exit; use std::process::exit;
use thinp::file_utils; use thinp::file_utils;
@ -53,18 +54,20 @@ fn main() {
let matches = parser.get_matches(); let matches = parser.get_matches();
// FIXME: check these look like xml // FIXME: check these look like xml
let input_file = matches.value_of("INPUT").unwrap(); let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = matches.value_of("OUTPUT").unwrap(); let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
let size = matches.value_of("SIZE").unwrap().parse::<u64>().unwrap(); let size = matches.value_of("SIZE").unwrap().parse::<u64>().unwrap();
let data_file = matches.value_of("DATA").unwrap(); let data_file = Path::new(matches.value_of("DATA").unwrap());
let do_copy = !matches.is_present("NOCOPY"); let do_copy = !matches.is_present("NOCOPY");
if !file_utils::file_exists(input_file) { if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file); eprintln!("Couldn't find input file '{}'.", input_file.display());
exit(1); exit(1);
} }
if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy) { if let Err(reason) =
thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy)
{
println!("Application error: {}\n", reason); println!("Application error: {}\n", reason);
exit(1); exit(1);
} }

View File

@ -1,9 +1,10 @@
use nix::sys::stat; use nix::sys::stat;
use nix::sys::stat::{FileStat, SFlag}; use nix::sys::stat::{FileStat, SFlag};
use std::fs::File; use std::fs::{File, OpenOptions};
use std::io; use std::io;
use std::io::{Seek, Write}; use std::io::{Seek, Write};
use std::os::unix::io::AsRawFd; use std::os::unix::io::AsRawFd;
use std::path::Path;
use tempfile::tempfile; use tempfile::tempfile;
//--------------------------------------- //---------------------------------------
@ -17,7 +18,7 @@ pub fn is_file_or_blk(info: FileStat) -> bool {
|| check_bits(info.st_mode, &stat::SFlag::S_IFREG) || check_bits(info.st_mode, &stat::SFlag::S_IFREG)
} }
pub fn file_exists(path: &str) -> bool { pub fn file_exists(path: &Path) -> bool {
match stat::stat(path) { match stat::stat(path) {
Ok(info) => is_file_or_blk(info), Ok(info) => is_file_or_blk(info),
_ => { _ => {
@ -39,7 +40,7 @@ pub fn fail<T>(msg: &str) -> io::Result<T> {
Err(e) Err(e)
} }
fn get_device_size(path: &str) -> io::Result<u64> { fn get_device_size(path: &Path) -> io::Result<u64> {
let file = File::open(path)?; let file = File::open(path)?;
let fd = file.as_raw_fd(); let fd = file.as_raw_fd();
let mut cap = 0u64; let mut cap = 0u64;
@ -51,7 +52,7 @@ fn get_device_size(path: &str) -> io::Result<u64> {
} }
} }
pub fn file_size(path: &str) -> io::Result<u64> { pub fn file_size(path: &Path) -> io::Result<u64> {
match stat::stat(path) { match stat::stat(path) {
Ok(info) => { Ok(info) => {
if check_bits(info.st_mode, &SFlag::S_IFREG) { if check_bits(info.st_mode, &SFlag::S_IFREG) {
@ -68,16 +69,31 @@ pub fn file_size(path: &str) -> io::Result<u64> {
//--------------------------------------- //---------------------------------------
pub fn temp_file_sized(nr_bytes: u64) -> io::Result<std::fs::File> { fn set_size<W: Write + Seek>(w: &mut W, nr_bytes: u64) -> io::Result<()> {
let mut file = tempfile()?;
let zeroes: Vec<u8> = vec![0; 1]; let zeroes: Vec<u8> = vec![0; 1];
if nr_bytes > 0 { if nr_bytes > 0 {
file.seek(io::SeekFrom::Start(nr_bytes - 1))?; w.seek(io::SeekFrom::Start(nr_bytes - 1))?;
file.write_all(&zeroes)?; w.write_all(&zeroes)?;
} }
Ok(())
}
pub fn temp_file_sized(nr_bytes: u64) -> io::Result<std::fs::File> {
let mut file = tempfile()?;
set_size(&mut file, nr_bytes)?;
Ok(file)
}
pub fn create_sized_file(path: &Path, nr_bytes: u64) -> io::Result<std::fs::File> {
let mut file = OpenOptions::new()
.read(false)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
set_size(&mut file, nr_bytes)?;
Ok(file) Ok(file)
} }

View File

@ -6,6 +6,7 @@ use std::os::unix::fs::OpenOptionsExt;
use std::{ use std::{
error::Error, error::Error,
fs::OpenOptions, fs::OpenOptions,
path::Path,
io, io,
io::prelude::*, io::prelude::*,
io::Cursor, io::Cursor,
@ -67,7 +68,7 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec<Vec<(u64, u64)>> {
vs vs
} }
pub fn pack(input_file: &str, output_file: &str) -> Result<(), Box<dyn Error>> { pub fn pack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>> {
let nr_blocks = get_nr_blocks(&input_file)?; let nr_blocks = get_nr_blocks(&input_file)?;
let nr_jobs = std::cmp::max(1, std::cmp::min(num_cpus::get() as u64, nr_blocks / 128)); let nr_jobs = std::cmp::max(1, std::cmp::min(num_cpus::get() as u64, nr_blocks / 128));
let chunk_vecs = mk_chunk_vecs(nr_blocks, nr_jobs); let chunk_vecs = mk_chunk_vecs(nr_blocks, nr_jobs);
@ -192,7 +193,7 @@ where
r.read_u64::<LittleEndian>() r.read_u64::<LittleEndian>()
} }
fn get_nr_blocks(path: &str) -> io::Result<u64> { fn get_nr_blocks(path: &Path) -> io::Result<u64> {
let len = file_utils::file_size(path)?; let len = file_utils::file_size(path)?;
Ok(len / (BLOCK_SIZE as u64)) Ok(len / (BLOCK_SIZE as u64))
} }
@ -306,7 +307,7 @@ where
Ok(()) Ok(())
} }
pub fn unpack(input_file: &str, output_file: &str) -> Result<(), Box<dyn Error>> { pub fn unpack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>> {
let mut input = OpenOptions::new() let mut input = OpenOptions::new()
.read(true) .read(true)
.write(false) .write(false)

View File

@ -1,5 +1,6 @@
use anyhow::Result; use anyhow::Result;
use std::fs::OpenOptions; use std::fs::OpenOptions;
use std::path::Path;
use std::io::{Seek, SeekFrom, Write, Read}; use std::io::{Seek, SeekFrom, Write, Read};
//use std::os::unix::fs::OpenOptionsExt; //use std::os::unix::fs::OpenOptionsExt;
@ -43,7 +44,7 @@ where
Ok(()) Ok(())
} }
pub fn copy(path: &str, regions: &[Region]) -> Result<()> { pub fn copy(path: &Path, regions: &[Region]) -> Result<()> {
let mut input = OpenOptions::new() let mut input = OpenOptions::new()
.read(true) .read(true)
.write(true) .write(true)

View File

@ -3,6 +3,7 @@ use fixedbitset::FixedBitSet;
use std::fs::OpenOptions; use std::fs::OpenOptions;
use std::io::Write; use std::io::Write;
use std::os::unix::fs::OpenOptionsExt; use std::os::unix::fs::OpenOptionsExt;
use std::path::Path;
use crate::shrink::copier::{self, Region}; use crate::shrink::copier::{self, Region};
use crate::thin::xml::{self, Visit}; use crate::thin::xml::{self, Visit};
@ -426,7 +427,7 @@ fn build_copy_regions(remaps: &[(BlockRange, BlockRange)], block_size: u64) -> V
rs rs
} }
fn process_xml<MV: xml::MetadataVisitor>(input_path: &str, pass: &mut MV) -> Result<()> { fn process_xml<MV: xml::MetadataVisitor>(input_path: &Path, pass: &mut MV) -> Result<()> {
let input = OpenOptions::new() let input = OpenOptions::new()
.read(true) .read(true)
.write(false) .write(false)
@ -437,7 +438,13 @@ fn process_xml<MV: xml::MetadataVisitor>(input_path: &str, pass: &mut MV) -> Res
Ok(()) Ok(())
} }
pub fn shrink(input_path: &str, output_path: &str, data_path: &str, nr_blocks: u64, do_copy: bool) -> Result<()> { pub fn shrink(
input_path: &Path,
output_path: &Path,
data_path: &Path,
nr_blocks: u64,
do_copy: bool,
) -> Result<()> {
let mut pass1 = Pass1::new(nr_blocks); let mut pass1 = Pass1::new(nr_blocks);
eprint!("Reading xml..."); eprint!("Reading xml...");
process_xml(input_path, &mut pass1)?; process_xml(input_path, &mut pass1)?;

View File

@ -3,6 +3,8 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::Rng; use rand::Rng;
use std::fs::OpenOptions; use std::fs::OpenOptions;
use std::io::{Cursor, Read, Seek, SeekFrom, Write}; use std::io::{Cursor, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use tempfile::tempdir;
use thinp::file_utils; use thinp::file_utils;
use thinp::thin::xml::{self, Visit}; use thinp::thin::xml::{self, Visit};
@ -22,6 +24,7 @@ struct ThinReadRef {
struct ThinWriteRef<'a, W: Write + Seek> { struct ThinWriteRef<'a, W: Write + Seek> {
file: &'a mut W, file: &'a mut W,
block_byte: u64,
pub data: Vec<u8>, pub data: Vec<u8>,
} }
@ -38,6 +41,7 @@ impl ThinBlock {
fn zero_ref<'a, W: Write + Seek>(&self, w: &'a mut W) -> ThinWriteRef<'a, W> { fn zero_ref<'a, W: Write + Seek>(&self, w: &'a mut W) -> ThinWriteRef<'a, W> {
ThinWriteRef { ThinWriteRef {
file: w, file: w,
block_byte: self.data_block * (self.block_size as u64),
data: vec![0; self.block_size], data: vec![0; self.block_size],
} }
} }
@ -52,6 +56,7 @@ impl ThinBlock {
let wr = ThinWriteRef { let wr = ThinWriteRef {
file: w, file: w,
block_byte: self.data_block * (self.block_size as u64),
data: vec![0; self.block_size], data: vec![0; self.block_size],
}; };
@ -61,6 +66,10 @@ impl ThinBlock {
impl<'a, W: Write + Seek> Drop for ThinWriteRef<'a, W> { impl<'a, W: Write + Seek> Drop for ThinWriteRef<'a, W> {
fn drop(&mut self) { fn drop(&mut self) {
// FIXME: We shouldn't panic in a drop function, so any IO
// errors will have to make their way back to the user
// another way (eg, via a flush() method).
self.file.seek(SeekFrom::Start(self.block_byte)).unwrap();
self.file.write_all(&self.data[0..]).unwrap(); self.file.write_all(&self.data[0..]).unwrap();
} }
} }
@ -232,28 +241,105 @@ impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> {
//------------------------------------ //------------------------------------
fn create_data_file(xml_path: &str) -> Result<std::fs::File> { fn mk_path(dir: &Path, file: &str) -> PathBuf {
let mut p = PathBuf::new();
p.push(dir);
p.push(PathBuf::from(file));
p
}
fn generate_xml<G>(path: &Path, g: &mut G) -> Result<()>
where
G: XmlGenerator,
{
let xml_out = OpenOptions::new()
.read(false)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
let mut w = xml::XmlWriter::new(xml_out);
g.generate(&mut w)
}
fn create_data_file(data_path: &Path, xml_path: &Path) -> Result<()> {
let input = OpenOptions::new().read(true).write(false).open(xml_path)?; let input = OpenOptions::new().read(true).write(false).open(xml_path)?;
let sb = xml::read_superblock(input)?; let sb = xml::read_superblock(input)?;
let nr_blocks = sb.nr_data_blocks as u64; let nr_blocks = sb.nr_data_blocks as u64;
let block_size = sb.data_block_size as u64 * 512; let block_size = sb.data_block_size as u64 * 512;
let file = file_utils::temp_file_sized(nr_blocks * block_size)?; let _file = file_utils::create_sized_file(data_path, nr_blocks * block_size)?;
Ok(file) Ok(())
} }
fn main(xml_path: &str) -> Result<()> { fn stamp(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> {
let mut data_file = create_data_file(xml_path)?; let mut data = OpenOptions::new()
let mut xml_in = OpenOptions::new().read(true).write(false).open(xml_path)?; .read(false)
.write(true)
.open(&data_path)?;
let xml = OpenOptions::new().read(true).write(false).open(&xml_path)?;
let mut stamper = Stamper::new(&mut data, seed);
thin_visit(xml, &mut stamper)
}
fn verify(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> {
let mut data = OpenOptions::new()
.read(true)
.write(false)
.open(&data_path)?;
let xml = OpenOptions::new().read(true).write(false).open(&xml_path)?;
let mut verifier = Verifier::new(&mut data, seed);
thin_visit(xml, &mut verifier)
}
//------------------------------------
trait XmlGenerator {
fn generate(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>;
}
struct EmptyPoolG {}
impl XmlGenerator for EmptyPoolG {
fn generate(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
v.superblock_b(&xml::Superblock {
uuid: "".to_string(),
time: 0,
transaction: 0,
flags: None,
version: None,
data_block_size: 64,
nr_data_blocks: 1024,
metadata_snap: None,
})?;
v.superblock_e()?;
Ok(())
}
}
#[test]
fn shrink_empty_pool() -> Result<()> {
let dir = tempdir()?;
let xml_before = mk_path(dir.path(), "before.xml");
let xml_after = mk_path(dir.path(), "after.xml");
let data_path = mk_path(dir.path(), "bin");
let mut gen = EmptyPoolG {};
generate_xml(&xml_before, &mut gen)?;
create_data_file(&data_path, &xml_before)?;
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let seed = rng.gen::<u64>(); let seed = rng.gen::<u64>();
let mut stamper = Stamper::new(&mut data_file, seed); stamp(&xml_before, &data_path, seed)?;
thin_visit(&mut xml_in, &mut stamper)?;
let mut verifier = Verifier::new(&mut data_file, seed); let new_nr_blocks = 10;
thin_visit(&mut xml_in, &mut verifier)?; thinp::shrink::toplevel::shrink(&xml_before, &xml_after, &data_path, new_nr_blocks, true)?;
verify(&xml_after, &data_path, seed)?;
Ok(()) Ok(())
} }