[thin_shrink] Get test framework built
This verifies data has been moved to the correct place.
This commit is contained in:
parent
d03dac8f75
commit
3618da3a12
@ -2,6 +2,7 @@ extern crate clap;
|
||||
extern crate thinp;
|
||||
|
||||
use clap::{App, Arg};
|
||||
use std::path::Path;
|
||||
use std::process::exit;
|
||||
use thinp::file_utils;
|
||||
|
||||
@ -23,11 +24,11 @@ fn main() {
|
||||
.takes_value(true));
|
||||
|
||||
let matches = parser.get_matches();
|
||||
let input_file = matches.value_of("INPUT").unwrap();
|
||||
let output_file = matches.value_of("OUTPUT").unwrap();
|
||||
let input_file = Path::new(matches.value_of("INPUT").unwrap());
|
||||
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
|
||||
|
||||
if !file_utils::file_exists(input_file) {
|
||||
eprintln!("Couldn't find input file '{}'.", &input_file);
|
||||
if !file_utils::file_exists(&input_file) {
|
||||
eprintln!("Couldn't find input file '{}'.", &input_file.display());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
@ -2,6 +2,7 @@ extern crate clap;
|
||||
extern crate thinp;
|
||||
|
||||
use clap::{App, Arg};
|
||||
use std::path::Path;
|
||||
use std::process;
|
||||
use thinp::file_utils;
|
||||
|
||||
@ -9,27 +10,31 @@ use std::process::exit;
|
||||
|
||||
fn main() {
|
||||
let parser = App::new("thin_metadata_unpack")
|
||||
.version(thinp::version::TOOLS_VERSION)
|
||||
.version(thinp::version::TOOLS_VERSION)
|
||||
.about("Unpack a compressed file of thin metadata.")
|
||||
.arg(Arg::with_name("INPUT")
|
||||
.help("Specify thinp metadata binary device/file")
|
||||
.required(true)
|
||||
.short("i")
|
||||
.value_name("DEV")
|
||||
.takes_value(true))
|
||||
.arg(Arg::with_name("OUTPUT")
|
||||
.help("Specify packed output file")
|
||||
.required(true)
|
||||
.short("o")
|
||||
.value_name("FILE")
|
||||
.takes_value(true));
|
||||
.arg(
|
||||
Arg::with_name("INPUT")
|
||||
.help("Specify thinp metadata binary device/file")
|
||||
.required(true)
|
||||
.short("i")
|
||||
.value_name("DEV")
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("OUTPUT")
|
||||
.help("Specify packed output file")
|
||||
.required(true)
|
||||
.short("o")
|
||||
.value_name("FILE")
|
||||
.takes_value(true),
|
||||
);
|
||||
|
||||
let matches = parser.get_matches();
|
||||
let input_file = matches.value_of("INPUT").unwrap();
|
||||
let output_file = matches.value_of("OUTPUT").unwrap();
|
||||
let input_file = Path::new(matches.value_of("INPUT").unwrap());
|
||||
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
|
||||
|
||||
if !file_utils::file_exists(input_file) {
|
||||
eprintln!("Couldn't find input file '{}'.", &input_file);
|
||||
eprintln!("Couldn't find input file '{}'.", &input_file.display());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
@ -2,6 +2,7 @@ extern crate clap;
|
||||
extern crate thinp;
|
||||
|
||||
use clap::{App, Arg};
|
||||
use std::path::Path;
|
||||
use std::process::exit;
|
||||
use thinp::file_utils;
|
||||
|
||||
@ -53,18 +54,20 @@ fn main() {
|
||||
let matches = parser.get_matches();
|
||||
|
||||
// FIXME: check these look like xml
|
||||
let input_file = matches.value_of("INPUT").unwrap();
|
||||
let output_file = matches.value_of("OUTPUT").unwrap();
|
||||
let input_file = Path::new(matches.value_of("INPUT").unwrap());
|
||||
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
|
||||
let size = matches.value_of("SIZE").unwrap().parse::<u64>().unwrap();
|
||||
let data_file = matches.value_of("DATA").unwrap();
|
||||
let data_file = Path::new(matches.value_of("DATA").unwrap());
|
||||
let do_copy = !matches.is_present("NOCOPY");
|
||||
|
||||
if !file_utils::file_exists(input_file) {
|
||||
eprintln!("Couldn't find input file '{}'.", &input_file);
|
||||
eprintln!("Couldn't find input file '{}'.", input_file.display());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy) {
|
||||
if let Err(reason) =
|
||||
thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy)
|
||||
{
|
||||
println!("Application error: {}\n", reason);
|
||||
exit(1);
|
||||
}
|
||||
|
@ -1,9 +1,10 @@
|
||||
use nix::sys::stat;
|
||||
use nix::sys::stat::{FileStat, SFlag};
|
||||
use std::fs::File;
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::io::{Seek, Write};
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::path::Path;
|
||||
use tempfile::tempfile;
|
||||
|
||||
//---------------------------------------
|
||||
@ -17,7 +18,7 @@ pub fn is_file_or_blk(info: FileStat) -> bool {
|
||||
|| check_bits(info.st_mode, &stat::SFlag::S_IFREG)
|
||||
}
|
||||
|
||||
pub fn file_exists(path: &str) -> bool {
|
||||
pub fn file_exists(path: &Path) -> bool {
|
||||
match stat::stat(path) {
|
||||
Ok(info) => is_file_or_blk(info),
|
||||
_ => {
|
||||
@ -39,7 +40,7 @@ pub fn fail<T>(msg: &str) -> io::Result<T> {
|
||||
Err(e)
|
||||
}
|
||||
|
||||
fn get_device_size(path: &str) -> io::Result<u64> {
|
||||
fn get_device_size(path: &Path) -> io::Result<u64> {
|
||||
let file = File::open(path)?;
|
||||
let fd = file.as_raw_fd();
|
||||
let mut cap = 0u64;
|
||||
@ -51,7 +52,7 @@ fn get_device_size(path: &str) -> io::Result<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn file_size(path: &str) -> io::Result<u64> {
|
||||
pub fn file_size(path: &Path) -> io::Result<u64> {
|
||||
match stat::stat(path) {
|
||||
Ok(info) => {
|
||||
if check_bits(info.st_mode, &SFlag::S_IFREG) {
|
||||
@ -68,16 +69,31 @@ pub fn file_size(path: &str) -> io::Result<u64> {
|
||||
|
||||
//---------------------------------------
|
||||
|
||||
pub fn temp_file_sized(nr_bytes: u64) -> io::Result<std::fs::File> {
|
||||
let mut file = tempfile()?;
|
||||
|
||||
fn set_size<W: Write + Seek>(w: &mut W, nr_bytes: u64) -> io::Result<()> {
|
||||
let zeroes: Vec<u8> = vec![0; 1];
|
||||
|
||||
if nr_bytes > 0 {
|
||||
file.seek(io::SeekFrom::Start(nr_bytes - 1))?;
|
||||
file.write_all(&zeroes)?;
|
||||
w.seek(io::SeekFrom::Start(nr_bytes - 1))?;
|
||||
w.write_all(&zeroes)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn temp_file_sized(nr_bytes: u64) -> io::Result<std::fs::File> {
|
||||
let mut file = tempfile()?;
|
||||
set_size(&mut file, nr_bytes)?;
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
pub fn create_sized_file(path: &Path, nr_bytes: u64) -> io::Result<std::fs::File> {
|
||||
let mut file = OpenOptions::new()
|
||||
.read(false)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
set_size(&mut file, nr_bytes)?;
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
|
@ -6,6 +6,7 @@ use std::os::unix::fs::OpenOptionsExt;
|
||||
use std::{
|
||||
error::Error,
|
||||
fs::OpenOptions,
|
||||
path::Path,
|
||||
io,
|
||||
io::prelude::*,
|
||||
io::Cursor,
|
||||
@ -67,7 +68,7 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec<Vec<(u64, u64)>> {
|
||||
vs
|
||||
}
|
||||
|
||||
pub fn pack(input_file: &str, output_file: &str) -> Result<(), Box<dyn Error>> {
|
||||
pub fn pack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>> {
|
||||
let nr_blocks = get_nr_blocks(&input_file)?;
|
||||
let nr_jobs = std::cmp::max(1, std::cmp::min(num_cpus::get() as u64, nr_blocks / 128));
|
||||
let chunk_vecs = mk_chunk_vecs(nr_blocks, nr_jobs);
|
||||
@ -192,7 +193,7 @@ where
|
||||
r.read_u64::<LittleEndian>()
|
||||
}
|
||||
|
||||
fn get_nr_blocks(path: &str) -> io::Result<u64> {
|
||||
fn get_nr_blocks(path: &Path) -> io::Result<u64> {
|
||||
let len = file_utils::file_size(path)?;
|
||||
Ok(len / (BLOCK_SIZE as u64))
|
||||
}
|
||||
@ -306,7 +307,7 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn unpack(input_file: &str, output_file: &str) -> Result<(), Box<dyn Error>> {
|
||||
pub fn unpack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>> {
|
||||
let mut input = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(false)
|
||||
|
@ -1,5 +1,6 @@
|
||||
use anyhow::Result;
|
||||
use std::fs::OpenOptions;
|
||||
use std::path::Path;
|
||||
use std::io::{Seek, SeekFrom, Write, Read};
|
||||
//use std::os::unix::fs::OpenOptionsExt;
|
||||
|
||||
@ -43,7 +44,7 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn copy(path: &str, regions: &[Region]) -> Result<()> {
|
||||
pub fn copy(path: &Path, regions: &[Region]) -> Result<()> {
|
||||
let mut input = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
|
@ -3,6 +3,7 @@ use fixedbitset::FixedBitSet;
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::shrink::copier::{self, Region};
|
||||
use crate::thin::xml::{self, Visit};
|
||||
@ -426,7 +427,7 @@ fn build_copy_regions(remaps: &[(BlockRange, BlockRange)], block_size: u64) -> V
|
||||
rs
|
||||
}
|
||||
|
||||
fn process_xml<MV: xml::MetadataVisitor>(input_path: &str, pass: &mut MV) -> Result<()> {
|
||||
fn process_xml<MV: xml::MetadataVisitor>(input_path: &Path, pass: &mut MV) -> Result<()> {
|
||||
let input = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(false)
|
||||
@ -437,7 +438,13 @@ fn process_xml<MV: xml::MetadataVisitor>(input_path: &str, pass: &mut MV) -> Res
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn shrink(input_path: &str, output_path: &str, data_path: &str, nr_blocks: u64, do_copy: bool) -> Result<()> {
|
||||
pub fn shrink(
|
||||
input_path: &Path,
|
||||
output_path: &Path,
|
||||
data_path: &Path,
|
||||
nr_blocks: u64,
|
||||
do_copy: bool,
|
||||
) -> Result<()> {
|
||||
let mut pass1 = Pass1::new(nr_blocks);
|
||||
eprint!("Reading xml...");
|
||||
process_xml(input_path, &mut pass1)?;
|
||||
|
@ -3,6 +3,8 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
|
||||
use rand::Rng;
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tempfile::tempdir;
|
||||
|
||||
use thinp::file_utils;
|
||||
use thinp::thin::xml::{self, Visit};
|
||||
@ -22,6 +24,7 @@ struct ThinReadRef {
|
||||
|
||||
struct ThinWriteRef<'a, W: Write + Seek> {
|
||||
file: &'a mut W,
|
||||
block_byte: u64,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
@ -38,6 +41,7 @@ impl ThinBlock {
|
||||
fn zero_ref<'a, W: Write + Seek>(&self, w: &'a mut W) -> ThinWriteRef<'a, W> {
|
||||
ThinWriteRef {
|
||||
file: w,
|
||||
block_byte: self.data_block * (self.block_size as u64),
|
||||
data: vec![0; self.block_size],
|
||||
}
|
||||
}
|
||||
@ -52,6 +56,7 @@ impl ThinBlock {
|
||||
|
||||
let wr = ThinWriteRef {
|
||||
file: w,
|
||||
block_byte: self.data_block * (self.block_size as u64),
|
||||
data: vec![0; self.block_size],
|
||||
};
|
||||
|
||||
@ -61,6 +66,10 @@ impl ThinBlock {
|
||||
|
||||
impl<'a, W: Write + Seek> Drop for ThinWriteRef<'a, W> {
|
||||
fn drop(&mut self) {
|
||||
// FIXME: We shouldn't panic in a drop function, so any IO
|
||||
// errors will have to make their way back to the user
|
||||
// another way (eg, via a flush() method).
|
||||
self.file.seek(SeekFrom::Start(self.block_byte)).unwrap();
|
||||
self.file.write_all(&self.data[0..]).unwrap();
|
||||
}
|
||||
}
|
||||
@ -232,28 +241,105 @@ impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> {
|
||||
|
||||
//------------------------------------
|
||||
|
||||
fn create_data_file(xml_path: &str) -> Result<std::fs::File> {
|
||||
fn mk_path(dir: &Path, file: &str) -> PathBuf {
|
||||
let mut p = PathBuf::new();
|
||||
p.push(dir);
|
||||
p.push(PathBuf::from(file));
|
||||
p
|
||||
}
|
||||
|
||||
fn generate_xml<G>(path: &Path, g: &mut G) -> Result<()>
|
||||
where
|
||||
G: XmlGenerator,
|
||||
{
|
||||
let xml_out = OpenOptions::new()
|
||||
.read(false)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
let mut w = xml::XmlWriter::new(xml_out);
|
||||
|
||||
g.generate(&mut w)
|
||||
}
|
||||
|
||||
fn create_data_file(data_path: &Path, xml_path: &Path) -> Result<()> {
|
||||
let input = OpenOptions::new().read(true).write(false).open(xml_path)?;
|
||||
|
||||
let sb = xml::read_superblock(input)?;
|
||||
let nr_blocks = sb.nr_data_blocks as u64;
|
||||
let block_size = sb.data_block_size as u64 * 512;
|
||||
|
||||
let file = file_utils::temp_file_sized(nr_blocks * block_size)?;
|
||||
Ok(file)
|
||||
let _file = file_utils::create_sized_file(data_path, nr_blocks * block_size)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main(xml_path: &str) -> Result<()> {
|
||||
let mut data_file = create_data_file(xml_path)?;
|
||||
let mut xml_in = OpenOptions::new().read(true).write(false).open(xml_path)?;
|
||||
fn stamp(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> {
|
||||
let mut data = OpenOptions::new()
|
||||
.read(false)
|
||||
.write(true)
|
||||
.open(&data_path)?;
|
||||
let xml = OpenOptions::new().read(true).write(false).open(&xml_path)?;
|
||||
|
||||
let mut stamper = Stamper::new(&mut data, seed);
|
||||
thin_visit(xml, &mut stamper)
|
||||
}
|
||||
|
||||
fn verify(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> {
|
||||
let mut data = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(false)
|
||||
.open(&data_path)?;
|
||||
let xml = OpenOptions::new().read(true).write(false).open(&xml_path)?;
|
||||
|
||||
let mut verifier = Verifier::new(&mut data, seed);
|
||||
thin_visit(xml, &mut verifier)
|
||||
}
|
||||
|
||||
//------------------------------------
|
||||
|
||||
trait XmlGenerator {
|
||||
fn generate(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>;
|
||||
}
|
||||
|
||||
struct EmptyPoolG {}
|
||||
|
||||
impl XmlGenerator for EmptyPoolG {
|
||||
fn generate(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
|
||||
v.superblock_b(&xml::Superblock {
|
||||
uuid: "".to_string(),
|
||||
time: 0,
|
||||
transaction: 0,
|
||||
flags: None,
|
||||
version: None,
|
||||
data_block_size: 64,
|
||||
nr_data_blocks: 1024,
|
||||
metadata_snap: None,
|
||||
})?;
|
||||
v.superblock_e()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shrink_empty_pool() -> Result<()> {
|
||||
let dir = tempdir()?;
|
||||
let xml_before = mk_path(dir.path(), "before.xml");
|
||||
let xml_after = mk_path(dir.path(), "after.xml");
|
||||
let data_path = mk_path(dir.path(), "bin");
|
||||
|
||||
let mut gen = EmptyPoolG {};
|
||||
generate_xml(&xml_before, &mut gen)?;
|
||||
create_data_file(&data_path, &xml_before)?;
|
||||
|
||||
let mut rng = rand::thread_rng();
|
||||
let seed = rng.gen::<u64>();
|
||||
|
||||
let mut stamper = Stamper::new(&mut data_file, seed);
|
||||
thin_visit(&mut xml_in, &mut stamper)?;
|
||||
stamp(&xml_before, &data_path, seed)?;
|
||||
|
||||
let mut verifier = Verifier::new(&mut data_file, seed);
|
||||
thin_visit(&mut xml_in, &mut verifier)?;
|
||||
let new_nr_blocks = 10;
|
||||
thinp::shrink::toplevel::shrink(&xml_before, &xml_after, &data_path, new_nr_blocks, true)?;
|
||||
|
||||
verify(&xml_after, &data_path, seed)?;
|
||||
Ok(())
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user