emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[thin_shrink] Get test framework built

Browse Source 
This verifies data has been moved to the correct place.
This commit is contained in:
Joe Thornber 2020-06-29 10:49:40 +01:00
parent d03dac8f75
commit 3618da3a12
8 changed files with 172 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/bin/thin_metadata_pack.rs
View File

@ -2,6 +2,7 @@ extern crate clap;
extern crate thinp;
use clap::{App, Arg};
use std::path::Path;
use std::process::exit;
use thinp::file_utils;
@ -23,14 +24,14 @@ fn main() {
.takes_value(true));
let matches = parser.get_matches();
let input_file = matches.value_of("INPUT").unwrap();
let output_file = matches.value_of("OUTPUT").unwrap();
let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file);
if !file_utils::file_exists(&input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file.display());
exit(1);
}
if let Err(reason) = thinp::pack::toplevel::pack(&input_file, &output_file) {
println!("Application error: {}\n", reason);
exit(1);

39
src/bin/thin_metadata_unpack.rs
View File

@ -2,6 +2,7 @@ extern crate clap;
extern crate thinp;
use clap::{App, Arg};
use std::path::Path;
use std::process;
use thinp::file_utils;
@ -9,30 +10,34 @@ use std::process::exit;
fn main() {
let parser = App::new("thin_metadata_unpack")
.version(thinp::version::TOOLS_VERSION)
.version(thinp::version::TOOLS_VERSION)
.about("Unpack a compressed file of thin metadata.")
.arg(Arg::with_name("INPUT")
.help("Specify thinp metadata binary device/file")
.required(true)
.short("i")
.value_name("DEV")
.takes_value(true))
.arg(Arg::with_name("OUTPUT")
.help("Specify packed output file")
.required(true)
.short("o")
.value_name("FILE")
.takes_value(true));
.arg(
Arg::with_name("INPUT")
.help("Specify thinp metadata binary device/file")
.required(true)
.short("i")
.value_name("DEV")
.takes_value(true),
)
.arg(
Arg::with_name("OUTPUT")
.help("Specify packed output file")
.required(true)
.short("o")
.value_name("FILE")
.takes_value(true),
);
let matches = parser.get_matches();
let input_file = matches.value_of("INPUT").unwrap();
let output_file = matches.value_of("OUTPUT").unwrap();
let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file);
eprintln!("Couldn't find input file '{}'.", &input_file.display());
exit(1);
}
if let Err(reason) = thinp::pack::toplevel::unpack(&input_file, &output_file) {
println!("Application error: {}", reason);
process::exit(1);

13
src/bin/thin_shrink.rs
View File

@ -2,6 +2,7 @@ extern crate clap;
extern crate thinp;
use clap::{App, Arg};
use std::path::Path;
use std::process::exit;
use thinp::file_utils;
@ -53,18 +54,20 @@ fn main() {
let matches = parser.get_matches();
// FIXME: check these look like xml
let input_file = matches.value_of("INPUT").unwrap();
let output_file = matches.value_of("OUTPUT").unwrap();
let input_file = Path::new(matches.value_of("INPUT").unwrap());
let output_file = Path::new(matches.value_of("OUTPUT").unwrap());
let size = matches.value_of("SIZE").unwrap().parse::<u64>().unwrap();
let data_file = matches.value_of("DATA").unwrap();
let data_file = Path::new(matches.value_of("DATA").unwrap());
let do_copy = !matches.is_present("NOCOPY");
if !file_utils::file_exists(input_file) {
eprintln!("Couldn't find input file '{}'.", &input_file);
eprintln!("Couldn't find input file '{}'.", input_file.display());
exit(1);
}
if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy) {
if let Err(reason) =
thinp::shrink::toplevel::shrink(&input_file, &output_file, &data_file, size, do_copy)
{
println!("Application error: {}\n", reason);
exit(1);
}

34
src/file_utils.rs
View File

@ -1,9 +1,10 @@
use nix::sys::stat;
use nix::sys::stat::{FileStat, SFlag};
use std::fs::File;
use std::fs::{File, OpenOptions};
use std::io;
use std::io::{Seek, Write};
use std::os::unix::io::AsRawFd;
use std::path::Path;
use tempfile::tempfile;
//---------------------------------------
@ -17,7 +18,7 @@ pub fn is_file_or_blk(info: FileStat) -> bool {
|| check_bits(info.st_mode, &stat::SFlag::S_IFREG)
}
pub fn file_exists(path: &str) -> bool {
pub fn file_exists(path: &Path) -> bool {
match stat::stat(path) {
Ok(info) => is_file_or_blk(info),
_ => {
@ -39,7 +40,7 @@ pub fn fail<T>(msg: &str) -> io::Result<T> {
Err(e)
}
fn get_device_size(path: &str) -> io::Result<u64> {
fn get_device_size(path: &Path) -> io::Result<u64> {
let file = File::open(path)?;
let fd = file.as_raw_fd();
let mut cap = 0u64;
@ -51,7 +52,7 @@ fn get_device_size(path: &str) -> io::Result<u64> {
}
}
pub fn file_size(path: &str) -> io::Result<u64> {
pub fn file_size(path: &Path) -> io::Result<u64> {
match stat::stat(path) {
Ok(info) => {
if check_bits(info.st_mode, &SFlag::S_IFREG) {
@ -68,16 +69,31 @@ pub fn file_size(path: &str) -> io::Result<u64> {
//---------------------------------------
pub fn temp_file_sized(nr_bytes: u64) -> io::Result<std::fs::File> {
let mut file = tempfile()?;
fn set_size<W: Write + Seek>(w: &mut W, nr_bytes: u64) -> io::Result<()> {
let zeroes: Vec<u8> = vec![0; 1];
if nr_bytes > 0 {
file.seek(io::SeekFrom::Start(nr_bytes - 1))?;
file.write_all(&zeroes)?;
w.seek(io::SeekFrom::Start(nr_bytes - 1))?;
w.write_all(&zeroes)?;
}
Ok(())
}
pub fn temp_file_sized(nr_bytes: u64) -> io::Result<std::fs::File> {
let mut file = tempfile()?;
set_size(&mut file, nr_bytes)?;
Ok(file)
}
pub fn create_sized_file(path: &Path, nr_bytes: u64) -> io::Result<std::fs::File> {
let mut file = OpenOptions::new()
.read(false)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
set_size(&mut file, nr_bytes)?;
Ok(file)
}

7
src/pack/toplevel.rs
View File

@ -6,6 +6,7 @@ use std::os::unix::fs::OpenOptionsExt;
use std::{
error::Error,
fs::OpenOptions,
path::Path,
io,
io::prelude::*,
io::Cursor,
@ -67,7 +68,7 @@ fn mk_chunk_vecs(nr_blocks: u64, nr_jobs: u64) -> Vec<Vec<(u64, u64)>> {
vs
}
pub fn pack(input_file: &str, output_file: &str) -> Result<(), Box<dyn Error>> {
pub fn pack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>> {
let nr_blocks = get_nr_blocks(&input_file)?;
let nr_jobs = std::cmp::max(1, std::cmp::min(num_cpus::get() as u64, nr_blocks / 128));
let chunk_vecs = mk_chunk_vecs(nr_blocks, nr_jobs);
@ -192,7 +193,7 @@ where
r.read_u64::<LittleEndian>()
}
fn get_nr_blocks(path: &str) -> io::Result<u64> {
fn get_nr_blocks(path: &Path) -> io::Result<u64> {
let len = file_utils::file_size(path)?;
Ok(len / (BLOCK_SIZE as u64))
}
@ -306,7 +307,7 @@ where
Ok(())
}
pub fn unpack(input_file: &str, output_file: &str) -> Result<(), Box<dyn Error>> {
pub fn unpack(input_file: &Path, output_file: &Path) -> Result<(), Box<dyn Error>> {
let mut input = OpenOptions::new()
.read(true)
.write(false)

3
src/shrink/copier.rs
View File

@ -1,5 +1,6 @@
use anyhow::Result;
use std::fs::OpenOptions;
use std::path::Path;
use std::io::{Seek, SeekFrom, Write, Read};
//use std::os::unix::fs::OpenOptionsExt;
@ -43,7 +44,7 @@ where
Ok(())
}
pub fn copy(path: &str, regions: &[Region]) -> Result<()> {
pub fn copy(path: &Path, regions: &[Region]) -> Result<()> {
let mut input = OpenOptions::new()
.read(true)
.write(true)

11
src/shrink/toplevel.rs
View File

@ -3,6 +3,7 @@ use fixedbitset::FixedBitSet;
use std::fs::OpenOptions;
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
use std::path::Path;
use crate::shrink::copier::{self, Region};
use crate::thin::xml::{self, Visit};
@ -426,7 +427,7 @@ fn build_copy_regions(remaps: &[(BlockRange, BlockRange)], block_size: u64) -> V
rs
}
fn process_xml<MV: xml::MetadataVisitor>(input_path: &str, pass: &mut MV) -> Result<()> {
fn process_xml<MV: xml::MetadataVisitor>(input_path: &Path, pass: &mut MV) -> Result<()> {
let input = OpenOptions::new()
.read(true)
.write(false)
@ -437,7 +438,13 @@ fn process_xml<MV: xml::MetadataVisitor>(input_path: &str, pass: &mut MV) -> Res
Ok(())
}
pub fn shrink(input_path: &str, output_path: &str, data_path: &str, nr_blocks: u64, do_copy: bool) -> Result<()> {
pub fn shrink(
input_path: &Path,
output_path: &Path,
data_path: &Path,
nr_blocks: u64,
do_copy: bool,
) -> Result<()> {
let mut pass1 = Pass1::new(nr_blocks);
eprint!("Reading xml...");
process_xml(input_path, &mut pass1)?;

106
tests/thin_shrink.rs
View File

@ -3,6 +3,8 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::Rng;
use std::fs::OpenOptions;
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use tempfile::tempdir;
use thinp::file_utils;
use thinp::thin::xml::{self, Visit};
@ -22,6 +24,7 @@ struct ThinReadRef {
struct ThinWriteRef<'a, W: Write + Seek> {
file: &'a mut W,
block_byte: u64,
pub data: Vec<u8>,
}
@ -38,6 +41,7 @@ impl ThinBlock {
fn zero_ref<'a, W: Write + Seek>(&self, w: &'a mut W) -> ThinWriteRef<'a, W> {
ThinWriteRef {
file: w,
block_byte: self.data_block * (self.block_size as u64),
data: vec![0; self.block_size],
}
}
@ -52,6 +56,7 @@ impl ThinBlock {
let wr = ThinWriteRef {
file: w,
block_byte: self.data_block * (self.block_size as u64),
data: vec![0; self.block_size],
};
@ -61,6 +66,10 @@ impl ThinBlock {
impl<'a, W: Write + Seek> Drop for ThinWriteRef<'a, W> {
fn drop(&mut self) {
// FIXME: We shouldn't panic in a drop function, so any IO
// errors will have to make their way back to the user
// another way (eg, via a flush() method).
self.file.seek(SeekFrom::Start(self.block_byte)).unwrap();
self.file.write_all(&self.data[0..]).unwrap();
}
}
@ -232,28 +241,105 @@ impl<'a, R: Read + Seek> ThinVisitor for Verifier<'a, R> {
//------------------------------------
fn create_data_file(xml_path: &str) -> Result<std::fs::File> {
fn mk_path(dir: &Path, file: &str) -> PathBuf {
let mut p = PathBuf::new();
p.push(dir);
p.push(PathBuf::from(file));
p
}
fn generate_xml<G>(path: &Path, g: &mut G) -> Result<()>
where
G: XmlGenerator,
{
let xml_out = OpenOptions::new()
.read(false)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
let mut w = xml::XmlWriter::new(xml_out);
g.generate(&mut w)
}
fn create_data_file(data_path: &Path, xml_path: &Path) -> Result<()> {
let input = OpenOptions::new().read(true).write(false).open(xml_path)?;
let sb = xml::read_superblock(input)?;
let nr_blocks = sb.nr_data_blocks as u64;
let block_size = sb.data_block_size as u64 * 512;
let file = file_utils::temp_file_sized(nr_blocks * block_size)?;
Ok(file)
let _file = file_utils::create_sized_file(data_path, nr_blocks * block_size)?;
Ok(())
}
fn main(xml_path: &str) -> Result<()> {
let mut data_file = create_data_file(xml_path)?;
let mut xml_in = OpenOptions::new().read(true).write(false).open(xml_path)?;
fn stamp(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> {
let mut data = OpenOptions::new()
.read(false)
.write(true)
.open(&data_path)?;
let xml = OpenOptions::new().read(true).write(false).open(&xml_path)?;
let mut stamper = Stamper::new(&mut data, seed);
thin_visit(xml, &mut stamper)
}
fn verify(xml_path: &Path, data_path: &Path, seed: u64) -> Result<()> {
let mut data = OpenOptions::new()
.read(true)
.write(false)
.open(&data_path)?;
let xml = OpenOptions::new().read(true).write(false).open(&xml_path)?;
let mut verifier = Verifier::new(&mut data, seed);
thin_visit(xml, &mut verifier)
}
//------------------------------------
trait XmlGenerator {
fn generate(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()>;
}
struct EmptyPoolG {}
impl XmlGenerator for EmptyPoolG {
fn generate(&mut self, v: &mut dyn xml::MetadataVisitor) -> Result<()> {
v.superblock_b(&xml::Superblock {
uuid: "".to_string(),
time: 0,
transaction: 0,
flags: None,
version: None,
data_block_size: 64,
nr_data_blocks: 1024,
metadata_snap: None,
})?;
v.superblock_e()?;
Ok(())
}
}
#[test]
fn shrink_empty_pool() -> Result<()> {
let dir = tempdir()?;
let xml_before = mk_path(dir.path(), "before.xml");
let xml_after = mk_path(dir.path(), "after.xml");
let data_path = mk_path(dir.path(), "bin");
let mut gen = EmptyPoolG {};
generate_xml(&xml_before, &mut gen)?;
create_data_file(&data_path, &xml_before)?;
let mut rng = rand::thread_rng();
let seed = rng.gen::<u64>();
let mut stamper = Stamper::new(&mut data_file, seed);
thin_visit(&mut xml_in, &mut stamper)?;
stamp(&xml_before, &data_path, seed)?;
let mut verifier = Verifier::new(&mut data_file, seed);
thin_visit(&mut xml_in, &mut verifier)?;
let new_nr_blocks = 10;
thinp::shrink::toplevel::shrink(&xml_before, &xml_after, &data_path, new_nr_blocks, true)?;
verify(&xml_after, &data_path, seed)?;
Ok(())
}