emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[thin_check (rust)] Add progress bar

Browse Source
This commit is contained in:
Joe Thornber
2020-08-11 13:22:14 +01:00
parent 34425521e2
commit e65d2dec6f
4 changed files with 201 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
Cargo.lock generated
View File

@@ -98,6 +98,23 @@ dependencies = [
"vec_map",
]
[[package]]
name = "console"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"regex",
"terminal_size",
"termios",
"unicode-width",
"winapi",
"winapi-util",
]
[[package]]
name = "crc32c"
version = "0.4.0"
@@ -125,6 +142,12 @@ dependencies = [
"shared_child",
]
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "env_logger"
version = "0.7.1"
@@ -268,6 +291,18 @@ dependencies = [
"libc",
]
[[package]]
name = "indicatif"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
dependencies = [
"console",
"lazy_static",
"number_prefix",
"regex",
]
[[package]]
name = "io-uring"
version = "0.3.5"
@@ -387,6 +422,12 @@ dependencies = [
"libc",
]
[[package]]
name = "number_prefix"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
[[package]]
name = "once_cell"
version = "1.4.0"
@@ -630,6 +671,25 @@ dependencies = [
"winapi",
]
[[package]]
name = "terminal_size"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "termios"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2"
dependencies = [
"libc",
]
[[package]]
name = "textwrap"
version = "0.11.0"
@@ -652,6 +712,7 @@ dependencies = [
"fixedbitset",
"flate2",
"futures",
"indicatif",
"io-uring",
"json",
"libc",
@@ -759,6 +820,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"

1
Cargo.toml
View File

@@ -16,6 +16,7 @@ fixedbitset = "0.3"
futures = "0.3"
flate2 = "1.0"
io-uring = "0.3"
indicatif = "0.15"
libc = "0.2.71"
nix = "0.17"
nom = "5.1"

32
src/pdata/space_map.rs
View File

@@ -156,6 +156,7 @@ impl Unpack for Bitmap {
pub trait SpaceMap {
fn get_nr_blocks(&self) -> Result<u64>;
fn get_nr_allocated(&self) -> Result<u64>;
fn get(&self, b: u64) -> Result<u32>;
fn inc(&mut self, begin: u64, len: u64) -> Result<()>;
}
@@ -163,6 +164,7 @@ pub trait SpaceMap {
//------------------------------------------
pub struct CoreSpaceMap<T> {
nr_allocated: u64,
counts: Vec<T>,
}
@@ -172,6 +174,7 @@ where
{
pub fn new(nr_entries: u64) -> CoreSpaceMap<V> {
CoreSpaceMap {
nr_allocated: 0,
counts: vec![V::default(); nr_entries as usize],
}
}
@@ -179,19 +182,29 @@ where
impl<V> SpaceMap for CoreSpaceMap<V>
where
V: Copy + Default + std::ops::AddAssign + From<u8> + Into<u32>,
V: Copy + Default + Eq + std::ops::AddAssign + From<u8> + Into<u32>,
{
fn get_nr_blocks(&self) -> Result<u64> {
Ok(self.counts.len() as u64)
}
fn get_nr_allocated(&self) -> Result<u64> {
Ok(self.nr_allocated)
}
fn get(&self, b: u64) -> Result<u32> {
Ok(self.counts[b as usize].into())
}
fn inc(&mut self, begin: u64, len: u64) -> Result<()> {
for b in begin..(begin + len) {
self.counts[b as usize] += V::from(1u8);
if self.counts[b as usize] == V::from(0u8) {
// FIXME: can we get a ref to save dereferencing counts twice?
self.nr_allocated += 1;
self.counts[b as usize] = V::from(1u8);
} else {
self.counts[b as usize] += V::from(1u8);
}
}
Ok(())
}
@@ -213,12 +226,14 @@ pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc<Mutex<dyn SpaceMap + Send
// btrees when we want to avoid visiting a node more than once, but
// aren't interested in counting how many times we've visited.
pub struct RestrictedSpaceMap {
nr_allocated: u64,
counts: FixedBitSet,
}
impl RestrictedSpaceMap {
pub fn new(nr_entries: u64) -> RestrictedSpaceMap {
RestrictedSpaceMap {
nr_allocated: 0,
counts: FixedBitSet::with_capacity(nr_entries as usize),
}
}
@@ -228,7 +243,11 @@ impl SpaceMap for RestrictedSpaceMap {
fn get_nr_blocks(&self) -> Result<u64> {
Ok(self.counts.len() as u64)
}
fn get_nr_allocated(&self) -> Result<u64> {
Ok(self.nr_allocated)
}
fn get(&self, b: u64) -> Result<u32> {
if self.counts.contains(b as usize) {
Ok(1)
@@ -239,7 +258,10 @@ impl SpaceMap for RestrictedSpaceMap {
fn inc(&mut self, begin: u64, len: u64) -> Result<()> {
for b in begin..(begin + len) {
self.counts.insert(b as usize);
if !self.counts.contains(b as usize) {
self.nr_allocated += 1;
self.counts.insert(b as usize);
}
}
Ok(())
}

112
src/thin/check.rs
View File

@@ -1,8 +1,11 @@
use anyhow::{anyhow, Result};
use indicatif::{ProgressBar, ProgressStyle};
use nom::{number::complete::*, IResult};
use std::collections::BTreeMap;
use std::path::Path;
use std::sync::mpsc::{channel, Receiver, Sender, TryRecvError};
use std::sync::{Arc, Mutex};
use std::{thread, time};
use threadpool::ThreadPool;
use crate::checksum;
@@ -176,6 +179,86 @@ impl<'a> NodeVisitor<u32> for OverflowChecker<'a> {
//------------------------------------------
enum SpinnerCmd {
Complete,
Abort,
Title(String),
}
struct Spinner {
tx: Sender<SpinnerCmd>,
tid: thread::JoinHandle<()>,
}
impl Spinner {
fn new(sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>, total_allocated: u64) -> Result<Spinner> {
let (tx, rx) = channel();
let tid = thread::spawn(move || spinner_thread(sm, total_allocated, rx));
Ok(Spinner { tx, tid })
}
fn complete(self) -> Result<()> {
self.tx.send(SpinnerCmd::Complete)?;
self.tid.join();
Ok(())
}
fn abort(self) -> Result<()> {
self.tx.send(SpinnerCmd::Abort)?;
self.tid.join();
Ok(())
}
fn set_title(&mut self, txt: &str) -> Result<()> {
self.tx.send(SpinnerCmd::Title(txt.to_string()))?;
Ok(())
}
}
fn spinner_thread(
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
total_allocated: u64,
rx: Receiver<SpinnerCmd>,
) {
let interval = time::Duration::from_millis(250);
let bar = ProgressBar::new(total_allocated);
loop {
match rx.try_recv() {
Ok(SpinnerCmd::Complete) => {
bar.finish();
return;
}
Ok(SpinnerCmd::Abort) => {
return;
}
Ok(SpinnerCmd::Title(txt)) => {
let mut fmt = "Checking thin metadata [{bar:40.cyan/blue}] Remaining {eta}, ".to_string();
fmt.push_str(&txt);
bar.set_style(
ProgressStyle::default_bar()
.template(&fmt)
.progress_chars("=> "),
);
}
Err(TryRecvError::Disconnected) => {
return;
}
Err(TryRecvError::Empty) => {}
}
let sm = sm.lock().unwrap();
let nr_allocated = sm.get_nr_allocated().unwrap();
drop(sm);
bar.set_position(nr_allocated);
bar.tick();
thread::sleep(interval);
}
}
//------------------------------------------
const MAX_CONCURRENT_IO: u32 = 1024;
pub struct ThinCheckOptions<'a> {
@@ -191,7 +274,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
nr_threads = std::cmp::min(4, num_cpus::get());
engine = Arc::new(AsyncIoEngine::new(opts.dev, MAX_CONCURRENT_IO)?);
} else {
eprintln!("falling back to synchronous io");
eprintln!("Using synchronous io");
nr_threads = num_cpus::get() * 2;
engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads)?);
}
@@ -199,30 +282,39 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
// superblock
let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?;
let nr_allocated_metadata;
{
let root = unpack::<SMRoot>(&sb.metadata_sm_root[0..])?;
nr_allocated_metadata = root.nr_allocated;
}
// Device details. We read this once to get the number of thin devices, and hence the
// maximum metadata ref count. Then create metadata space map, and reread to increment
// the ref counts for that metadata.
let devs = btree_to_map::<DeviceDetail>(engine.clone(), false, sb.details_root)?;
let nr_devs = devs.len();
let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32);
let mut spinner = Spinner::new(metadata_sm.clone(), nr_allocated_metadata)?;
spinner.set_title("device details tree")?;
let _devs = btree_to_map_with_sm::<DeviceDetail>(
engine.clone(),
metadata_sm.clone(),
false,
sb.details_root,
)?;
println!("found {} devices", nr_devs);
// increment superblock
{
let mut sm = metadata_sm.lock().unwrap();
sm.inc(SUPERBLOCK_LOCATION, 1)?;
}
// mapping top level
let roots = btree_to_map::<u64>(engine.clone(), false, sb.mapping_root)?;
// Check the mappings filling in the data_sm as we go.
spinner.set_title("mapping tree")?;
let data_sm;
{
// FIXME: with a thread pool we need to return errors another way.
@@ -232,7 +324,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
data_sm = core_sm(root.nr_blocks, nr_devs as u32);
for (thin_id, root) in roots {
for (_thin_id, root) in roots {
let mut w = BTreeWalker::new_with_sm(engine.clone(), metadata_sm.clone(), false)?;
let data_sm = data_sm.clone();
pool.execute(move || {
@@ -244,8 +336,8 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
eprintln!("walk failed {:?}", e);
std::process::abort();
}
Ok(result) => {
eprintln!("checked thin_dev {} -> {:?}", thin_id, result);
Ok(_result) => {
//eprintln!("checked thin_dev {} -> {:?}", thin_id, result);
}
}
});
@@ -256,10 +348,10 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
// Check the data space map.
{
spinner.set_title("data space map")?;
let data_sm = data_sm.lock().unwrap();
let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
let nr_data_blocks = root.nr_blocks;
eprintln!("data root: {:?}", root);
// overflow btree
{
@@ -270,7 +362,6 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
// Bitmaps
let entries = btree_to_map::<IndexEntry>(engine.clone(), false, root.bitmap_root)?;
eprintln!("{} index entries", entries.len());
let mut blocks = Vec::new();
for (_k, i) in &entries {
@@ -302,7 +393,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
BitmapEntry::Small(actual) => {
let expected = data_sm.get(blocknr)?;
if actual == 1 && expected == 0 {
eprintln!("Data block {} leaked.", blocknr);
// eprintln!("Data block {} leaked.", blocknr);
leaks += 1;
} else if actual != expected as u8 {
eprintln!("Bad reference count for data block {}. Expected {}, but space map contains {}.",
@@ -331,12 +422,15 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
}
if fail {
spinner.abort()?;
return Err(anyhow!("Inconsistent data space map"));
}
}
// Check the metadata space map.
spinner.set_title("metadata space map")?;
spinner.complete()?;
Ok(())
}