[thin_check (rust)] Add progress bar

Cargo.lock

@@ -98,6 +98,23 @@ dependencies = [
  "vec_map",
 ]
 
+[[package]]
+name = "console"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "regex",
+ "terminal_size",
+ "termios",
+ "unicode-width",
+ "winapi",
+ "winapi-util",
+]
+
 [[package]]
 name = "crc32c"
 version = "0.4.0"
@@ -125,6 +142,12 @@ dependencies = [
  "shared_child",
 ]
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
 [[package]]
 name = "env_logger"
 version = "0.7.1"
@@ -268,6 +291,18 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
+dependencies = [
+ "console",
+ "lazy_static",
+ "number_prefix",
+ "regex",
+]
+
 [[package]]
 name = "io-uring"
 version = "0.3.5"
@@ -387,6 +422,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
+
 [[package]]
 name = "once_cell"
 version = "1.4.0"
@@ -630,6 +671,25 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "terminal_size"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "termios"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"
@@ -652,6 +712,7 @@ dependencies = [
  "fixedbitset",
  "flate2",
  "futures",
+ "indicatif",
  "io-uring",
  "json",
  "libc",
@@ -759,6 +820,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"

Cargo.toml

@@ -16,6 +16,7 @@ fixedbitset = "0.3"
 futures = "0.3"
 flate2 = "1.0"
 io-uring = "0.3"
+indicatif = "0.15"
 libc = "0.2.71"
 nix = "0.17"
 nom = "5.1"

src/pdata/space_map.rs

@@ -156,6 +156,7 @@ impl Unpack for Bitmap {
 
 pub trait SpaceMap {
     fn get_nr_blocks(&self) -> Result<u64>;
+    fn get_nr_allocated(&self) -> Result<u64>;
     fn get(&self, b: u64) -> Result<u32>;
     fn inc(&mut self, begin: u64, len: u64) -> Result<()>;
 }
@@ -163,6 +164,7 @@ pub trait SpaceMap {
 //------------------------------------------
 
 pub struct CoreSpaceMap<T> {
+    nr_allocated: u64,
     counts: Vec<T>,
 }
 
@@ -172,6 +174,7 @@ where
 {
     pub fn new(nr_entries: u64) -> CoreSpaceMap<V> {
         CoreSpaceMap {
+            nr_allocated: 0,
             counts: vec![V::default(); nr_entries as usize],
         }
     }
@@ -179,19 +182,29 @@ where
 
 impl<V> SpaceMap for CoreSpaceMap<V>
 where
-    V: Copy + Default + std::ops::AddAssign + From<u8> + Into<u32>,
+    V: Copy + Default + Eq + std::ops::AddAssign + From<u8> + Into<u32>,
 {
     fn get_nr_blocks(&self) -> Result<u64> {
         Ok(self.counts.len() as u64)
     }
-    
+
+    fn get_nr_allocated(&self) -> Result<u64> {
+        Ok(self.nr_allocated)
+    }
+
     fn get(&self, b: u64) -> Result<u32> {
         Ok(self.counts[b as usize].into())
     }
 
     fn inc(&mut self, begin: u64, len: u64) -> Result<()> {
         for b in begin..(begin + len) {
-            self.counts[b as usize] += V::from(1u8);
+            if self.counts[b as usize] == V::from(0u8) {
+                // FIXME: can we get a ref to save dereferencing counts twice?
+                self.nr_allocated += 1;
+                self.counts[b as usize] = V::from(1u8);
+            } else {
+                self.counts[b as usize] += V::from(1u8);
+            }
         }
         Ok(())
     }
@@ -213,12 +226,14 @@ pub fn core_sm(nr_entries: u64, max_count: u32) -> Arc<Mutex<dyn SpaceMap + Send
 // btrees when we want to avoid visiting a node more than once, but
 // aren't interested in counting how many times we've visited.
 pub struct RestrictedSpaceMap {
+    nr_allocated: u64,
     counts: FixedBitSet,
 }
 
 impl RestrictedSpaceMap {
     pub fn new(nr_entries: u64) -> RestrictedSpaceMap {
         RestrictedSpaceMap {
+            nr_allocated: 0,
             counts: FixedBitSet::with_capacity(nr_entries as usize),
         }
     }
@@ -228,7 +243,11 @@ impl SpaceMap for RestrictedSpaceMap {
     fn get_nr_blocks(&self) -> Result<u64> {
         Ok(self.counts.len() as u64)
     }
-    
+
+    fn get_nr_allocated(&self) -> Result<u64> {
+        Ok(self.nr_allocated)
+    }
+
     fn get(&self, b: u64) -> Result<u32> {
         if self.counts.contains(b as usize) {
             Ok(1)
@@ -239,7 +258,10 @@ impl SpaceMap for RestrictedSpaceMap {
 
     fn inc(&mut self, begin: u64, len: u64) -> Result<()> {
         for b in begin..(begin + len) {
-            self.counts.insert(b as usize);
+            if !self.counts.contains(b as usize) {
+                self.nr_allocated += 1;
+                self.counts.insert(b as usize);
+            }
         }
         Ok(())
     }

src/thin/check.rs

@@ -1,8 +1,11 @@
 use anyhow::{anyhow, Result};
+use indicatif::{ProgressBar, ProgressStyle};
 use nom::{number::complete::*, IResult};
 use std::collections::BTreeMap;
 use std::path::Path;
+use std::sync::mpsc::{channel, Receiver, Sender, TryRecvError};
 use std::sync::{Arc, Mutex};
+use std::{thread, time};
 use threadpool::ThreadPool;
 
 use crate::checksum;
@@ -176,6 +179,86 @@ impl<'a> NodeVisitor<u32> for OverflowChecker<'a> {
 
 //------------------------------------------
 
+enum SpinnerCmd {
+    Complete,
+    Abort,
+    Title(String),
+}
+
+struct Spinner {
+    tx: Sender<SpinnerCmd>,
+    tid: thread::JoinHandle<()>,
+}
+
+impl Spinner {
+    fn new(sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>, total_allocated: u64) -> Result<Spinner> {
+        let (tx, rx) = channel();
+        let tid = thread::spawn(move || spinner_thread(sm, total_allocated, rx));
+        Ok(Spinner { tx, tid })
+    }
+
+    fn complete(self) -> Result<()> {
+        self.tx.send(SpinnerCmd::Complete)?;
+        self.tid.join();
+        Ok(())
+    }
+
+    fn abort(self) -> Result<()> {
+        self.tx.send(SpinnerCmd::Abort)?;
+        self.tid.join();
+        Ok(())
+    }
+
+    fn set_title(&mut self, txt: &str) -> Result<()> {
+        self.tx.send(SpinnerCmd::Title(txt.to_string()))?;
+        Ok(())
+    }
+}
+
+fn spinner_thread(
+    sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
+    total_allocated: u64,
+    rx: Receiver<SpinnerCmd>,
+) {
+    let interval = time::Duration::from_millis(250);
+    let bar = ProgressBar::new(total_allocated);
+    loop {
+        match rx.try_recv() {
+            Ok(SpinnerCmd::Complete) => {
+                bar.finish();
+                return;
+            }
+            Ok(SpinnerCmd::Abort) => {
+                return;
+            }
+            Ok(SpinnerCmd::Title(txt)) => {
+                let mut fmt = "Checking thin metadata [{bar:40.cyan/blue}] Remaining {eta}, ".to_string();
+                fmt.push_str(&txt);
+                bar.set_style(
+                    ProgressStyle::default_bar()
+                        .template(&fmt)
+                        .progress_chars("=> "),
+                );
+            }
+            Err(TryRecvError::Disconnected) => {
+                return;
+            }
+            Err(TryRecvError::Empty) => {}
+        }
+
+        let sm = sm.lock().unwrap();
+        let nr_allocated = sm.get_nr_allocated().unwrap();
+        drop(sm);
+
+        bar.set_position(nr_allocated);
+        bar.tick();
+
+        thread::sleep(interval);
+    }
+}
+
+//------------------------------------------
+
 const MAX_CONCURRENT_IO: u32 = 1024;
 
 pub struct ThinCheckOptions<'a> {
@@ -191,7 +274,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
         nr_threads = std::cmp::min(4, num_cpus::get());
         engine = Arc::new(AsyncIoEngine::new(opts.dev, MAX_CONCURRENT_IO)?);
     } else {
-        eprintln!("falling back to synchronous io");
+        eprintln!("Using synchronous io");
         nr_threads = num_cpus::get() * 2;
         engine = Arc::new(SyncIoEngine::new(opts.dev, nr_threads)?);
     }
@@ -199,30 +282,39 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
     // superblock
     let sb = read_superblock(engine.as_ref(), SUPERBLOCK_LOCATION)?;
 
+    let nr_allocated_metadata;
+    {
+        let root = unpack::<SMRoot>(&sb.metadata_sm_root[0..])?;
+        nr_allocated_metadata = root.nr_allocated;
+    }
+
     // Device details.   We read this once to get the number of thin devices, and hence the
     // maximum metadata ref count.  Then create metadata space map, and reread to increment
     // the ref counts for that metadata.
     let devs = btree_to_map::<DeviceDetail>(engine.clone(), false, sb.details_root)?;
     let nr_devs = devs.len();
     let metadata_sm = core_sm(engine.get_nr_blocks(), nr_devs as u32);
+    let mut spinner = Spinner::new(metadata_sm.clone(), nr_allocated_metadata)?;
+
+    spinner.set_title("device details tree")?;
     let _devs = btree_to_map_with_sm::<DeviceDetail>(
         engine.clone(),
         metadata_sm.clone(),
         false,
         sb.details_root,
     )?;
-    println!("found {} devices", nr_devs);
 
     // increment superblock
     {
         let mut sm = metadata_sm.lock().unwrap();
         sm.inc(SUPERBLOCK_LOCATION, 1)?;
     }
-    
+
     // mapping top level
     let roots = btree_to_map::<u64>(engine.clone(), false, sb.mapping_root)?;
 
     // Check the mappings filling in the data_sm as we go.
+    spinner.set_title("mapping tree")?;
     let data_sm;
     {
         // FIXME: with a thread pool we need to return errors another way.
@@ -232,7 +324,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
         let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
         data_sm = core_sm(root.nr_blocks, nr_devs as u32);
 
-        for (thin_id, root) in roots {
+        for (_thin_id, root) in roots {
             let mut w = BTreeWalker::new_with_sm(engine.clone(), metadata_sm.clone(), false)?;
             let data_sm = data_sm.clone();
             pool.execute(move || {
@@ -244,8 +336,8 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
                         eprintln!("walk failed {:?}", e);
                         std::process::abort();
                     }
-                    Ok(result) => {
+                    Ok(_result) => {
-                        eprintln!("checked thin_dev {} -> {:?}", thin_id, result);
+                        //eprintln!("checked thin_dev {} -> {:?}", thin_id, result);
                     }
                 }
             });
@@ -256,10 +348,10 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
 
     // Check the data space map.
     {
+        spinner.set_title("data space map")?;
         let data_sm = data_sm.lock().unwrap();
         let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
         let nr_data_blocks = root.nr_blocks;
-        eprintln!("data root: {:?}", root);
 
         // overflow btree
         {
@@ -270,7 +362,6 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
 
         // Bitmaps
         let entries = btree_to_map::<IndexEntry>(engine.clone(), false, root.bitmap_root)?;
-        eprintln!("{} index entries", entries.len());
 
         let mut blocks = Vec::new();
         for (_k, i) in &entries {
@@ -302,7 +393,7 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
                     BitmapEntry::Small(actual) => {
                         let expected = data_sm.get(blocknr)?;
                         if actual == 1 && expected == 0 {
-                            eprintln!("Data block {} leaked.", blocknr);
+                            // eprintln!("Data block {} leaked.", blocknr);
                             leaks += 1;
                         } else if actual != expected as u8 {
                             eprintln!("Bad reference count for data block {}.  Expected {}, but space map contains {}.",
@@ -331,12 +422,15 @@ pub fn check(opts: &ThinCheckOptions) -> Result<()> {
         }
 
         if fail {
+            spinner.abort()?;
             return Err(anyhow!("Inconsistent data space map"));
         }
     }
 
     // Check the metadata space map.
+    spinner.set_title("metadata space map")?;
 
+    spinner.complete()?;
     Ok(())
 }