[thin_shrink] calculate remaps

2020-06-25 10:44:57 +01:00 · 2020-06-25 10:44:57 +01:00 · 259eef9eee
commit 259eef9eee
parent 3f1b776359
5 changed files with 265 additions and 37 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -105,6 +105,11 @@ dependencies = [
 "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "fixedbitset"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "flate2"
 version = "1.0.14"
@ -397,6 +402,7 @@ dependencies = [
 "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
 "clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)",
 "crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fixedbitset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 "flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
 "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)",
 "nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -483,6 +489,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum crc32c 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77ba37ef26c12988c1cee882d522d65e1d5d2ad8c3864665b88ee92767ed84c5"
 "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
 "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
+"checksum fixedbitset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4fcacf5cd3681968f6524ea159383132937739c6c40dabab9e37ed515911b"
 "checksum flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42"
 "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
 "checksum hermit-abi 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -11,6 +11,7 @@ byteorder = "1.3"
 clap = "2.33"
 crc32c = "0.4"
 flate2 = "1.0"
+fixedbitset = "0.3"
 libc = "0.2.71"
 quick-xml = "0.18"
 nix = "0.17"
--- a/src/bin/thin_shrink.rs
+++ b/src/bin/thin_shrink.rs
@ -7,33 +7,47 @@ use thinp::file_utils;

 fn main() {
    let parser = App::new("thin_shrink")
-	.version(thinp::version::TOOLS_VERSION)
+        .version(thinp::version::TOOLS_VERSION)
        .about("Rewrite xml metadata and move data in an inactive pool.")
-        .arg(Arg::with_name("INPUT")
-            .help("Specify thinp metadata xml file")
-            .required(true)
-            .long("input")
-            .value_name("INPUT")
-            .takes_value(true))
-        .arg(Arg::with_name("OUTPUT")
-            .help("Specify output xml file")
-            .required(true)
-            .long("output")
-            .value_name("OUTPUT")
-            .takes_value(true));
+        .arg(
+            Arg::with_name("INPUT")
+                .help("Specify thinp metadata xml file")
+                .required(true)
+                .long("input")
+                .value_name("INPUT")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("OUTPUT")
+                .help("Specify output xml file")
+                .required(true)
+                .long("output")
+                .value_name("OUTPUT")
+                .takes_value(true),
+        )
+        // FIXME: support various disk units
+        .arg(
+            Arg::with_name("SIZE")
+                .help("Specify new size for the pool (in data blocks)")
+                .required(true)
+                .long("nr-blocks")
+                .value_name("SIZE")
+                .takes_value(true),
+        );

    let matches = parser.get_matches();

    // FIXME: check these look like xml
    let input_file = matches.value_of("INPUT").unwrap();
    let output_file = matches.value_of("OUTPUT").unwrap();
+    let size = matches.value_of("SIZE").unwrap().parse::<u64>().unwrap();

    if !file_utils::file_exists(input_file) {
        eprintln!("Couldn't find input file '{}'.", &input_file);
        exit(1);
    }

-    if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file) {
+    if let Err(reason) = thinp::shrink::toplevel::shrink(&input_file, &output_file, size) {
        println!("Application error: {}\n", reason);
        exit(1);
    }
--- a/src/shrink/toplevel.rs
+++ b/src/shrink/toplevel.rs
@ -1,4 +1,5 @@
 use anyhow::Result;
+use fixedbitset::{FixedBitSet, IndexRange};
 use std::fs::OpenOptions;
 use std::os::unix::fs::OpenOptionsExt;

@ -6,7 +7,178 @@ use crate::shrink::xml;

 //---------------------------------------

-pub fn shrink(input_file: &str, _output_file: &str) -> Result<()> {
+#[derive(Debug)]
+struct Pass1 {
+    // FIXME: Inefficient, use a range_set of some description
+    allocated_blocks: FixedBitSet,
+
+    nr_blocks: u64,
+
+    /// High blocks are beyond the new, reduced end of the pool.  These
+    /// will need to be moved.
+    nr_high_blocks: u64,
+}
+
+impl Pass1 {
+    fn new(nr_blocks: u64) -> Pass1 {
+        Pass1 {
+            allocated_blocks: FixedBitSet::with_capacity(0),
+            nr_blocks,
+            nr_high_blocks: 0,
+        }
+    }
+}
+
+impl xml::MetadataVisitor for Pass1 {
+    fn superblock_b(&mut self, sb: &xml::Superblock) -> Result<()> {
+        self.allocated_blocks.grow(sb.nr_data_blocks as usize);
+        Ok(())
+    }
+
+    fn superblock_e(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    fn device_b(&mut self, _d: &xml::Device) -> Result<()> {
+        Ok(())
+    }
+
+    fn device_e(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    fn map(&mut self, m: xml::Map) -> Result<()> {
+        for i in m.data_begin..(m.data_begin + m.len) {
+            if i > self.nr_blocks {
+                self.nr_high_blocks += 1;
+            }
+            self.allocated_blocks.insert(i as usize);
+        }
+        Ok(())
+    }
+
+    fn eof(&mut self) -> Result<()> {
+        Ok(())
+    }
+}
+
+type BlockRange = std::ops::Range<u64>;
+
+fn bits_to_ranges(bits: &FixedBitSet) -> Vec<BlockRange> {
+    let mut ranges = Vec::new();
+    let mut start = None;
+
+    for i in 0..bits.len() {
+        match (bits[i], start) {
+            (false, None) => {}
+            (true, None) => {
+                start = Some((i as u64, 1));
+            }
+            (false, Some((b, len))) => {
+                ranges.push(b..(b + len));
+                start = None;
+            }
+            (true, Some((b, len))) => {
+                start = Some((b, len + 1));
+            }
+        }
+    }
+
+    if let Some((b, len)) = start {
+        ranges.push(b..(b + len));
+    }
+
+    ranges
+}
+
+// Splits the ranges into those below threshold, and those equal or
+// above threshold below threshold, and those equal or above threshold
+fn ranges_split(ranges: &Vec<BlockRange>, threshold: u64) -> (Vec<BlockRange>, Vec<BlockRange>) {
+    use std::ops::Range;
+
+    let mut below = Vec::new();
+    let mut above = Vec::new();
+    for r in ranges {
+        match r {
+            Range { start, end } if *end <= threshold => below.push(*start..*end),
+            Range { start, end } if *start < threshold => {
+                below.push(*start..threshold);
+                above.push(threshold..*end);
+            }
+            Range { start, end } => above.push(*start..*end),
+        }
+    }
+    (below, above)
+}
+
+fn negate_ranges(ranges: &Vec<BlockRange>) -> Vec<BlockRange> {
+    use std::ops::Range;
+
+    let mut result = Vec::new();
+    let mut cursor = 0;
+
+    for r in ranges {
+        match r {
+            Range { start, end } if cursor < *start => {
+                result.push(cursor..*start);
+                cursor = *end;
+            }
+            Range { start: _, end } => {
+                cursor = *end;
+            }
+        }
+    }
+
+    result
+}
+
+fn range_len(r: &BlockRange) -> u64 {
+    r.end - r.start
+}
+
+fn ranges_total(rs: &Vec<BlockRange>) -> u64 {
+    rs.into_iter().fold(0, |sum, r| sum + range_len(r))
+}
+
+// Assumes there is enough space to remap.
+fn remap_ranges(ranges: Vec<BlockRange>, free: Vec<BlockRange>) -> Vec<(BlockRange, BlockRange)> {
+    use std::cmp::Ordering;
+
+    let mut remap = Vec::new();
+    let mut range_iter = ranges.into_iter();
+    let mut free_iter = free.into_iter();
+
+    let mut r_ = range_iter.next();
+    let mut f_ = free_iter.next();
+
+    while let (Some(r), Some(f)) = (r_, f_) {
+        let rlen = range_len(&r);
+        let flen = range_len(&f);
+
+        match rlen.cmp(&flen) {
+            Ordering::Less => {
+                // range fits into the free chunk
+                remap.push((r, f.start..(f.start + rlen)));
+                f_ = Some((f.start + rlen)..f.end);
+                r_ = range_iter.next();
+            },
+            Ordering::Equal => {
+                remap.push((r, f));
+                r_ = range_iter.next();
+                f_ = free_iter.next();
+            },
+            Ordering::Greater => {
+                remap.push((r.start..(r.start + flen), f));
+                r_ = Some((r.start + flen)..r.end);
+                f_ = free_iter.next();
+            }
+        }
+    }
+
+    remap
+}
+
+pub fn shrink(input_file: &str, _output_file: &str, nr_blocks: u64) -> Result<()> {
    let input = OpenOptions::new()
        .read(true)
        .write(false)
@ -14,8 +186,42 @@ pub fn shrink(input_file: &str, _output_file: &str) -> Result<()> {
        .open(input_file)?;

    // let mut visitor = xml::XmlWriter::new(std::io::stdout());
-    let mut visitor = xml::NoopVisitor::new();
-    xml::read(input, &mut visitor)?;
+    // let mut visitor = xml::NoopVisitor::new();
+    let mut pass1 = Pass1::new(nr_blocks);
+    xml::read(input, &mut pass1)?;
+    eprintln!("{} blocks need moving", pass1.nr_high_blocks);
+
+    let mut free_blocks = 0u64;
+    for i in 0..pass1.allocated_blocks.len() {
+        if !pass1.allocated_blocks[i] {
+            free_blocks += 1;
+        }
+    }
+    eprintln!("{} free blocks below new end.", free_blocks);
+
+    let ranges = bits_to_ranges(&pass1.allocated_blocks);
+    eprintln!("{} allocated ranges:", ranges.len());
+
+    eprintln!("{:?}", &ranges);
+
+    let (below, above) = ranges_split(&ranges, nr_blocks);
+    eprintln!("ranges split at {}: ({:?}, {:?})", nr_blocks, below, above);
+
+    let free = negate_ranges(&below);
+    eprintln!("free {:?}.", free);
+
+    let nr_moving = ranges_total(&above);
+    eprintln!("{} blocks need to be remapped.", nr_moving);
+
+    let free_blocks = ranges_total(&free);
+    eprintln!("{} free blocks.", free_blocks);
+
+    if free_blocks < nr_moving {
+        panic!("Insufficient space");
+    }
+
+    let remaps = remap_ranges(above, free);
+    eprintln!("remappings {:?}.", remaps);

    Ok(())
 }
--- a/src/shrink/xml.rs
+++ b/src/shrink/xml.rs
@ -14,29 +14,29 @@ use quick_xml::{Reader, Writer};
 //---------------------------------------

 pub struct Superblock {
-    uuid: String,
-    time: u64,
-    transaction: u64,
-    flags: Option<u32>,
-    version: Option<u32>,
-    data_block_size: u32,
-    nr_data_blocks: u64,
-    metadata_snap: Option<u64>,
+    pub uuid: String,
+    pub time: u64,
+    pub transaction: u64,
+    pub flags: Option<u32>,
+    pub version: Option<u32>,
+    pub data_block_size: u32,
+    pub nr_data_blocks: u64,
+    pub metadata_snap: Option<u64>,
 }

 pub struct Device {
-    dev_id: u32,
-    mapped_blocks: u64,
-    transaction: u64,
-    creation_time: u64,
-    snap_time: u64,
+    pub dev_id: u32,
+    pub mapped_blocks: u64,
+    pub transaction: u64,
+    pub creation_time: u64,
+    pub snap_time: u64,
 }

 pub struct Map {
-    thin_begin: u64,
-    data_begin: u64,
-    time: u32,
-    len: u64,
+    pub thin_begin: u64,
+    pub data_begin: u64,
+    pub time: u32,
+    pub len: u64,
 }

 pub trait MetadataVisitor {
@ -238,11 +238,11 @@ fn parse_superblock(e: &BytesStart) -> Result<Superblock> {
        uuid: check_attr(tag, "uuid", uuid)?,
        time: check_attr(tag, "time", time)?,
        transaction: check_attr(tag, "transaction", transaction)?,
-        flags: flags,
-        version: version,
+        flags,
+        version,
        data_block_size: check_attr(tag, "data_block_size", data_block_size)?,
        nr_data_blocks: check_attr(tag, "nr_data_blocks", nr_data_blocks)?,
-        metadata_snap: metadata_snap,
+        metadata_snap,
    })
 }