emo/thin-provisioning-tools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[thin_explore] First code drop

Browse Source
This commit is contained in:
Joe Thornber 2020-09-16 15:10:01 +01:00
parent 5168621f02
commit 8493cf7081
8 changed files with 1219 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
Cargo.lock generated
View File

@ -71,6 +71,12 @@ version = "1.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
[[package]]
name = "cassowary"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
[[package]]
name = "cc"
version = "1.0.59"
@ -426,6 +432,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
[[package]]
name = "numtoa"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef"
[[package]]
name = "once_cell"
version = "1.4.1"
@ -488,9 +500,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a"
[[package]]
name = "proc-macro2"
version = "1.0.19"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12"
checksum = "175c513d55719db99da20232b06cda8bab6b83ec2d04e3283edf0213c37c1a29"
dependencies = [
"unicode-xid",
]
@ -583,6 +595,15 @@ version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
[[package]]
name = "redox_termios"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
dependencies = [
"redox_syscall",
]
[[package]]
name = "regex"
version = "1.3.9"
@ -646,9 +667,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "syn"
version = "1.0.39"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9"
checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350"
dependencies = [
"proc-macro2",
"quote",
@ -679,6 +700,18 @@ dependencies = [
"winapi",
]
[[package]]
name = "termion"
version = "1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c22cec9d8978d906be5ac94bceb5a010d885c626c4c8855721a4dbd20e3ac905"
dependencies = [
"libc",
"numtoa",
"redox_syscall",
"redox_termios",
]
[[package]]
name = "termios"
version = "0.3.2"
@ -725,8 +758,10 @@ dependencies = [
"quickcheck_macros",
"rand",
"tempfile",
"termion",
"thiserror",
"threadpool",
"tui",
]
[[package]]
@ -767,6 +802,25 @@ dependencies = [
"num_cpus",
]
[[package]]
name = "tui"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a977b0bb2e2033a6fef950f218f13622c3c34e59754b704ce3492dedab1dfe95"
dependencies = [
"bitflags",
"cassowary",
"termion",
"unicode-segmentation",
"unicode-width",
]
[[package]]
name = "unicode-segmentation"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
[[package]]
name = "unicode-width"
version = "0.1.8"

2
Cargo.toml
View File

@ -29,6 +29,8 @@ rand = "0.7"
tempfile = "3.1"
threadpool = "1.8"
thiserror = "1.0"
tui = "0.10"
termion = "1.5"
[dev-dependencies]
json = "0.12"

555
src/bin/thin_explore.rs Normal file
View File

@ -0,0 +1,555 @@
extern crate clap;
use anyhow::{anyhow, Result};
use clap::{App, Arg};
use std::fmt;
use std::io::{self, Read, Write};
use std::path::Path;
use std::sync::mpsc;
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use std::thread;
use std::time::Duration;
use termion::clear;
use termion::color;
use termion::event::Key;
use termion::input::TermRead;
use termion::raw::IntoRawMode;
use tui::{
backend::{Backend, TermionBackend},
buffer::Buffer,
layout::{Constraint, Direction, Layout, Rect},
style::{Color, Modifier, Style},
terminal::Frame,
text::{Span, Spans},
widgets::{Block, Borders, List, ListItem, ListState, Row, StatefulWidget, Table, Widget},
Terminal,
};
use thinp::io_engine::*;
use thinp::pdata::btree;
use thinp::pdata::unpack::*;
use thinp::thin::block_time::*;
use thinp::thin::superblock::*;
//------------------------------------
pub enum Event<I> {
Input(I),
Tick,
}
pub struct Events {
rx: mpsc::Receiver<Event<Key>>,
input_handle: thread::JoinHandle<()>,
ignore_exit_key: Arc<AtomicBool>,
tick_handle: thread::JoinHandle<()>,
}
#[derive(Debug, Clone, Copy)]
pub struct Config {
pub exit_key: Key,
pub tick_rate: Duration,
}
impl Default for Config {
fn default() -> Config {
Config {
exit_key: Key::Char('q'),
tick_rate: Duration::from_millis(250),
}
}
}
impl Events {
pub fn new() -> Events {
Events::with_config(Config::default())
}
pub fn with_config(config: Config) -> Events {
let (tx, rx) = mpsc::channel();
let ignore_exit_key = Arc::new(AtomicBool::new(false));
let input_handle = {
let tx = tx.clone();
let ignore_exit_key = ignore_exit_key.clone();
thread::spawn(move || {
let stdin = io::stdin();
for evt in stdin.keys() {
if let Ok(key) = evt {
if let Err(err) = tx.send(Event::Input(key)) {
eprintln!("{}", err);
return;
}
if !ignore_exit_key.load(Ordering::Relaxed) && key == config.exit_key {
return;
}
}
}
})
};
let tick_handle = {
thread::spawn(move || loop {
if tx.send(Event::Tick).is_err() {
break;
}
thread::sleep(config.tick_rate);
})
};
Events {
rx,
ignore_exit_key,
input_handle,
tick_handle,
}
}
pub fn next(&self) -> Result<Event<Key>, mpsc::RecvError> {
self.rx.recv()
}
pub fn disable_exit_key(&mut self) {
self.ignore_exit_key.store(true, Ordering::Relaxed);
}
pub fn enable_exit_key(&mut self) {
self.ignore_exit_key.store(false, Ordering::Relaxed);
}
}
//------------------------------------
fn ls_next(ls: &mut ListState, max: usize) {
let i = match ls.selected() {
Some(i) => {
if i >= max - 1 {
max - 1
} else {
i + 1
}
}
None => 0,
};
ls.select(Some(i));
}
fn ls_previous(ls: &mut ListState, max: usize) {
let i = match ls.selected() {
Some(i) => {
if i == 0 {
0
} else {
i - 1
}
}
None => 0,
};
ls.select(Some(i));
}
//------------------------------------
struct SBWidget {
sb: Superblock,
}
impl Widget for SBWidget {
fn render(self, area: Rect, buf: &mut Buffer) {
let sb = &self.sb;
let flags = ["flags".to_string(), format!("{}", sb.flags)];
let block = ["block".to_string(), format!("{}", sb.block)];
let uuid = ["uuid".to_string(), format!("-")];
let version = ["version".to_string(), format!("{}", sb.version)];
let time = ["time".to_string(), format!("{}", sb.time)];
let transaction_id = [
"transaction_id".to_string(),
format!("{}", sb.transaction_id),
];
let metadata_snap = [
"metadata_snap".to_string(),
if sb.metadata_snap == 0 {
"-".to_string()
} else {
format!("{}", sb.metadata_snap)
},
];
let mapping_root = ["mapping root".to_string(), format!("{}", sb.mapping_root)];
let details_root = ["details root".to_string(), format!("{}", sb.details_root)];
let data_block_size = [
"data block size".to_string(),
format!("{}k", sb.data_block_size * 2),
];
let row_style = Style::default().fg(Color::White);
let table = Table::new(
["Field", "Value"].iter(),
vec![
Row::Data(flags.iter()),
Row::Data(block.iter()),
Row::Data(uuid.iter()),
Row::Data(version.iter()),
Row::Data(time.iter()),
Row::Data(transaction_id.iter()),
Row::Data(metadata_snap.iter()),
Row::Data(mapping_root.iter()),
Row::Data(details_root.iter()),
Row::Data(data_block_size.iter()),
]
.into_iter(),
)
.block(
Block::default()
.borders(Borders::ALL)
.title("Superblock".to_string()),
)
.header_style(Style::default().fg(Color::Yellow))
.widths(&[Constraint::Length(20), Constraint::Length(60)])
.style(Style::default().fg(Color::White))
.column_spacing(1);
Widget::render(table, area, buf);
}
}
//------------------------------------
struct HeaderWidget<'a> {
title: String,
hdr: &'a btree::NodeHeader,
}
impl<'a> Widget for HeaderWidget<'a> {
fn render(self, area: Rect, buf: &mut Buffer) {
let hdr = &self.hdr;
let kind = [
"type".to_string(),
match hdr.is_leaf {
true => "LEAF".to_string(),
false => "INTERNAL".to_string(),
},
];
let nr_entries = ["nr_entries".to_string(), format!("{}", hdr.nr_entries)];
let max_entries = ["max_entries".to_string(), format!("{}", hdr.max_entries)];
let value_size = ["value size".to_string(), format!("{}", hdr.value_size)];
let row_style = Style::default().fg(Color::White);
let table = Table::new(
["Field", "Value"].iter(),
vec![
Row::Data(kind.iter()),
Row::Data(nr_entries.iter()),
Row::Data(max_entries.iter()),
Row::Data(value_size.iter()),
]
.into_iter(),
)
.block(Block::default().borders(Borders::ALL).title(self.title))
.header_style(Style::default().fg(Color::Yellow))
.widths(&[Constraint::Length(20), Constraint::Length(60)])
.style(Style::default().fg(Color::White))
.column_spacing(1);
Widget::render(table, area, buf);
}
}
fn read_node_header(engine: &dyn IoEngine, loc: u64) -> Result<btree::NodeHeader> {
let b = engine.read(loc)?;
unpack(&b.get_data()).map_err(|_| anyhow!("couldn't unpack btree header"))
}
fn read_node<V: Unpack>(engine: &dyn IoEngine, loc: u64) -> Result<btree::Node<V>> {
let b = engine.read(loc)?;
btree::unpack_node(&b.get_data(), true, false)
.map_err(|_| anyhow!("couldn't unpack btree node"))
}
//------------------------------------
struct NodeWidget<'a, V: Unpack> {
title: String,
node: &'a btree::Node<V>,
}
impl<'a, V: Unpack + fmt::Display> StatefulWidget for NodeWidget<'a, V> {
type State = ListState;
fn render(self, area: Rect, buf: &mut Buffer, state: &mut ListState) {
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Min(10), Constraint::Percentage(80)].as_ref())
.split(area);
let hdr = HeaderWidget {
title: self.title,
hdr: self.node.get_header(),
};
hdr.render(chunks[0], buf);
let mut items: Vec<ListItem> = Vec::new();
match self.node {
btree::Node::Internal { keys, values, .. } => {
for (k, v) in keys.iter().zip(values.iter()) {
items.push(ListItem::new(Span::raw(format!("{} -> {}", k, v))));
}
}
btree::Node::Leaf { keys, values, .. } => {
for (k, v) in keys.iter().zip(values.iter()) {
items.push(ListItem::new(Span::raw(format!("{} -> {}", k, v))));
}
}
}
let items = List::new(items)
.block(Block::default().borders(Borders::ALL).title("Entries"))
.highlight_style(
Style::default()
.bg(Color::LightGreen)
.add_modifier(Modifier::BOLD),
);
StatefulWidget::render(items, chunks[1], buf, state);
}
}
//------------------------------------
enum Action {
PushTopLevel(u64),
PushBottomLevel(u32, u64),
PopPanel,
}
type Frame_<'a, 'b> = Frame<'a, TermionBackend<termion::raw::RawTerminal<std::io::StdoutLock<'b>>>>;
trait Panel {
fn render(&mut self, area: Rect, f: &mut Frame_);
fn input(&mut self, k: Key) -> Option<Action>;
}
struct SBPanel {
sb: Superblock,
}
impl Panel for SBPanel {
fn render(&mut self, area: Rect, f: &mut Frame_) {
// FIXME: get rid of clone
let w = SBWidget {
sb: self.sb.clone(),
};
f.render_widget(w, area);
}
fn input(&mut self, k: Key) -> Option<Action> {
None
}
}
struct TopLevelPanel {
node: btree::Node<u64>,
nr_entries: usize,
state: ListState,
}
impl TopLevelPanel {
fn new(node: btree::Node<u64>) -> TopLevelPanel {
let nr_entries = node.get_header().nr_entries as usize;
let mut state = ListState::default();
state.select(Some(0));
TopLevelPanel {
node,
nr_entries,
state,
}
}
}
impl Panel for TopLevelPanel {
fn render(&mut self, area: Rect, f: &mut Frame_) {
let w = NodeWidget {
title: "Top Level".to_string(),
node: &self.node, // FIXME: get rid of clone
};
f.render_stateful_widget(w, area, &mut self.state);
}
fn input(&mut self, k: Key) -> Option<Action> {
match k {
Key::Char('j') | Key::Down => {
ls_next(&mut self.state, self.nr_entries);
None
}
Key::Char('k') | Key::Up => {
ls_previous(&mut self.state, self.nr_entries);
None
}
Key::Char('l') | Key::Right => match &self.node {
btree::Node::Internal { values, .. } => {
Some(Action::PushTopLevel(values[self.state.selected().unwrap()]))
}
btree::Node::Leaf { values, keys, .. } => {
let index = self.state.selected().unwrap();
Some(Action::PushBottomLevel(keys[index] as u32, values[index]))
}
},
Key::Char('h') | Key::Left => Some(Action::PopPanel),
_ => None,
}
}
}
struct BottomLevelPanel {
thin_id: u32,
node: btree::Node<BlockTime>,
nr_entries: usize,
state: ListState,
}
impl BottomLevelPanel {
fn new(thin_id: u32, node: btree::Node<BlockTime>) -> BottomLevelPanel {
let nr_entries = node.get_header().nr_entries as usize;
let mut state = ListState::default();
state.select(Some(0));
BottomLevelPanel {
thin_id,
node,
nr_entries,
state,
}
}
}
impl Panel for BottomLevelPanel {
fn render(&mut self, area: Rect, f: &mut Frame_) {
let w = NodeWidget {
title: format!("Thin dev #{}", self.thin_id),
node: &self.node,
};
f.render_stateful_widget(w, area, &mut self.state);
}
fn input(&mut self, k: Key) -> Option<Action> {
match k {
Key::Char('j') | Key::Down => {
ls_next(&mut self.state, self.nr_entries);
None
}
Key::Char('k') | Key::Up => {
ls_previous(&mut self.state, self.nr_entries);
None
}
Key::Char('l') | Key::Right => match &self.node {
btree::Node::Internal { values, .. } => Some(Action::PushBottomLevel(
self.thin_id,
values[self.state.selected().unwrap()],
)),
_ => None,
},
Key::Char('h') | Key::Left => Some(Action::PopPanel),
_ => None,
}
}
}
//------------------------------------
fn explore(path: &Path) -> Result<()> {
let stdout = io::stdout();
let mut stdout = stdout.lock().into_raw_mode()?;
write!(stdout, "{}", termion::clear::All);
let backend = TermionBackend::new(stdout);
let mut terminal = Terminal::new(backend)?;
let path = std::path::Path::new("bz1763895/meta.bin");
let engine = SyncIoEngine::new(&path, 1, false)?;
let mut panels: Vec<Box<dyn Panel>> = Vec::new();
let sb = read_superblock(&engine, 0)?;
panels.push(Box::new(SBPanel { sb: sb.clone() }));
let node = read_node::<u64>(&engine, sb.mapping_root)?;
panels.push(Box::new(TopLevelPanel::new(node)));
let events = Events::new();
'main: loop {
let render_panels = |f: &mut Frame_| {
let chunks = Layout::default()
.direction(Direction::Horizontal)
.constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref())
.split(f.size());
let mut base = panels.len();
if base >= 2 {
base -= 2;
} else {
base = 0;
}
for i in base..panels.len() {
panels[i].render(chunks[i - base], f);
}
};
terminal.draw(render_panels)?;
let last = panels.len() - 1;
let active_panel = &mut panels[last];
if let Event::Input(key) = events.next()? {
match key {
Key::Char('q') => break 'main,
_ => match active_panel.input(key) {
Some(Action::PushTopLevel(b)) => {
let node = read_node::<u64>(&engine, b)?;
panels.push(Box::new(TopLevelPanel::new(node)));
}
Some(Action::PushBottomLevel(thin_id, b)) => {
let node = read_node::<BlockTime>(&engine, b)?;
panels.push(Box::new(BottomLevelPanel::new(thin_id, node)));
}
Some(Action::PopPanel) => {
panels.pop();
}
_ => {}
},
}
}
}
Ok(())
}
//------------------------------------
fn main() -> Result<()> {
let parser = App::new("thin_explore")
.version(thinp::version::TOOLS_VERSION)
.about("A text user interface for examining thin metadata.")
.arg(
Arg::with_name("INPUT")
.help("Specify the input device to check")
.required(true)
.index(1),
);
let matches = parser.get_matches();
let input_file = Path::new(matches.value_of("INPUT").unwrap());
explore(&input_file)
}
//------------------------------------

629
src/pdata/btree.rs
View File

@ -1,7 +1,8 @@
use anyhow::{anyhow, Result};
use nom::{number::complete::*, IResult};
use std::collections::BTreeMap;
use std::fmt;
use std::sync::{Arc, Mutex};
use thiserror::Error;
use threadpool::ThreadPool;
use crate::checksum;
@ -13,39 +14,346 @@ use crate::pdata::unpack::*;
//------------------------------------------
#[derive(Clone, Debug, PartialEq)]
pub struct KeyRange {
start: Option<u64>,
end: Option<u64>, // This is the one-past-the-end value
}
impl fmt::Display for KeyRange {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match (self.start, self.end) {
(None, None) => write!(f, "[..]"),
(None, Some(e)) => write!(f, "[..{}]", e),
(Some(s), None) => write!(f, "[{}..]", s),
(Some(s), Some(e)) => write!(f, "[{}..{}]", s, e),
}
}
}
impl KeyRange {
// None will be returned if either range would be zero length
fn split(&self, n: u64) -> Option<(KeyRange, KeyRange)> {
match (self.start, self.end) {
(None, None) => Some((
KeyRange {
start: None,
end: Some(n),
},
KeyRange {
start: Some(n),
end: None,
},
)),
(None, Some(e)) => {
if n < e {
Some((
KeyRange {
start: None,
end: Some(n),
},
KeyRange {
start: Some(n),
end: Some(e),
},
))
} else {
None
}
}
(Some(s), None) => {
if s < n {
Some((
KeyRange {
start: Some(s),
end: Some(n),
},
KeyRange {
start: Some(n),
end: None,
},
))
} else {
None
}
}
(Some(s), Some(e)) => {
if s < n && n < e {
Some((
KeyRange {
start: Some(s),
end: Some(n),
},
KeyRange {
start: Some(n),
end: Some(e),
},
))
} else {
None
}
}
}
}
}
#[test]
fn test_split_range() {
struct Test(Option<u64>, Option<u64>, u64, Option<(KeyRange, KeyRange)>);
let tests = vec![
Test(
None,
None,
100,
Some((
KeyRange {
start: None,
end: Some(100),
},
KeyRange {
start: Some(100),
end: None,
},
)),
),
Test(None, Some(100), 1000, None),
Test(
None,
Some(100),
50,
Some((
KeyRange {
start: None,
end: Some(50),
},
KeyRange {
start: Some(50),
end: Some(100),
},
)),
),
Test(None, Some(100), 100, None),
Test(Some(100), None, 50, None),
Test(
Some(100),
None,
150,
Some((
KeyRange {
start: Some(100),
end: Some(150),
},
KeyRange {
start: Some(150),
end: None,
},
)),
),
Test(Some(100), Some(200), 50, None),
Test(Some(100), Some(200), 250, None),
Test(
Some(100),
Some(200),
150,
Some((
KeyRange {
start: Some(100),
end: Some(150),
},
KeyRange {
start: Some(150),
end: Some(200),
},
)),
),
];
for Test(start, end, n, expected) in tests {
let kr = KeyRange { start, end };
let actual = kr.split(n);
assert_eq!(actual, expected);
}
}
fn split_one(kr: &KeyRange, k: u64) -> Result<(KeyRange, KeyRange)> {
match kr.split(k) {
None => {
return Err(node_err(&format!(
"couldn't split key range {} at {}",
kr, k
)));
}
Some(pair) => Ok(pair),
}
}
fn split_key_ranges_(kr: &KeyRange, keys: &[u64]) -> Result<Vec<KeyRange>> {
let mut krs = Vec::with_capacity(keys.len());
if keys.len() == 0 {
return Err(node_err("split_key_ranges: no keys present"));
}
// The first key gives the lower bound
let mut kr = KeyRange {
start: Some(keys[0]),
end: kr.end,
};
for i in 1..keys.len() {
let (first, rest) = split_one(&kr, keys[i])?;
krs.push(first);
kr = rest;
}
krs.push(kr);
Ok(krs)
}
fn split_key_ranges(kr: &KeyRange, keys: &[u64]) -> Result<Vec<KeyRange>> {
let msg = format!("split: {:?} at {:?}", &kr, &keys);
let r = split_key_ranges_(kr, keys);
if r.is_err() {
eprintln!("{} -> {:?}", msg, &r);
}
r
}
//------------------------------------------
const NODE_HEADER_SIZE: usize = 32;
#[derive(Error, Clone, Debug)]
pub enum BTreeError {
// #[error("io error")]
IoError, // (std::io::Error), // FIXME: we can't clone an io_error
// #[error("node error: {0}")]
NodeError(String),
// #[error("value error: {0}")]
ValueError(String),
// #[error("keys: {0:?}")]
KeyContext(KeyRange, Box<BTreeError>),
// #[error("aggregate: {0:?}")]
Aggregate(Vec<BTreeError>),
// #[error("{0:?}, {1}")]
Path(u64, Box<BTreeError>),
}
fn extract_path(e: &BTreeError) -> (Vec<u64>, BTreeError) {
let mut path = Vec::new();
let mut e = e;
loop {
match e {
BTreeError::Path(b, next) => {
path.push(*b);
e = next.as_ref();
}
_ => return (path, e.clone()),
}
}
}
impl fmt::Display for BTreeError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let (path, e) = extract_path(self);
match e {
BTreeError::IoError => write!(f, "io error, path{:?}", path),
BTreeError::NodeError(msg) => write!(f, "node error: {}, path{:?}", msg, path),
BTreeError::ValueError(msg) => write!(f, "value error: {}, path{:?}", msg, path),
BTreeError::KeyContext(kr, be) => {
write!(f, "{}, effecting keys {}, path{:?}", be, kr, path)
}
BTreeError::Aggregate(errs) => {
for e in errs {
write!(f, "{}", e)?
}
Ok(())
}
// Can't happen
BTreeError::Path(_, e) => write!(f, "{}", e),
}
}
}
pub fn node_err(msg: &str) -> BTreeError {
BTreeError::NodeError(msg.to_string())
}
fn node_err_s(msg: String) -> BTreeError {
BTreeError::NodeError(msg)
}
pub fn io_err() -> BTreeError {
BTreeError::IoError
}
pub fn value_err(msg: String) -> BTreeError {
BTreeError::ValueError(msg)
}
fn aggregate_error(rs: Vec<BTreeError>) -> BTreeError {
BTreeError::Aggregate(rs)
}
impl BTreeError {
pub fn keys_context(self, keys: &KeyRange) -> BTreeError {
BTreeError::KeyContext(keys.clone(), Box::new(self))
}
}
pub type Result<T> = std::result::Result<T, BTreeError>;
//------------------------------------------
#[derive(Debug, Clone)]
pub struct NodeHeader {
is_leaf: bool,
nr_entries: u32,
max_entries: u32,
value_size: u32,
pub block: u64,
pub is_leaf: bool,
pub nr_entries: u32,
pub max_entries: u32,
pub value_size: u32,
}
#[allow(dead_code)]
const INTERNAL_NODE: u32 = 1;
const LEAF_NODE: u32 = 2;
pub fn unpack_node_header(data: &[u8]) -> IResult<&[u8], NodeHeader> {
let (i, _csum) = le_u32(data)?;
let (i, flags) = le_u32(i)?;
let (i, _block) = le_u64(i)?;
let (i, nr_entries) = le_u32(i)?;
let (i, max_entries) = le_u32(i)?;
let (i, value_size) = le_u32(i)?;
let (i, _padding) = le_u32(i)?;
impl Unpack for NodeHeader {
fn disk_size() -> u32 {
32
}
Ok((
i,
NodeHeader {
is_leaf: flags == LEAF_NODE,
nr_entries,
max_entries,
value_size,
},
))
fn unpack(data: &[u8]) -> IResult<&[u8], NodeHeader> {
let (i, _csum) = le_u32(data)?;
let (i, flags) = le_u32(i)?;
let (i, block) = le_u64(i)?;
let (i, nr_entries) = le_u32(i)?;
let (i, max_entries) = le_u32(i)?;
let (i, value_size) = le_u32(i)?;
let (i, _padding) = le_u32(i)?;
Ok((
i,
NodeHeader {
block,
is_leaf: flags == LEAF_NODE,
nr_entries,
max_entries,
value_size,
},
))
}
}
#[derive(Clone)]
pub enum Node<V: Unpack> {
Internal {
header: NodeHeader,
@ -59,17 +367,22 @@ pub enum Node<V: Unpack> {
},
}
pub fn node_err<V>(msg: String) -> Result<V> {
let msg = format!("btree node error: {}", msg);
Err(anyhow!(msg))
impl<V: Unpack> Node<V> {
pub fn get_header(&self) -> &NodeHeader {
use Node::*;
match self {
Internal { header, .. } => header,
Leaf { header, .. } => header,
}
}
}
pub fn to_any<'a, V>(r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> {
if let Ok((i, v)) = r {
Ok((i, v))
} else {
Err(anyhow!("btree node error: parse error"))
}
pub fn convert_result<'a, V>(r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> {
r.map_err(|_e| node_err("parse error"))
}
pub fn convert_io_err<V>(r: std::io::Result<V>) -> Result<V> {
r.map_err(|_| io_err())
}
pub fn unpack_node<V: Unpack>(
@ -79,45 +392,52 @@ pub fn unpack_node<V: Unpack>(
) -> Result<Node<V>> {
use nom::multi::count;
let (i, header) = to_any(unpack_node_header(data))?;
let (i, header) =
NodeHeader::unpack(data).map_err(|_e| node_err("couldn't parse node header"))?;
if header.is_leaf && header.value_size != V::disk_size() {
return node_err(format!(
return Err(node_err_s(format!(
"value_size mismatch: expected {}, was {}",
V::disk_size(),
header.value_size
));
)));
}
let elt_size = header.value_size + 8;
if elt_size as usize * header.max_entries as usize + NODE_HEADER_SIZE > BLOCK_SIZE {
return node_err(format!("max_entries is too large ({})", header.max_entries));
return Err(node_err_s(format!(
"max_entries is too large ({})",
header.max_entries
)));
}
if header.nr_entries > header.max_entries {
return node_err("nr_entries > max_entries".to_string());
return Err(node_err("nr_entries > max_entries"));
}
if !ignore_non_fatal {
if header.max_entries % 3 != 0 {
return node_err("max_entries is not divisible by 3".to_string());
return Err(node_err("max_entries is not divisible by 3"));
}
if !is_root {
let min = header.max_entries / 3;
if header.nr_entries < min {
return node_err("too few entries".to_string());
return Err(node_err_s(format!(
"too few entries {}, expected at least {}",
header.nr_entries, min
)));
}
}
}
let (i, keys) = to_any(count(le_u64, header.nr_entries as usize)(i))?;
let (i, keys) = convert_result(count(le_u64, header.nr_entries as usize)(i))?;
let mut last = None;
for k in &keys {
if let Some(l) = last {
if k <= l {
return node_err("keys out of order".to_string());
return Err(node_err("keys out of order"));
}
}
@ -125,10 +445,10 @@ pub fn unpack_node<V: Unpack>(
}
let nr_free = header.max_entries - header.nr_entries;
let (i, _padding) = to_any(count(le_u64, nr_free as usize)(i))?;
let (i, _padding) = convert_result(count(le_u64, nr_free as usize)(i))?;
if header.is_leaf {
let (_i, values) = to_any(count(V::unpack, header.nr_entries as usize)(i))?;
let (_i, values) = convert_result(count(V::unpack, header.nr_entries as usize)(i))?;
Ok(Node::Leaf {
header,
@ -136,7 +456,7 @@ pub fn unpack_node<V: Unpack>(
values,
})
} else {
let (_i, values) = to_any(count(le_u64, header.nr_entries as usize)(i))?;
let (_i, values) = convert_result(count(le_u64, header.nr_entries as usize)(i))?;
Ok(Node::Internal {
header,
keys,
@ -149,13 +469,15 @@ pub fn unpack_node<V: Unpack>(
pub trait NodeVisitor<V: Unpack> {
// &self is deliberately non mut to allow the walker to use multiple threads.
fn visit(&self, header: &NodeHeader, keys: &[u64], values: &[V]) -> Result<()>;
fn visit(&self, keys: &KeyRange, header: &NodeHeader, keys: &[u64], values: &[V])
-> Result<()>;
}
#[derive(Clone)]
pub struct BTreeWalker {
engine: Arc<dyn IoEngine + Send + Sync>,
sm: Arc<Mutex<dyn SpaceMap + Send + Sync>>,
fails: Arc<Mutex<BTreeMap<u64, BTreeError>>>,
ignore_non_fatal: bool,
}
@ -165,6 +487,7 @@ impl BTreeWalker {
let r: BTreeWalker = BTreeWalker {
engine,
sm: Arc::new(Mutex::new(RestrictedSpaceMap::new(nr_blocks as u64))),
fails: Arc::new(Mutex::new(BTreeMap::new())),
ignore_non_fatal,
};
r
@ -177,53 +500,116 @@ impl BTreeWalker {
) -> Result<BTreeWalker> {
{
let sm = sm.lock().unwrap();
assert_eq!(sm.get_nr_blocks()?, engine.get_nr_blocks());
assert_eq!(sm.get_nr_blocks().unwrap(), engine.get_nr_blocks());
}
Ok(BTreeWalker {
engine,
sm,
fails: Arc::new(Mutex::new(BTreeMap::new())),
ignore_non_fatal,
})
}
// Atomically increments the ref count, and returns the _old_ count.
fn sm_inc(&self, b: u64) -> Result<u32> {
let mut sm = self.sm.lock().unwrap();
let count = sm.get(b)?;
sm.inc(b, 1)?;
Ok(count)
fn failed(&self, b: u64) -> Option<BTreeError> {
let fails = self.fails.lock().unwrap();
match fails.get(&b) {
None => None,
Some(e) => Some(e.clone()),
}
}
fn walk_nodes<NV, V>(&self, visitor: &NV, bs: &[u64]) -> Result<()>
fn set_fail(&self, b: u64, err: BTreeError) {
// FIXME: should we monitor the size of fails, and abort if too many errors?
let mut fails = self.fails.lock().unwrap();
fails.insert(b, err);
}
// Atomically increments the ref count, and returns the _old_ count.
fn sm_inc(&self, b: u64) -> u32 {
let mut sm = self.sm.lock().unwrap();
let count = sm.get(b).unwrap();
sm.inc(b, 1).unwrap();
count
}
fn build_aggregate(&self, b: u64, errs: Vec<BTreeError>) -> Result<()> {
match errs.len() {
0 => Ok(()),
1 => {
let e = errs[0].clone();
self.set_fail(b, e.clone());
Err(e)
}
_ => {
let e = aggregate_error(errs);
self.set_fail(b, e.clone());
Err(e)
}
}
}
fn walk_nodes<NV, V>(&self, visitor: &NV, kr: &[KeyRange], bs: &[u64]) -> Vec<BTreeError>
where
NV: NodeVisitor<V>,
V: Unpack,
{
let mut errs: Vec<BTreeError> = Vec::new();
let mut blocks = Vec::with_capacity(bs.len());
for b in bs {
if self.sm_inc(*b)? == 0 {
if self.sm_inc(*b) == 0 {
// Node not yet seen
blocks.push(*b);
} else {
// This node has already been checked ...
match self.failed(*b) {
None => {
// ... it was clean so we can ignore.
}
Some(e) => {
// ... there was an error
errs.push(e.clone());
}
}
}
}
let blocks = self.engine.read_many(&blocks[0..])?;
match self.engine.read_many(&blocks[0..]) {
Err(_) => {
// IO completely failed, error every block
for (i, b) in blocks.iter().enumerate() {
let e = io_err().keys_context(&kr[i]);
errs.push(e.clone());
self.set_fail(*b, e);
}
}
Ok(rblocks) => {
let mut i = 0;
for rb in rblocks {
match rb {
Err(_) => {
let e = io_err().keys_context(&kr[i]);
errs.push(e.clone());
self.set_fail(blocks[i], e);
}
Ok(b) => match self.walk_node(visitor, &kr[i], &b, false) {
Err(e) => {
errs.push(e);
}
Ok(()) => {}
},
}
for b in blocks {
match b {
Err(_e) => {
todo!();
},
Ok(b) => {
self.walk_node(visitor, &b, false)?;
},
i += 1;
}
}
}
Ok(())
errs
}
fn walk_node<NV, V>(&self, visitor: &NV, b: &Block, is_root: bool) -> Result<()>
fn walk_node_<NV, V>(&self, visitor: &NV, kr: &KeyRange, b: &Block, is_root: bool) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
@ -232,51 +618,78 @@ impl BTreeWalker {
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::NODE {
return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt));
return Err(
node_err_s(format!("checksum failed for node {}, {:?}", b.loc, bt))
.keys_context(kr),
);
}
let node = unpack_node::<V>(&b.get_data(), self.ignore_non_fatal, is_root)?;
match node {
Internal {
header: _h,
keys: _k,
values,
} => {
self.walk_nodes(visitor, &values)?;
Internal { keys, values, .. } => {
let krs = split_key_ranges(&kr, &keys)?;
let errs = self.walk_nodes(visitor, &krs, &values);
return self.build_aggregate(b.loc, errs);
}
Leaf {
header,
keys,
values,
} => {
visitor.visit(&header, &keys, &values)?;
if let Err(e) = visitor.visit(&kr, &header, &keys, &values) {
self.set_fail(b.loc, e.clone());
return Err(e);
}
}
}
Ok(())
}
fn walk_node<NV, V>(&self, visitor: &NV, kr: &KeyRange, b: &Block, is_root: bool) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
{
let r = self.walk_node_(visitor, kr, b, is_root);
let r = match r {
Err(e) => Err(BTreeError::Path(b.loc, Box::new(e))),
Ok(v) => Ok(v),
};
r
}
pub fn walk<NV, V>(&self, visitor: &NV, root: u64) -> Result<()>
where
NV: NodeVisitor<V>,
V: Unpack,
{
if self.sm_inc(root)? > 0 {
Ok(())
if self.sm_inc(root) > 0 {
if let Some(e) = self.failed(root) {
Err(e.clone())
} else {
Ok(())
}
} else {
let root = self.engine.read(root)?;
self.walk_node(visitor, &root, true)
let root = self.engine.read(root).map_err(|_| io_err())?;
let kr = KeyRange {
start: None,
end: None,
};
self.walk_node(visitor, &kr, &root, true)
}
}
}
//--------------------------------
/*
fn walk_node_threaded<NV, V>(
w: Arc<BTreeWalker>,
pool: &ThreadPool,
visitor: Arc<NV>,
kr: &KeyRange,
b: &Block,
is_root: bool,
) -> Result<()>
@ -288,25 +701,25 @@ where
let bt = checksum::metadata_block_type(b.get_data());
if bt != checksum::BT::NODE {
return Err(anyhow!("checksum failed for node {}, {:?}", b.loc, bt));
return Err(node_err_s(format!(
"checksum failed for node {}, {:?}",
b.loc, bt
)));
}
let node = unpack_node::<V>(&b.get_data(), w.ignore_non_fatal, is_root)?;
match node {
Internal {
header: _h,
keys: _k,
values,
} => {
walk_nodes_threaded(w, pool, visitor, &values)?;
Internal { keys, values, .. } => {
let krs = BTreeWalker::split_key_ranges(&kr, &keys)?;
walk_nodes_threaded(w, pool, visitor, &krs, &values)?;
}
Leaf {
header,
keys,
values,
} => {
visitor.visit(&header, &keys, &values)?;
visitor.visit(kr, &header, &keys, &values)?;
}
}
@ -317,6 +730,7 @@ fn walk_nodes_threaded<NV, V>(
w: Arc<BTreeWalker>,
pool: &ThreadPool,
visitor: Arc<NV>,
krs: &[KeyRange],
bs: &[u64],
) -> Result<()>
where
@ -330,22 +744,30 @@ where
}
}
let blocks = w.engine.read_many(&blocks[0..])?;
let rblocks = convert_io_err(w.engine.read_many(&blocks[0..]))?;
for b in blocks {
let mut i = 0;
for b in rblocks {
match b {
Err(_e) => {
todo!();
},
Err(_) => {
// FIXME: aggregate these
return Err(io_err());
}
Ok(b) => {
let w = w.clone();
let visitor = visitor.clone();
let kr = krs[i].clone();
pool.execute(move || {
// FIXME: return result
w.walk_node(visitor.as_ref(), &b, false);
let result = w.walk_node(visitor.as_ref(), &kr, &b, false);
if result.is_err() {
todo!();
}
});
}
}
i += 1;
}
pool.join();
@ -365,10 +787,28 @@ where
if w.sm_inc(root)? > 0 {
Ok(())
} else {
let root = w.engine.read(root)?;
walk_node_threaded(w, pool, visitor, &root, true)
let root = convert_io_err(w.engine.read(root))?;
let kr = KeyRange {
start: None,
end: None,
};
walk_node_threaded(w, pool, visitor, &kr, &root, true)
}
}
*/
pub fn walk_threaded<NV, V>(
w: Arc<BTreeWalker>,
pool: &ThreadPool,
visitor: Arc<NV>,
root: u64,
) -> Result<()>
where
NV: NodeVisitor<V> + Send + Sync + 'static,
V: Unpack,
{
w.walk(visitor.as_ref(), root)
}
//------------------------------------------
@ -385,7 +825,7 @@ impl<V> ValueCollector<V> {
}
impl<V: Unpack + Clone> NodeVisitor<V> for ValueCollector<V> {
fn visit(&self, _h: &NodeHeader, keys: &[u64], values: &[V]) -> Result<()> {
fn visit(&self, _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], values: &[V]) -> Result<()> {
let mut vals = self.values.lock().unwrap();
for n in 0..keys.len() {
vals.insert(keys[n], values[n].clone());
@ -402,7 +842,6 @@ pub fn btree_to_map<V: Unpack + Clone>(
) -> Result<BTreeMap<u64, V>> {
let walker = BTreeWalker::new(engine, ignore_non_fatal);
let visitor = ValueCollector::<V>::new();
walker.walk(&visitor, root)?;
Ok(visitor.values.into_inner().unwrap())
}

40
src/thin/block_time.rs Normal file
View File

@ -0,0 +1,40 @@
use nom::{number::complete::*, IResult};
use std::fmt;
use crate::pdata::unpack::*;
//------------------------------------------
#[derive(Clone)]
pub struct BlockTime {
pub block: u64,
pub time: u32,
}
impl Unpack for BlockTime {
fn disk_size() -> u32 {
8
}
fn unpack(i: &[u8]) -> IResult<&[u8], BlockTime> {
let (i, n) = le_u64(i)?;
let block = n >> 24;
let time = n & ((1 << 24) - 1);
Ok((
i,
BlockTime {
block,
time: time as u32,
},
))
}
}
impl fmt::Display for BlockTime {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} @ {}", self.block, self.time)
}
}
//------------------------------------------

24
src/thin/check.rs
View File

@ -9,7 +9,7 @@ use threadpool::ThreadPool;
use crate::checksum;
use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine};
use crate::pdata::btree::*;
use crate::pdata::btree::{self, *};
use crate::pdata::space_map::*;
use crate::pdata::unpack::*;
use crate::report::*;
@ -50,7 +50,7 @@ struct BottomLevelVisitor {
//------------------------------------------
impl NodeVisitor<BlockTime> for BottomLevelVisitor {
fn visit(&self, _h: &NodeHeader, _k: &[u64], values: &[BlockTime]) -> Result<()> {
fn visit(&self, _kr: &KeyRange, _h: &NodeHeader, _k: &[u64], values: &[BlockTime]) -> btree::Result<()> {
// FIXME: do other checks
if values.len() == 0 {
@ -67,13 +67,13 @@ impl NodeVisitor<BlockTime> for BottomLevelVisitor {
if block == start + len {
len += 1;
} else {
data_sm.inc(start, len)?;
data_sm.inc(start, len).unwrap();
start = block;
len = 1;
}
}
data_sm.inc(start, len)?;
data_sm.inc(start, len).unwrap();
Ok(())
}
}
@ -124,14 +124,14 @@ impl<'a> OverflowChecker<'a> {
}
impl<'a> NodeVisitor<u32> for OverflowChecker<'a> {
fn visit(&self, _h: &NodeHeader, keys: &[u64], values: &[u32]) -> Result<()> {
fn visit(&self, _kr: &KeyRange, _h: &NodeHeader, keys: &[u64], values: &[u32]) -> btree::Result<()> {
for n in 0..keys.len() {
let k = keys[n];
let v = values[n];
let expected = self.data_sm.get(k)?;
let expected = self.data_sm.get(k).unwrap();
if expected != v {
return Err(anyhow!("Bad reference count for data block {}. Expected {}, but space map contains {}.",
k, expected, v));
return Err(value_err(format!("Bad reference count for data block {}. Expected {}, but space map contains {}.",
k, expected, v)));
}
}
@ -382,7 +382,7 @@ fn check_mapping_bottom_level(
false,
)?);
if roots.len() > 64 {
if roots.len() > 64000 {
ctx.report.info("spreading load across devices");
for (_thin_id, root) in roots {
let data_sm = data_sm.clone();
@ -390,6 +390,7 @@ fn check_mapping_bottom_level(
let v = BottomLevelVisitor { data_sm };
let w = w.clone();
ctx.pool.execute(move || {
// FIXME: propogate errors + share fails.
let _r = w.walk(&v, root);
});
}
@ -401,6 +402,7 @@ fn check_mapping_bottom_level(
let data_sm = data_sm.clone();
let root = *root;
let v = Arc::new(BottomLevelVisitor { data_sm });
// FIXME: propogate errors + share fails.
walk_threaded(w, &ctx.pool, v, root)?
}
}
@ -480,15 +482,16 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
)?;
// mapping top level
report.set_sub_title("mapping tree");
let roots =
btree_to_map_with_sm::<u64>(engine.clone(), metadata_sm.clone(), false, sb.mapping_root)?;
// mapping bottom level
report.set_sub_title("mapping tree");
let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
let data_sm = core_sm(root.nr_blocks, nr_devs as u32);
check_mapping_bottom_level(&ctx, &metadata_sm, &data_sm, &roots)?;
bail_out(&ctx, "mapping tree")?;
eprintln!("checked mapping");
report.set_sub_title("data space map");
let root = unpack::<SMRoot>(&sb.data_sm_root[0..])?;
@ -556,7 +559,6 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
let mut stop_progress = stop_progress.lock().unwrap();
*stop_progress = true;
}
tid.join();
Ok(())

1
src/thin/mod.rs
View File

@ -1,3 +1,4 @@
pub mod block_time;
pub mod superblock;
pub mod check;
pub mod xml;

20
src/thin/superblock.rs
View File

@ -1,17 +1,27 @@
use crate::io_engine::*;
use anyhow::{anyhow, Result};
use nom::{bytes::complete::*, number::complete::*, IResult};
use std::fmt;
pub const SUPERBLOCK_LOCATION: u64 = 0;
//const UUID_SIZE: usize = 16;
const SPACE_MAP_ROOT_SIZE: usize = 128;
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct SuperblockFlags {
pub needs_check: bool,
}
#[derive(Debug)]
impl fmt::Display for SuperblockFlags {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.needs_check {
write!(f, "NEEDS_CHECK")
} else {
write!(f, "-")
}
}
}
#[derive(Debug, Clone)]
pub struct Superblock {
pub flags: SuperblockFlags,
pub block: u64,
@ -76,7 +86,9 @@ fn unpack(data: &[u8]) -> IResult<&[u8], Superblock> {
Ok((
i,
Superblock {
flags: SuperblockFlags {needs_check: (flags & 0x1) != 0},
flags: SuperblockFlags {
needs_check: (flags & 0x1) != 0,
},
block,
//uuid: uuid[0..UUID_SIZE],
version,