[cache_check (rust)] Detect structural errors on arrays

- Define error types for arrays
- Propagate low-level errors to tools
- Rename array_block.rs to array.rs

src/cache/check.rs

@@ -8,6 +8,7 @@ use crate::cache::hint::*;
 use crate::cache::mapping::*;
 use crate::cache::superblock::*;
 use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine};
+use crate::pdata::array;
 use crate::pdata::array_walker::*;
 use crate::pdata::bitset_walker::*;
 
@@ -33,30 +34,30 @@ mod format1 {
             }
         }
 
-        fn check_flags(&self, m: &Mapping) -> anyhow::Result<()> {
+        fn check_flags(&self, m: &Mapping) -> array::Result<()> {
             if (m.flags & !(MappingFlags::Valid as u32 | MappingFlags::Dirty as u32)) != 0 {
-                return Err(anyhow!("unknown flags in mapping: {}", m.flags));
+                return Err(array::value_err(format!("unknown flags in mapping: {}", m.flags)));
             }
             if !m.is_valid() && m.is_dirty() {
-                return Err(anyhow!("dirty bit found on an unmapped block"));
+                return Err(array::value_err(format!("dirty bit found on an unmapped block")));
             }
             Ok(())
         }
 
-        fn check_oblock(&self, m: &Mapping) -> anyhow::Result<()> {
+        fn check_oblock(&self, m: &Mapping) -> array::Result<()> {
             if !m.is_valid() {
                 if m.oblock > 0 {
-                    return Err(anyhow!("invalid mapped block"));
+                    return Err(array::value_err(format!("invalid block is mapped")));
                 }
                 return Ok(());
             }
             if m.oblock >= self.nr_origin_blocks {
-                return Err(anyhow!("mapping beyond end of the origin device"));
+                return Err(array::value_err(format!("mapping beyond end of the origin device")));
             }
 
             let mut seen_oblocks = self.seen_oblocks.lock().unwrap();
             if seen_oblocks.contains(&m.oblock) {
-                return Err(anyhow!("origin block already mapped"));
+                return Err(array::value_err(format!("origin block already mapped")));
             }
             seen_oblocks.insert(m.oblock);
 
@@ -65,7 +66,7 @@ mod format1 {
     }
 
     impl ArrayVisitor<Mapping> for MappingChecker {
-        fn visit(&self, _index: u64, m: Mapping) -> anyhow::Result<()> {
+        fn visit(&self, _index: u64, m: Mapping) -> array::Result<()> {
             self.check_flags(&m)?;
             self.check_oblock(&m)?;
 
@@ -98,28 +99,28 @@ mod format2 {
             }
         }
 
-        fn check_flags(&self, m: &Mapping, dirty_bit: bool) -> anyhow::Result<()> {
+        fn check_flags(&self, m: &Mapping, dirty_bit: bool) -> array::Result<()> {
             if (m.flags & !(MappingFlags::Valid as u32)) != 0 {
-                return Err(anyhow!("unknown flags in mapping: {}", m.flags));
+                return Err(array::value_err(format!("unknown flags in mapping: {}", m.flags)));
             }
             if !m.is_valid() && dirty_bit {
-                return Err(anyhow!("dirty bit found on an unmapped block"));
+                return Err(array::value_err(format!("dirty bit found on an unmapped block")));
             }
             Ok(())
         }
 
-        fn check_oblock(&self, m: &Mapping, seen_oblocks: &mut BTreeSet<u64>) -> anyhow::Result<()> {
+        fn check_oblock(&self, m: &Mapping, seen_oblocks: &mut BTreeSet<u64>) -> array::Result<()> {
             if !m.is_valid() {
                 if m.oblock > 0 {
-                    return Err(anyhow!("invalid mapped block"));
+                    return Err(array::value_err(format!("invalid mapped block")));
                 }
                 return Ok(());
             }
             if m.oblock >= self.nr_origin_blocks {
-                return Err(anyhow!("mapping beyond end of the origin device"));
+                return Err(array::value_err(format!("mapping beyond end of the origin device")));
             }
             if seen_oblocks.contains(&m.oblock) {
-                return Err(anyhow!("origin block already mapped"));
+                return Err(array::value_err(format!("origin block already mapped")));
             }
             seen_oblocks.insert(m.oblock);
 
@@ -128,7 +129,7 @@ mod format2 {
     }
 
     impl ArrayVisitor<Mapping> for MappingChecker {
-        fn visit(&self, index: u64, m: Mapping) -> anyhow::Result<()> {
+        fn visit(&self, index: u64, m: Mapping) -> array::Result<()> {
             let mut inner = self.inner.lock().unwrap();
             self.check_flags(&m, inner.dirty_bits.contains(index as usize))?;
             self.check_oblock(&m, &mut inner.seen_oblocks)?;
@@ -149,7 +150,7 @@ impl HintChecker {
 }
 
 impl ArrayVisitor<Hint> for HintChecker {
-    fn visit(&self, _index: u64, _hint: Hint) -> anyhow::Result<()> {
+    fn visit(&self, _index: u64, _hint: Hint) -> array::Result<()> {
         // TODO: check hints
         Ok(())
     }

src/cache/dump.rs

@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Result};
+use anyhow::anyhow;
 use fixedbitset::FixedBitSet;
 use std::path::Path;
 use std::sync::{Arc, Mutex};
@@ -8,6 +8,7 @@ use crate::cache::mapping::Mapping;
 use crate::cache::superblock::*;
 use crate::cache::xml::{self, MetadataVisitor};
 use crate::io_engine::{AsyncIoEngine, IoEngine, SyncIoEngine};
+use crate::pdata::array;
 use crate::pdata::array_walker::*;
 
 //------------------------------------------
@@ -45,7 +46,7 @@ mod format1 {
     }
 
     impl<'a> ArrayVisitor<Mapping> for MappingEmitter<'a> {
-        fn visit(&self, index: u64, m: Mapping) -> Result<()> {
+        fn visit(&self, index: u64, m: Mapping) -> array::Result<()> {
             if m.is_valid() {
                 let m = xml::Map {
                     cblock: index as u32,
@@ -55,7 +56,7 @@ mod format1 {
 
                 let mut inner = self.inner.lock().unwrap();
                 inner.valid_mappings.set(index as usize, true);
-                inner.visitor.mapping(&m)?;
+                inner.visitor.mapping(&m).map_err(|e| array::value_err(format!("{}", e)))?;
             }
 
             Ok(())
@@ -89,7 +90,7 @@ mod format2 {
     }
 
     impl ArrayVisitor<u64> for DirtyVisitor {
-        fn visit(&self, index: u64, bits: u64) -> Result<()> {
+        fn visit(&self, index: u64, bits: u64) -> array::Result<()> {
             for i in 0..64u64 {
                 if (index + i) >= self.nr_entries as u64 {
                     break;
@@ -132,7 +133,7 @@ mod format2 {
     }
 
     impl<'a> ArrayVisitor<Mapping> for MappingEmitter<'a> {
-        fn visit(&self, index: u64, m: Mapping) -> Result<()> {
+        fn visit(&self, index: u64, m: Mapping) -> array::Result<()> {
             if m.is_valid() {
                 let mut inner = self.inner.lock().unwrap();
                 let dirty = inner.dirty_bits.contains(index as usize);
@@ -143,7 +144,7 @@ mod format2 {
                 };
 
                 inner.valid_mappings.set(index as usize, true);
-                inner.visitor.mapping(&m)?;
+                inner.visitor.mapping(&m).map_err(|e| array::value_err(format!("{}", e)))?;
             }
 
             Ok(())
@@ -168,14 +169,18 @@ impl<'a> HintEmitter<'a> {
 }
 
 impl<'a> ArrayVisitor<Hint> for HintEmitter<'a> {
-    fn visit(&self, index: u64, hint: Hint) -> anyhow::Result<()> {
+    fn visit(&self, index: u64, hint: Hint) -> array::Result<()> {
         if self.valid_mappings.contains(index as usize) {
             let h = xml::Hint {
                 cblock: index as u32,
                 data: hint.hint.to_vec(),
             };
 
-            self.emitter.lock().unwrap().hint(&h)?;
+            self.emitter
+                .lock()
+                .unwrap()
+                .hint(&h)
+                .map_err(|e| array::value_err(format!("{}", e)))?;
         }
 
         Ok(())

src/checksum.rs

@@ -11,6 +11,7 @@ const SUPERBLOCK_CSUM_XOR: u32 = 160774;
 const BITMAP_CSUM_XOR: u32 = 240779;
 const INDEX_CSUM_XOR: u32 = 160478;
 const BTREE_CSUM_XOR: u32 = 121107;
+const ARRAY_CSUM_XOR: u32 = 595846735;
 
 fn checksum(buf: &[u8]) -> u32 {
     crc32c(&buf[4..]) ^ 0xffffffff
@@ -22,6 +23,7 @@ pub enum BT {
     NODE,
     INDEX,
     BITMAP,
+    ARRAY,
     UNKNOWN,
 }
 
@@ -41,6 +43,7 @@ pub fn metadata_block_type(buf: &[u8]) -> BT {
         BTREE_CSUM_XOR => BT::NODE,
         BITMAP_CSUM_XOR => BT::BITMAP,
         INDEX_CSUM_XOR => BT::INDEX,
+        ARRAY_CSUM_XOR => BT::ARRAY,
         _ => BT::UNKNOWN,
     }
 }
@@ -56,6 +59,7 @@ pub fn write_checksum(buf: &mut [u8], kind: BT) -> Result<()> {
         NODE => BTREE_CSUM_XOR,
         BITMAP => BITMAP_CSUM_XOR,
         INDEX => INDEX_CSUM_XOR,
+        ARRAY => ARRAY_CSUM_XOR,
         UNKNOWN => {return Err(anyhow!("Invalid block type"));}
     };
     

src/pack/node_encode.rs

@@ -114,4 +114,8 @@ pub fn pack_index<W: Write>(w: &mut W, bytes: &[u8]) -> PResult<()> {
     io_to_pr(pack_literal(w, bytes))
 }
 
+pub fn pack_array<W: Write>(w: &mut W, bytes: &[u8]) -> PResult<()> {
+    io_to_pr(pack_literal(w, bytes))
+}
+
 //-------------------------------------

src/pack/toplevel.rs

@@ -209,6 +209,7 @@ fn pack_block<W: Write>(w: &mut W, kind: BT, buf: &[u8]) -> Result<()> {
         BT::NODE => pack_btree_node(w, buf).context("unable to pack btree node")?,
         BT::INDEX => pack_index(w, buf).context("unable to pack space map index")?,
         BT::BITMAP => pack_bitmap(w, buf).context("unable to pack space map bitmap")?,
+        BT::ARRAY => pack_array(w, buf).context("unable to pack array block")?,
         BT::UNKNOWN => return Err(anyhow!("asked to pack an unknown block type")),
     }
 

src/pdata/array.rs

@@ -0,0 +1,133 @@
+use nom::{multi::count, number::complete::*, IResult};
+use thiserror::Error;
+
+use crate::io_engine::BLOCK_SIZE;
+use crate::pdata::btree;
+use crate::pdata::unpack::Unpack;
+
+//------------------------------------------
+
+const ARRAY_BLOCK_HEADER_SIZE: u32 = 24;
+
+pub struct ArrayBlockHeader {
+    pub csum: u32,
+    pub max_entries: u32,
+    pub nr_entries: u32,
+    pub value_size: u32,
+    pub blocknr: u64,
+}
+
+impl Unpack for ArrayBlockHeader {
+    fn disk_size() -> u32 {
+        ARRAY_BLOCK_HEADER_SIZE
+    }
+
+    fn unpack(data: &[u8]) -> IResult<&[u8], ArrayBlockHeader> {
+        let (i, csum) = le_u32(data)?;
+        let (i, max_entries) = le_u32(i)?;
+        let (i, nr_entries) = le_u32(i)?;
+        let (i, value_size) = le_u32(i)?;
+        let (i, blocknr) = le_u64(i)?;
+
+        Ok((
+            i,
+            ArrayBlockHeader {
+                csum,
+                max_entries,
+                nr_entries,
+                value_size,
+                blocknr,
+            },
+        ))
+    }
+}
+
+pub struct ArrayBlock<V: Unpack> {
+    pub header: ArrayBlockHeader,
+    pub values: Vec<V>,
+}
+
+//------------------------------------------
+
+#[derive(Error, Clone, Debug)]
+pub enum ArrayError {
+    #[error("io_error")]
+    IoError,
+
+    #[error("block error: {0}")]
+    BlockError(String),
+
+    #[error("value error: {0}")]
+    ValueError(String),
+
+    #[error("aggregate: {0:?}")]
+    Aggregate(Vec<ArrayError>),
+
+    #[error("{0:?}, {1}")]
+    Path(Vec<u64>, Box<ArrayError>),
+
+    #[error(transparent)]
+    BTreeError(#[from] btree::BTreeError),
+}
+
+pub fn array_block_err(path: &[u64], msg: &str) -> ArrayError {
+    ArrayError::Path(
+        path.to_vec(),
+        Box::new(ArrayError::BlockError(msg.to_string())),
+    )
+}
+
+pub fn value_err(msg: String) -> ArrayError {
+    ArrayError::ValueError(msg)
+}
+
+pub fn aggregate_error(errs: Vec<ArrayError>) -> ArrayError {
+    ArrayError::Aggregate(errs)
+}
+
+pub type Result<T> = std::result::Result<T, ArrayError>;
+
+//------------------------------------------
+
+fn convert_result<'a, V>(path: &[u64], r: IResult<&'a [u8], V>) -> Result<(&'a [u8], V)> {
+    r.map_err(|_| array_block_err(path, "parse error"))
+}
+
+pub fn unpack_array_block<V: Unpack>(path: &[u64], data: &[u8]) -> Result<ArrayBlock<V>> {
+    // TODO: collect errors
+    let (i, header) =
+        ArrayBlockHeader::unpack(data).map_err(|_| array_block_err(
+            path,
+            "Couldn't parse array block header"
+        ))?;
+
+    // check value_size
+    if header.value_size != V::disk_size() {
+        return Err(array_block_err(
+            path,
+            &format!(
+                "value_size mismatch: expected {}, was {}",
+                V::disk_size(),
+                header.value_size
+            )
+        ));
+    }
+
+    // check max_entries
+    if header.value_size * header.max_entries + ARRAY_BLOCK_HEADER_SIZE > BLOCK_SIZE as u32 {
+        return Err(array_block_err(
+            path,
+            &format!("max_entries is too large ({})", header.max_entries)
+        ));
+    }
+
+    // TODO: check nr_entries < max_entries
+
+    // TODO: check block_nr
+
+    let (_i, values) = convert_result(path, count(V::unpack, header.nr_entries as usize)(i))?;
+
+    Ok(ArrayBlock { header, values })
+}
+
+//------------------------------------------

src/pdata/array_block.rs

@@ -1,60 +0,0 @@
-use anyhow::anyhow;
-use anyhow::Result;
-use nom::{multi::count, number::complete::*, IResult};
-
-use crate::pdata::unpack::Unpack;
-
-//------------------------------------------
-
-pub struct ArrayBlockHeader {
-    pub csum: u32,
-    pub max_entries: u32,
-    pub nr_entries: u32,
-    pub value_size: u32,
-    pub blocknr: u64,
-}
-
-impl Unpack for ArrayBlockHeader {
-    fn disk_size() -> u32 {
-        24
-    }
-
-    fn unpack(data: &[u8]) -> IResult<&[u8], ArrayBlockHeader> {
-        let (i, csum) = le_u32(data)?;
-        let (i, max_entries) = le_u32(i)?;
-        let (i, nr_entries) = le_u32(i)?;
-        let (i, value_size) = le_u32(i)?;
-        let (i, blocknr) = le_u64(i)?;
-
-        Ok((
-            i,
-            ArrayBlockHeader {
-                csum,
-                max_entries,
-                nr_entries,
-                value_size,
-                blocknr,
-            },
-        ))
-    }
-}
-
-pub struct ArrayBlock<V: Unpack> {
-    pub header: ArrayBlockHeader,
-    pub values: Vec<V>,
-}
-
-fn convert_result<V>(r: IResult<&[u8], V>) -> Result<(&[u8], V)> {
-    r.map_err(|_| anyhow!("parse error"))
-}
-
-pub fn unpack_array_block<V: Unpack>(data: &[u8]) -> Result<ArrayBlock<V>> {
-    // TODO: collect errors
-    let (i, header) =
-        ArrayBlockHeader::unpack(data).map_err(|_e| anyhow!("Couldn't parse header"))?;
-    let (_i, values) = convert_result(count(V::unpack, header.nr_entries as usize)(i))?;
-
-    Ok(ArrayBlock { header, values })
-}
-
-//------------------------------------------

src/pdata/array_walker.rs

@@ -1,8 +1,9 @@
 use std::sync::Arc;
 
+use crate::checksum;
 use crate::io_engine::*;
-use crate::pdata::array_block::*;
+use crate::pdata::array::{self, *};
-use crate::pdata::btree::*;
+use crate::pdata::btree::{self, *};
 use crate::pdata::btree_walker::*;
 use crate::pdata::unpack::*;
 
@@ -15,12 +16,12 @@ pub struct ArrayWalker {
 
 // FIXME: define another Result type for array visiting?
 pub trait ArrayVisitor<V: Unpack> {
-    fn visit(&self, index: u64, v: V) -> anyhow::Result<()>;
+    fn visit(&self, index: u64, v: V) -> array::Result<()>;
 }
 
 struct BlockValueVisitor<'a, V> {
     engine: Arc<dyn IoEngine + Send + Sync>,
-    array_block_visitor: &'a mut dyn ArrayVisitor<V>,
+    array_visitor: &'a mut dyn ArrayVisitor<V>,
 }
 
 impl<'a, V: Unpack + Copy> BlockValueVisitor<'a, V> {
@@ -30,22 +31,37 @@ impl<'a, V: Unpack + Copy> BlockValueVisitor<'a, V> {
     ) -> BlockValueVisitor<'a, V> {
         BlockValueVisitor {
             engine: e,
-            array_block_visitor: v,
+            array_visitor: v,
         }
     }
 
-    pub fn visit_array_block(&self, index: u64, array_block: ArrayBlock<V>) {
+    pub fn visit_array_block(&self, index: u64, array_block: ArrayBlock<V>) -> array::Result<()>{
+        let mut errs: Vec<ArrayError> = Vec::new();
+
         let begin = index * array_block.header.max_entries as u64;
         for i in 0..array_block.header.nr_entries {
-            self.array_block_visitor
+            if let Err(e) = self.array_visitor.visit(begin + i as u64, array_block.values[i as usize]) {
-                .visit(begin + i as u64, array_block.values[i as usize])
+                errs.push(e); // TODO: add path or keys context?
-                .unwrap();
+            }
+        }
+
+        // FIXME: duplicate to BTreeWalker::build_aggregrate()
+        match errs.len() {
+            0 => Ok(()),
+            1 => {
+                let e = errs[0].clone();
+                Err(e)
+            }
+            _ => {
+                let e = array::aggregate_error(errs);
+                Err(e)
+            }
         }
     }
 }
 
 impl<'a, V: Unpack + Copy> NodeVisitor<u64> for BlockValueVisitor<'a, V> {
-    // FIXME: return errors
+    // FIXME: wrap ArrayError into BTreeError, rather than mapping to value_err?
     fn visit(
         &self,
         path: &[u64],
@@ -53,20 +69,54 @@ impl<'a, V: Unpack + Copy> NodeVisitor<u64> for BlockValueVisitor<'a, V> {
         _h: &NodeHeader,
         keys: &[u64],
         values: &[u64],
-    ) -> Result<()> {
+    ) -> btree::Result<()> {
+        let mut path = path.to_vec();
+        let mut errs: Vec<BTreeError> = Vec::new();
+
         for (n, index) in keys.iter().enumerate() {
-            let b = self.engine.read(values[n]).map_err(|_| io_err(path))?;
+            let b = self.engine.read(values[n]).map_err(|_| io_err(&path))?;
-            let array_block = unpack_array_block::<V>(b.get_data()).map_err(|_| io_err(path))?;
+
-            self.visit_array_block(*index, array_block);
+            // FIXME: move to unpack_array_block?
+            let bt = checksum::metadata_block_type(b.get_data());
+            if bt != checksum::BT::ARRAY {
+                errs.push(btree::value_err(
+                    format!("checksum failed for array block {}, {:?}", b.loc, bt)
+                ));
+            }
+
+            path.push(values[n]);
+            match unpack_array_block::<V>(&path, b.get_data()) {
+                Ok(array_block) => {
+                    if let Err(e) = self.visit_array_block(*index, array_block) {
+                        errs.push(btree::value_err(format!("{}", e)));
+                    }
+                },
+                Err(e) => {
+                    errs.push(btree::value_err(format!("{}", e)));
+                }
+            }
+            path.pop();
         }
+
+        // FIXME: duplicate to BTreeWalker::build_aggregrate()
+        match errs.len() {
+            0 => Ok(()),
+            1 => {
+                let e = errs[0].clone();
+                Err(e)
+            }
+            _ => {
+                let e = btree::aggregate_error(errs);
+                Err(e)
+            }
+        }
+    }
+
+    fn visit_again(&self, _path: &[u64], _b: u64) -> btree::Result<()> {
         Ok(())
     }
 
-    fn visit_again(&self, _path: &[u64], _b: u64) -> Result<()> {
+    fn end_walk(&self) -> btree::Result<()> {
-        Ok(())
-    }
-
-    fn end_walk(&self) -> Result<()> {
         Ok(())
     }
 }
@@ -81,7 +131,7 @@ impl ArrayWalker {
     }
 
     // FIXME: redefine the Result type for array visiting?
-    pub fn walk<V>(&self, visitor: &mut dyn ArrayVisitor<V>, root: u64) -> Result<()>
+    pub fn walk<V>(&self, visitor: &mut dyn ArrayVisitor<V>, root: u64) -> array::Result<()>
     where
         V: Unpack + Copy,
     {
@@ -89,7 +139,7 @@ impl ArrayWalker {
         let mut path = Vec::new();
         path.push(0);
         let v = BlockValueVisitor::<V>::new(self.engine.clone(), visitor);
-        w.walk(&mut path, &v, root)
+        w.walk(&mut path, &v, root).map_err(|e| ArrayError::BTreeError(e))
     }
 }
 

src/pdata/bitset_walker.rs

@@ -1,10 +1,9 @@
-use anyhow::{anyhow, Result};
 use fixedbitset::FixedBitSet;
 use std::sync::{Arc, Mutex};
 
 use crate::io_engine::IoEngine;
+use crate::pdata::array;
 use crate::pdata::array_walker::{ArrayVisitor, ArrayWalker};
-use crate::pdata::btree::{self};
 
 struct BitsetVisitor<'a> {
     nr_entries: u64,
@@ -21,10 +20,10 @@ impl<'a> BitsetVisitor<'a> {
 }
 
 impl<'a> ArrayVisitor<u64> for BitsetVisitor<'a> {
-    fn visit(&self, index: u64, bits: u64) -> Result<()> {
+    fn visit(&self, index: u64, bits: u64) -> array::Result<()> {
         let begin: u64 = index << 6;
         if begin > self.nr_entries {
-            return Err(anyhow!("bitset size exceeds expectation"));
+            return Err(array::value_err("bitset size exceeds expectation".to_string()));
         }
 
         let end: u64 = std::cmp::min(begin + 64, self.nr_entries);
@@ -44,7 +43,7 @@ pub fn read_bitset(
     root: u64,
     ignore_none_fatal: bool,
     bitset: &mut FixedBitSet,
-)-> btree::Result<()> {
+)-> array::Result<()> {
     let w = ArrayWalker::new(engine.clone(), ignore_none_fatal);
     let mut v = BitsetVisitor::new(bitset);
     w.walk(&mut v, root)

src/pdata/mod.rs

@@ -1,4 +1,4 @@
-pub mod array_block;
+pub mod array;
 pub mod array_walker;
 pub mod bitset_walker;
 pub mod btree;