From 6212b062699fdb90be8d4f4d2bffdde1bf0b183c Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 17 Oct 2017 23:40:51 -0700 Subject: few basic tests --- Cargo.lock | 10 +++ Cargo.toml | 3 + src/bin/geniza-register.rs | 2 +- src/lib.rs | 182 +++++++++++++++++++++++++++++++++++++-------- 4 files changed, 166 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2da5853..5952917 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,7 @@ dependencies = [ "integer-encoding 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -237,6 +238,14 @@ name = "strsim" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "tempdir" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "term_size" version = "0.3.0" @@ -350,6 +359,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rustc-demangle 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "aee45432acc62f7b9a108cc054142dac51f979e69e71ddce7d6fc7adf29e817e" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" +"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" "checksum term_size 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2b6b55df3198cc93372e85dd2ed817f0e38ce8cc0f22eb32391bfad9c4bf209" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df8e08afc40ae3459e4838f303e465aa50d823df8d7f83ca88108f6d3afe7edd" diff --git a/Cargo.toml b/Cargo.toml index 3660349..e00c9a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,3 +10,6 @@ clap = "2" error-chain = "0.7" env_logger = "0.3" rand = "0.3" + +[dev-dependencies] +tempdir = "0.3" diff --git a/src/bin/geniza-register.rs b/src/bin/geniza-register.rs index 8d70655..353857f 100644 --- a/src/bin/geniza-register.rs +++ b/src/bin/geniza-register.rs @@ -29,7 +29,7 @@ fn run() -> Result<()> { ("info", Some(subm)) => { let dir = Path::new(subm.value_of("DIR").unwrap()); let prefix = subm.value_of("prefix").unwrap(); - let sdr = SleepDirRegister::open(dir, prefix, false)?; + let mut sdr = SleepDirRegister::open(dir, prefix, false)?; //debug!(println!("{:?}", sdr)); println!("Entry count: {}", sdr.len()?); println!("Total size (bytes): {}", sdr.len_bytes()?); diff --git a/src/lib.rs b/src/lib.rs index 3535ba4..b36c36e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,9 @@ extern crate integer_encoding; extern crate crypto; extern crate rand; +#[cfg(test)] +extern crate tempdir; + use std::io::prelude::*; use std::io::SeekFrom; use std::path::Path; @@ -65,6 +68,9 @@ pub trait SleepStorage { /// Writes an entry at the given entry index (which is not a byte offset). fn write(&mut self, index: u64, data: &[u8]) -> Result<()>; + /// Writes a new entry at the end of the file + fn append(&mut self, data: &[u8]) -> Result<()>; + /// Returns the count of entries, meaning the highest index entry plus one (not necessarily the /// number of entries which have actually been written). fn len(&self) -> Result; @@ -177,6 +183,11 @@ impl SleepStorage for SleepFile { Ok(()) } + fn append(&mut self, data: &[u8]) -> Result<()> { + let index = self.len()?; + self.write(index, data) + } + fn len(&self) -> Result { let length = self.file.metadata()?.len(); if length < 32 || (length - 32) % (self.entry_size as u64) != 0 { @@ -186,6 +197,47 @@ impl SleepStorage for SleepFile { } } +#[test] +fn test_sleep_open() { + + let mut sf = SleepFile::open( + Path::new("test-data/sleep/empty/empty.sleep"), false).unwrap(); + + assert_eq!(sf.len().unwrap(), 0); + assert_eq!(sf.get_magic(), 0x050257FF); + assert_eq!(sf.get_algorithm(), None); + assert_eq!(sf.get_entry_size(), 1); + + let mut sf = SleepFile::open( + Path::new("test-data/dat/simple/.dat/metadata.tree"), false).unwrap(); + + // Calculated from 'dat log' + assert_eq!(sf.len().unwrap(), 5); + assert_eq!(sf.get_magic(), 0x05025702); + assert_eq!(sf.get_algorithm(), Some("BLAKE2b".to_string())); + assert_eq!(sf.get_entry_size(), 40); +} + +#[test] +fn test_sleep_create() { + + use tempdir::TempDir; + let tmp_dir = TempDir::new("geniza-test").unwrap(); + + let mut sf = SleepFile::create( + &tmp_dir.path().join("empty2.sleep"), + 0x050257FF, + 1, + None); + + // TODO: binary diff against 'test-data/sleep/empty/empty.sleep' + + let mut sf = SleepFile::create( + &tmp_dir.path().join("simple_metadata.sleep"), + 0x05025702, + 40, + Some("BLAKE2b".into())); +} /// Abstract access to Hypercore register pub trait HyperRegister { @@ -210,13 +262,13 @@ pub trait HyperRegister { fn len(&self) -> Result; /// Total size of this register in bytes. - fn len_bytes(&self) -> Result; + fn len_bytes(&mut self) -> Result; /// [UNIMPLEMENTED] Intended to do a deeper merkel-tree verification of all stored data - fn verify(&self) -> Result<()>; + fn verify(&mut self) -> Result<()>; /// Quick sanity checks on register store robust-ness - fn check(&self) -> Result<()>; + fn check(&mut self) -> Result<()>; /// Can this register be appended to? fn writable(&self) -> bool; @@ -266,7 +318,7 @@ impl SleepDirRegister { &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))), writable)?; let bitfield_sleep = SleepFile::open( &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))), writable)?; - let sf = SleepDirRegister { + let mut sf = SleepDirRegister { tree_sleep, sign_sleep, bitfield_sleep, @@ -305,14 +357,14 @@ impl SleepDirRegister { let sign_sleep = SleepFile::create( &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))), 0x05025701, - 65, + 64, Some("Ed25519".to_string()))?; let bitfield_sleep = SleepFile::create( &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))), 0x05025700, 3328, None)?; - let sf = SleepDirRegister { + let mut sf = SleepDirRegister { tree_sleep, sign_sleep, bitfield_sleep, @@ -408,7 +460,7 @@ impl HyperRegister { // log(N) would go up previous parent nodes (eg, use root_nodes()) let mut sum: u64 = 0; for i in 0..index { - let mut leaf = reg.get_tree_entry(i*2)?; + let leaf = reg.get_tree_entry(i*2)?; sum += u64::from_be(FixedInt::decode_fixed(&leaf[32..40])); } Ok(sum) @@ -460,12 +512,12 @@ impl HyperRegister for SleepDirRegister { bail!("Don't have that chunk"); } - let mut data_file = if let Some(ref mut df) = self.data_file { + let data_file = if let Some(ref mut df) = self.data_file { df } else { bail!("No data file in this register"); }; - let mut leaf = self.tree_sleep.read(index*2)?; + let leaf = self.tree_sleep.read(index*2)?; let data_len = u64::from_be(FixedInt::decode_fixed(&leaf[32..40])); // TODO: avoid foot-gun in development: cap at ~1 billion bytes assert!(data_len < 2u64.pow(29)); @@ -485,22 +537,33 @@ impl HyperRegister for SleepDirRegister { fn append(&mut self, data: &[u8]) -> Result { - let index = self.len()?; - let hash = HyperRegister::hash_roots(self, index+1)?; - - let mut data_file = if let Some(ref df) = self.data_file { - df - } else { + if !self.data_file.is_some() { bail!("No data file in this register"); }; + + let index = self.len()?; // 1. Hash data chunk + let leaf_hash = HyperRegister::hash_leaf(data); + // 2. Append data to data file - data_file.seek(SeekFrom::End(0))?; - data_file.write_all(data)?; - // 3. Add hash to tree + if let Some(ref mut df) = self.data_file { + df.seek(SeekFrom::End(0))?; + df.write_all(data)?; + df.sync_data()?; + } + + // 3. Add hash to tree file, update merkel tree + self.tree_sleep.write(index*2, &leaf_hash)?; + // TODO: tree_parent_index(u64) -> u64 function + // TODO: tree_child_entries(u64) function + // 4. Add signature to signature file + let root_hash = HyperRegister::hash_roots(self, index+1)?; + let root_sig = ed25519::signature(&root_hash, &self.secret_key.clone().unwrap()); + self.sign_sleep.append(&root_sig)?; + // 5. Update bitfile - unimplemented!() + Ok(index) } fn len(&self) -> Result { @@ -515,22 +578,22 @@ impl HyperRegister for SleepDirRegister { } } - fn len_bytes(&self) -> Result { - // Total binary size of data file. - let mut data_file = if let Some(ref df) = self.data_file { - df - } else { - bail!("No data file in this register"); - }; - // Elaborate version will iterate through tree root nodes. - Ok(data_file.metadata()?.len()) + fn len_bytes(&mut self) -> Result { + // TODO: this is a naive (linear) implementation + // log(N) would go up previous parent nodes (eg, use root_nodes()) + let mut sum: u64 = 0; + for i in 0..self.len()? { + let leaf = self.get_tree_entry(i*2)?; + sum += u64::from_be(FixedInt::decode_fixed(&leaf[32..40])); + } + Ok(sum) } - fn verify(&self) -> Result<()> { + fn verify(&mut self) -> Result<()> { unimplemented!() } - fn check(&self) -> Result<()> { + fn check(&mut self) -> Result<()> { let sign_len = self.sign_sleep.len()?; let tree_len = self.tree_sleep.len()?; if (tree_len == 0) && (sign_len == 0) { @@ -539,6 +602,13 @@ impl HyperRegister for SleepDirRegister { if tree_len != (sign_len * 2) - 1 { bail!("Inconsistent SLEEP signature/tree file sizes"); } + let computed = self.len_bytes()?; + if let Some(ref df) = self.data_file { + let file_size = df.metadata()?.len(); + if file_size != computed { + bail!("Computed vs. data file size mismatch"); + } + } Ok(()) } @@ -546,3 +616,55 @@ impl HyperRegister for SleepDirRegister { unimplemented!() } } + +#[test] +fn test_sdr_open() { + + let mut sdr = SleepDirRegister::open( + Path::new("test-data/dat/simple/.dat/"), "metadata", false).unwrap(); + + // Values from 'dat log' + assert_eq!(sdr.len().unwrap(), 3); + assert_eq!(sdr.len_bytes().unwrap(), 145); + + let mut sdr = SleepDirRegister::open( + Path::new("test-data/dat/simple/.dat/"), "content", false).unwrap(); + + // Values from 'dat log' + assert_eq!(sdr.len().unwrap(), 2); + assert_eq!(sdr.len_bytes().unwrap(), 204); +} + +#[test] +fn test_sdr_create() { + + use tempdir::TempDir; + + let tmp_dir = TempDir::new("geniza-test").unwrap(); + + let mut sdr = SleepDirRegister::create(tmp_dir.path(), "dummy").unwrap(); + + assert_eq!(sdr.len().unwrap(), 0); + assert_eq!(sdr.len_bytes().unwrap(), 0); +} + +#[test] +fn test_sdr_append() { + + use tempdir::TempDir; + + let tmp_dir = TempDir::new("geniza-test").unwrap(); + + let mut sdr = SleepDirRegister::create(tmp_dir.path(), "dummy").unwrap(); + + sdr.append("hello world!".as_bytes()).unwrap(); + sdr.check().unwrap(); + assert_eq!(sdr.len().unwrap(), 1); + assert_eq!(sdr.len_bytes().unwrap(), 12); + for i in 0..256 { + sdr.append(&[1,2,3,4,5]).unwrap(); + } + sdr.check().unwrap(); + assert_eq!(sdr.len().unwrap(), 1+256); + assert_eq!(sdr.len_bytes().unwrap(), 12 + (256*5)); +} -- cgit v1.2.3