From b30dcccd4f6e3098e6bba7cc58ee12e4196a54e4 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 17 Oct 2017 10:46:24 -0700 Subject: initial work on register directories --- src/bin/geniza-register.rs | 68 +++++++++++ src/bin/geniza-sleep.rs | 4 +- src/lib.rs | 274 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 339 insertions(+), 7 deletions(-) create mode 100644 src/bin/geniza-register.rs (limited to 'src') diff --git a/src/bin/geniza-register.rs b/src/bin/geniza-register.rs new file mode 100644 index 0000000..6962be1 --- /dev/null +++ b/src/bin/geniza-register.rs @@ -0,0 +1,68 @@ + +#[macro_use] +extern crate clap; + +extern crate geniza; + +// TODO: more careful import +use geniza::*; +use std::path::Path; + +use clap::{App, SubCommand}; + +fn run() -> Result<()> { + + let matches = App::new("geniza-register") + .version(env!("CARGO_PKG_VERSION")) + .subcommand(SubCommand::with_name("info") + .about("Reads a SLEEP dir register and shows some basic metadata") + .arg_from_usage(" 'directory containing files'") + .arg_from_usage(" 'prefix for each data file'")) + .subcommand(SubCommand::with_name("create") + .about("Creates an SLEEP directory register (with header)") + .arg_from_usage(" 'directory containing files'") + .arg_from_usage(" 'prefix for each data file'")) + .get_matches(); + + + match matches.subcommand() { + ("info", Some(subm)) => { + let dir = Path::new(subm.value_of("DIR").unwrap()); + let prefix = subm.value_of("prefix").unwrap(); + let sdr = SleepDirRegister::open(dir, prefix, false)?; + //debug!(println!("{:?}", sdr)); + println!("Entry count: {}", sdr.len()?); + }, + ("create", Some(subm)) => { + let dir = Path::new(subm.value_of("DIR").unwrap()); + let prefix = subm.value_of("prefix").unwrap(); + SleepDirRegister::create(dir, prefix)?; + println!("Done!"); + }, + _ => { + println!("Missing or unimplemented command!"); + println!("{}", matches.usage()); + ::std::process::exit(-1); + }, + } + Ok(()) +} + +// TODO: is there a shorter error_chain 'main()' to use here? +fn main() { + if let Err(ref e) = run() { + println!("error: {}", e); + + for e in e.iter().skip(1) { + println!("caused by: {}", e); + } + + // The backtrace is not always generated. Try to run this example + // with `RUST_BACKTRACE=1`. + if let Some(backtrace) = e.backtrace() { + println!("backtrace: {:?}", backtrace); + } + + ::std::process::exit(1); + } +} diff --git a/src/bin/geniza-sleep.rs b/src/bin/geniza-sleep.rs index b7bb88e..5061dc9 100644 --- a/src/bin/geniza-sleep.rs +++ b/src/bin/geniza-sleep.rs @@ -37,7 +37,7 @@ fn run() -> Result<()> { println!("Magic: 0x{:X}", sf.get_magic()); println!("Algorithm: '{}'", sf.get_algorithm().or(Some("".to_string())).unwrap()); println!("Entry Size (bytes): {}", sf.get_entry_size()); - println!("Entry count: {}", sf.length()?); + println!("Entry count: {}", sf.len()?); }, ("create", Some(subm)) => { let path = Path::new(subm.value_of("FILE").unwrap()); @@ -57,7 +57,7 @@ fn run() -> Result<()> { ("read-all", Some(subm)) => { let path = Path::new(subm.value_of("FILE").unwrap()); let mut sf = SleepFile::open(path, false)?; - for i in 0..sf.length()? { + for i in 0..sf.len()? { println!("{}: {:?}", i, sf.read(i)); } }, diff --git a/src/lib.rs b/src/lib.rs index 3754169..d61654d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,8 @@ #[macro_use] extern crate error_chain; extern crate integer_encoding; +extern crate crypto; +extern crate rand; use std::io::prelude::*; use std::io::SeekFrom; @@ -9,6 +11,10 @@ use std::path::Path; use std::fs::File; use integer_encoding::FixedInt; use std::fs::OpenOptions; +use crypto::ed25519; +use crypto::blake2b::Blake2b; +use crypto::digest::Digest; +use rand::Rng; mod errors { // Create the Error, ErrorKind, ResultExt, and Result types @@ -27,7 +33,7 @@ pub trait SleepStorage { fn get_entry_size(&self) -> u16; fn read(&mut self, index: u64) -> Result>; fn write(&mut self, index: u64, data: &[u8]) -> Result<()>; - fn length(&self) -> Result; + fn len(&self) -> Result; } #[derive(Debug)] @@ -43,6 +49,7 @@ impl SleepFile { // TODO: 'from' pre-existing File + // Something here to allow paths as references or actual Path... pub fn open(path: &Path, writable: bool) -> Result { let mut f = OpenOptions::new() @@ -63,12 +70,15 @@ impl SleepFile { let algorithm_name = if algo_len == 0 { None } else { Some(String::from_utf8_lossy(&header[8..(8+(algo_len as usize))]).into_owned()) }; - Ok(SleepFile { + let sf = SleepFile { file: f, magic: u32::from_be(FixedInt::decode_fixed(&header[0..4])), entry_size: u16::from_be(FixedInt::decode_fixed(&header[5..7])), algorithm_name: algorithm_name, - }) + }; + // call length for consistency checks + sf.len()?; + Ok(sf) } pub fn create(path: &Path, magic: u32, entry_size: u16, algo: Option) -> Result { @@ -113,7 +123,7 @@ impl SleepStorage for SleepFile { fn read(&mut self, index: u64) -> Result> { let entry_size = self.entry_size as usize; - if index + 1 > self.length()? { + if index + 1 > self.len()? { return Err("Tried to read beyond end of SLEEP file".into()); } let mut entry = vec![0; entry_size]; @@ -132,7 +142,7 @@ impl SleepStorage for SleepFile { Ok(()) } - fn length(&self) -> Result { + fn len(&self) -> Result { let length = self.file.metadata()?.len(); if length < 32 || (length - 32) % (self.entry_size as u64) != 0 { return Err("Bad SLEEP file: missing header or not multiple of entry_size".into()); @@ -140,3 +150,257 @@ impl SleepStorage for SleepFile { return Ok((length - 32) / (self.entry_size as u64)) } } + + +// Abstract access to hypercore register +pub trait HyperRegister { + fn has(&self, index: u64) -> Result; + fn has_all(&self) -> Result; + fn has_range(&self, start: u64, end: u64) -> Result; + fn get(&mut self, index: u64) -> Result>; + fn append(&mut self, data: &[u8]) -> Result; + fn len(&self) -> Result; + fn len_bytes(&self) -> Result; + fn verify(&self) -> Result<()>; + fn check(&self) -> Result<()>; + fn writable(&self) -> bool; +} + +#[derive(Debug)] +pub struct SleepDirRegister { + tree_sleep: SleepFile, + sign_sleep: SleepFile, + bitfield_sleep: SleepFile, + data_file: Option, + // Except, these should be Ed25519 keys, not bytes + pub_key: Vec, + secret_key: Option>, +} + +impl SleepDirRegister { + + pub fn open(directory: &Path, prefix: &str, writable: bool) -> Result { + // read public key from disk + let mut pub_key: Vec = vec![]; + { + let mut key_file = OpenOptions::new() + .read(true) + .write(false) + .open(directory.join(Path::new(&(prefix.to_owned() + ".key"))))?; + // TODO: check key length? + key_file.read_to_end(&mut pub_key)?; + } + let data_path = &(prefix.to_owned() + ".data"); + let data_path = Path::new(data_path); + let data_file = if data_path.is_file() { + Some(OpenOptions::new() + .read(true) + .write(writable) + .open(data_path)?) + } else { + None + }; + let tree_sleep = SleepFile::open( + &directory.join(Path::new(&(prefix.to_owned() + ".tree"))), writable)?; + let sign_sleep = SleepFile::open( + &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))), writable)?; + let bitfield_sleep = SleepFile::open( + &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))), writable)?; + let sf = SleepDirRegister { + tree_sleep, + sign_sleep, + bitfield_sleep, + data_file, + pub_key, + secret_key: None, + }; + sf.check()?; + Ok(sf) + } + + pub fn create(directory: &Path, prefix: &str) -> Result { + // TODO: audit this for crypto strength... is rand appropriate? + let mut rand_seed = vec![0; 32]; + let mut rng = rand::OsRng::new()?; + rng.fill_bytes(&mut rand_seed); + let (secret_key, pub_key) = ed25519::keypair(&rand_seed); + println!("{:?}", directory.join(Path::new(&(prefix.to_owned() + ".key")))); + { + let mut key_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(directory.join(Path::new(&(prefix.to_owned() + ".key"))))?; + key_file.write_all(&pub_key)?; + } + let data_file = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(directory.join(Path::new(&(prefix.to_owned() + ".data"))))?; + let tree_sleep = SleepFile::create( + &directory.join(Path::new(&(prefix.to_owned() + ".tree"))), + 0x05025702, + 40, + Some("BLAKE2b".to_string()))?; + let sign_sleep = SleepFile::create( + &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))), + 0x05025701, + 65, + Some("Ed25519".to_string()))?; + let bitfield_sleep = SleepFile::create( + &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))), + 0x05025700, + 3328, + None)?; + let sf = SleepDirRegister { + tree_sleep, + sign_sleep, + bitfield_sleep, + data_file: Some(data_file), + pub_key: pub_key.to_vec(), + secret_key: Some(secret_key.to_vec()), + }; + sf.check()?; + Ok(sf) + } +} + +impl HyperRegister { + + fn hash_leaf(data: &[u8]) -> [u8; 40] { + let mut buf = [0; 40]; + u64::to_be(data.len() as u64) + .encode_fixed(&mut buf[32..40]); + let mut hash = Blake2b::new(32); + hash.input(&[0; 1]); + hash.input(&buf[32..40]); + hash.input(&data); + hash.result(&mut buf[0..32]); + buf + } + + fn hash_parent(lhash: &[u8; 40], rhash: &[u8; 40]) -> [u8; 40] { + let mut buf = [0; 40]; + // TODO: check overflow + let sum_size = u64::from_be(FixedInt::decode_fixed(&lhash[32..40])) + + u64::from_be(FixedInt::decode_fixed(&rhash[32..40])); + u64::to_be(sum_size as u64) + .encode_fixed(&mut buf[32..40]); + + let mut hash = Blake2b::new(32); + hash.input(&[1; 1]); + hash.input(&buf[32..40]); + hash.input(&lhash[..]); + hash.input(&rhash[..]); + hash.result(&mut buf[0..32]); + buf + } + + fn root_nodes(len: u64) -> Vec { + // Calculates the root notes for a given length + // Basically factorize by powers of 2? + unimplemented!() + } +} + +/* +#[test] +root_index: + 0 -> [] + 1 -> 1 + 2 -> 2 + 3 -> + 8 -> 7 +*/ + +impl HyperRegister for SleepDirRegister { + + fn has(&self, index: u64) -> Result { + // looks in bitfield + unimplemented!() + } + + fn has_all(&self) -> Result { + self.has_range(0, self.len()?) + } + + fn has_range(&self, start: u64, end: u64) -> Result { + assert!(end > start); + for i in start..end { + if !self.has(i)? { + return Ok(false); + } + } + Ok(true) + } + + fn get(&mut self, index: u64) -> Result> { + // Do we even have this chunk? + if !self.has(index)? { + return Err("Don't have that chunk".into()); + } + // Get metadata about chunk (offset and length) + // Read chunk + unimplemented!() + } + + fn append(&mut self, data: &[u8]) -> Result { + let mut data_file = if let Some(ref df) = self.data_file { + df + } else { + return Err("No data file in this register".into()); + }; + let index = self.len(); + // 1. Hash data chunk + // 2. Append data to data file + data_file.seek(SeekFrom::End(0))?; + data_file.write_all(data)?; + // 3. Add hash to tree + // 4. Add signature to signature file + // 5. Update bitfile + unimplemented!() + } + + fn len(&self) -> Result { + // Length in entry count. + let tree_len = self.tree_sleep.len()?; + if tree_len == 0 { + Ok(0) + } else if tree_len % 2 != 1 { + Err("Even number of tree file SLEEP entries".into()) + } else { + Ok((self.tree_sleep.len()? / 2) + 1) + } + } + + fn len_bytes(&self) -> Result { + // Total binary size of data file. + let mut data_file = if let Some(ref df) = self.data_file { + df + } else { + return Err("No data file in this register".into()); + }; + // Elaborate version will iterate through tree root nodes. + Ok(data_file.metadata()?.len()) + } + + fn verify(&self) -> Result<()> { + unimplemented!() + } + + fn check(&self) -> Result<()> { + /* XXX: + let sign_len = self.sign_sleep.len()?; + let tree_len = self.tree_sleep.len()?; + if tree_len != sign_len * 2 { + return Err("Inconsistent SLEEP file sizes".into()); + } + */ + Ok(()) + } + + fn writable(&self) -> bool { + unimplemented!() + //self.sign_sleep.file.writable() + } +} -- cgit v1.2.3