initial work on register directories

3 files changed, 339 insertions, 7 deletions

diff --git a/src/bin/geniza-register.rs b/src/bin/geniza-register.rs
new file mode 100644
index 0000000..6962be1
--- /dev/null
+++ b/src/bin/geniza-register.rs
@@ -0,0 +1,68 @@
+
+#[macro_use]
+extern crate clap;
+
+extern crate geniza;
+
+// TODO: more careful import
+use geniza::*;
+use std::path::Path;
+
+use clap::{App, SubCommand};
+
+fn run() -> Result<()> {
+ 
+    let matches = App::new("geniza-register")
+        .version(env!("CARGO_PKG_VERSION"))
+        .subcommand(SubCommand::with_name("info")
+            .about("Reads a SLEEP dir register and shows some basic metadata")
+            .arg_from_usage("<DIR> 'directory containing files'")
+            .arg_from_usage("<prefix> 'prefix for each data file'"))
+        .subcommand(SubCommand::with_name("create")
+            .about("Creates an SLEEP directory register (with header)")
+            .arg_from_usage("<DIR> 'directory containing files'")
+            .arg_from_usage("<prefix> 'prefix for each data file'"))
+        .get_matches();
+
+
+    match matches.subcommand() {
+        ("info", Some(subm)) => {
+            let dir = Path::new(subm.value_of("DIR").unwrap());
+            let prefix = subm.value_of("prefix").unwrap();
+            let sdr = SleepDirRegister::open(dir, prefix, false)?;
+            //debug!(println!("{:?}", sdr));
+            println!("Entry count: {}", sdr.len()?);
+        },
+        ("create", Some(subm)) => {
+            let dir = Path::new(subm.value_of("DIR").unwrap());
+            let prefix = subm.value_of("prefix").unwrap();
+            SleepDirRegister::create(dir, prefix)?;
+            println!("Done!");
+        },
+        _ => {
+            println!("Missing or unimplemented command!");
+            println!("{}", matches.usage());
+            ::std::process::exit(-1);
+        },
+    }
+    Ok(())
+}
+
+// TODO: is there a shorter error_chain 'main()' to use here?
+fn main() {
+    if let Err(ref e) = run() {
+        println!("error: {}", e);
+
+        for e in e.iter().skip(1) {
+            println!("caused by: {}", e);
+        }
+
+        // The backtrace is not always generated. Try to run this example
+        // with `RUST_BACKTRACE=1`.
+        if let Some(backtrace) = e.backtrace() {
+            println!("backtrace: {:?}", backtrace);
+        }
+
+        ::std::process::exit(1);
+    }
+}
diff --git a/src/bin/geniza-sleep.rs b/src/bin/geniza-sleep.rs
index b7bb88e..5061dc9 100644
--- a/src/bin/geniza-sleep.rs
+++ b/src/bin/geniza-sleep.rs
@@ -37,7 +37,7 @@ fn run() -> Result<()> {
             println!("Magic: 0x{:X}", sf.get_magic());
             println!("Algorithm: '{}'", sf.get_algorithm().or(Some("".to_string())).unwrap());
             println!("Entry Size (bytes): {}", sf.get_entry_size());
-            println!("Entry count: {}", sf.length()?);
+            println!("Entry count: {}", sf.len()?);
         },
         ("create", Some(subm)) => {
             let path = Path::new(subm.value_of("FILE").unwrap());
@@ -57,7 +57,7 @@ fn run() -> Result<()> {
         ("read-all", Some(subm)) => {
             let path = Path::new(subm.value_of("FILE").unwrap());
             let mut sf = SleepFile::open(path, false)?;
-            for i in 0..sf.length()? {
+            for i in 0..sf.len()? {
                 println!("{}: {:?}", i, sf.read(i));
             }
         },
diff --git a/src/lib.rs b/src/lib.rs
index 3754169..d61654d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2,6 +2,8 @@
 #[macro_use]
 extern crate error_chain;
 extern crate integer_encoding;
+extern crate crypto;
+extern crate rand;
 
 use std::io::prelude::*;
 use std::io::SeekFrom;
@@ -9,6 +11,10 @@ use std::path::Path;
 use std::fs::File;
 use integer_encoding::FixedInt;
 use std::fs::OpenOptions;
+use crypto::ed25519;
+use crypto::blake2b::Blake2b;
+use crypto::digest::Digest;
+use rand::Rng;
 
 mod errors {
     // Create the Error, ErrorKind, ResultExt, and Result types
@@ -27,7 +33,7 @@ pub trait SleepStorage {
     fn get_entry_size(&self) -> u16;
     fn read(&mut self, index: u64) -> Result<Vec<u8>>;
     fn write(&mut self, index: u64, data: &[u8]) -> Result<()>;
-    fn length(&self) -> Result<u64>;
+    fn len(&self) -> Result<u64>;
 }
 
 #[derive(Debug)]
@@ -43,6 +49,7 @@ impl SleepFile {
 
     // TODO: 'from' pre-existing File
 
+    // Something here to allow paths as references or actual Path...
     pub fn open(path: &Path, writable: bool) -> Result<SleepFile> {
 
         let mut f = OpenOptions::new()
@@ -63,12 +70,15 @@ impl SleepFile {
         let algorithm_name = if algo_len == 0 { None } else {
             Some(String::from_utf8_lossy(&header[8..(8+(algo_len as usize))]).into_owned())
         };
-        Ok(SleepFile {
+        let sf = SleepFile {
             file: f,
             magic: u32::from_be(FixedInt::decode_fixed(&header[0..4])),
             entry_size: u16::from_be(FixedInt::decode_fixed(&header[5..7])),
             algorithm_name: algorithm_name,
-        })
+        };
+        // call length for consistency checks
+        sf.len()?;
+        Ok(sf)
     }
 
     pub fn create(path: &Path, magic: u32, entry_size: u16, algo: Option<String>) -> Result<SleepFile> {
@@ -113,7 +123,7 @@ impl SleepStorage for SleepFile {
 
     fn read(&mut self, index: u64) -> Result<Vec<u8>> {
         let entry_size = self.entry_size as usize;
-        if index + 1 > self.length()? {
+        if index + 1 > self.len()? {
             return Err("Tried to read beyond end of SLEEP file".into());
         }
         let mut entry = vec![0; entry_size];
@@ -132,7 +142,7 @@ impl SleepStorage for SleepFile {
         Ok(())
     }
 
-    fn length(&self) -> Result<u64> {
+    fn len(&self) -> Result<u64> {
         let length = self.file.metadata()?.len();
         if length < 32 || (length - 32) % (self.entry_size as u64) != 0 {
             return Err("Bad SLEEP file: missing header or not multiple of entry_size".into());
@@ -140,3 +150,257 @@ impl SleepStorage for SleepFile {
         return Ok((length - 32) / (self.entry_size as u64))
     }
 }
+
+
+// Abstract access to hypercore register
+pub trait HyperRegister {
+    fn has(&self, index: u64) -> Result<bool>;
+    fn has_all(&self) -> Result<bool>;
+    fn has_range(&self, start: u64, end: u64) -> Result<bool>;
+    fn get(&mut self, index: u64) -> Result<Vec<u8>>;
+    fn append(&mut self, data: &[u8]) -> Result<u64>;
+    fn len(&self) -> Result<u64>;
+    fn len_bytes(&self) -> Result<u64>;
+    fn verify(&self) -> Result<()>;
+    fn check(&self) -> Result<()>;
+    fn writable(&self) -> bool;
+}
+
+#[derive(Debug)]
+pub struct SleepDirRegister {
+    tree_sleep: SleepFile,
+    sign_sleep: SleepFile,
+    bitfield_sleep: SleepFile,
+    data_file: Option<File>,
+    // Except, these should be Ed25519 keys, not bytes
+    pub_key: Vec<u8>,
+    secret_key: Option<Vec<u8>>,
+}
+
+impl SleepDirRegister {
+
+    pub fn open(directory: &Path, prefix: &str, writable: bool) -> Result<SleepDirRegister> {
+        // read public key from disk
+        let mut pub_key: Vec<u8> = vec![];
+        {
+            let mut key_file = OpenOptions::new()
+                .read(true)
+                .write(false)
+                .open(directory.join(Path::new(&(prefix.to_owned() + ".key"))))?;
+            // TODO: check key length?
+            key_file.read_to_end(&mut pub_key)?;
+        }
+        let data_path = &(prefix.to_owned() + ".data");
+        let data_path = Path::new(data_path);
+        let data_file = if data_path.is_file() {
+            Some(OpenOptions::new()
+                .read(true)
+                .write(writable)
+                .open(data_path)?)
+        } else {
+            None
+        };
+        let tree_sleep = SleepFile::open(
+            &directory.join(Path::new(&(prefix.to_owned() + ".tree"))), writable)?;
+        let sign_sleep = SleepFile::open(
+            &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))), writable)?;
+        let bitfield_sleep = SleepFile::open(
+            &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))), writable)?;
+        let sf = SleepDirRegister {
+            tree_sleep,
+            sign_sleep,
+            bitfield_sleep,
+            data_file,
+            pub_key,
+            secret_key: None,
+        };
+        sf.check()?;
+        Ok(sf)
+    }
+
+    pub fn create(directory: &Path, prefix: &str) -> Result<SleepDirRegister> {
+        // TODO: audit this for crypto strength... is rand appropriate?
+        let mut rand_seed = vec![0; 32];
+        let mut rng = rand::OsRng::new()?;
+        rng.fill_bytes(&mut rand_seed);
+        let (secret_key, pub_key) = ed25519::keypair(&rand_seed);
+        println!("{:?}", directory.join(Path::new(&(prefix.to_owned() + ".key"))));
+        {
+            let mut key_file = OpenOptions::new()
+                .write(true)
+                .create_new(true)
+                .open(directory.join(Path::new(&(prefix.to_owned() + ".key"))))?;
+            key_file.write_all(&pub_key)?;
+        }
+        let data_file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create_new(true)
+            .open(directory.join(Path::new(&(prefix.to_owned() + ".data"))))?;
+        let tree_sleep = SleepFile::create(
+            &directory.join(Path::new(&(prefix.to_owned() + ".tree"))),
+            0x05025702,
+            40,
+            Some("BLAKE2b".to_string()))?;
+        let sign_sleep = SleepFile::create(
+            &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))),
+            0x05025701,
+            65,
+            Some("Ed25519".to_string()))?;
+        let bitfield_sleep = SleepFile::create(
+            &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))),
+            0x05025700,
+            3328,
+            None)?;
+        let sf = SleepDirRegister {
+            tree_sleep,
+            sign_sleep,
+            bitfield_sleep,
+            data_file: Some(data_file),
+            pub_key: pub_key.to_vec(),
+            secret_key: Some(secret_key.to_vec()),
+        };
+        sf.check()?;
+        Ok(sf)
+    }
+}
+
+impl HyperRegister {
+
+    fn hash_leaf(data: &[u8]) -> [u8; 40] {
+        let mut buf = [0; 40];
+        u64::to_be(data.len() as u64)
+            .encode_fixed(&mut buf[32..40]);
+        let mut hash = Blake2b::new(32);
+        hash.input(&[0; 1]);
+        hash.input(&buf[32..40]);
+        hash.input(&data);
+        hash.result(&mut buf[0..32]);
+        buf
+    }
+
+    fn hash_parent(lhash: &[u8; 40], rhash: &[u8; 40]) -> [u8; 40] {
+        let mut buf = [0; 40];
+        // TODO: check overflow
+        let sum_size = u64::from_be(FixedInt::decode_fixed(&lhash[32..40])) +
+                       u64::from_be(FixedInt::decode_fixed(&rhash[32..40]));
+        u64::to_be(sum_size as u64)
+            .encode_fixed(&mut buf[32..40]);
+
+        let mut hash = Blake2b::new(32);
+        hash.input(&[1; 1]);
+        hash.input(&buf[32..40]);
+        hash.input(&lhash[..]);
+        hash.input(&rhash[..]);
+        hash.result(&mut buf[0..32]);
+        buf
+    }
+
+    fn root_nodes(len: u64) -> Vec<u64> {
+        // Calculates the root notes for a given length
+        // Basically factorize by powers of 2?
+        unimplemented!()
+    }
+}
+
+/*
+#[test]
+root_index:
+    0 -> []
+    1 -> 1
+    2 -> 2
+    3 -> 
+    8 -> 7
+*/
+
+impl HyperRegister for SleepDirRegister {
+
+    fn has(&self, index: u64) -> Result<bool> {
+        // looks in bitfield
+        unimplemented!()
+    }
+
+    fn has_all(&self) -> Result<bool> {
+        self.has_range(0, self.len()?)
+    }
+
+    fn has_range(&self, start: u64, end: u64) -> Result<bool> {
+        assert!(end > start);
+        for i in start..end {
+            if !self.has(i)? {
+                return Ok(false);
+            }
+        }
+        Ok(true)
+    }
+
+    fn get(&mut self, index: u64) -> Result<Vec<u8>> {
+        // Do we even have this chunk?
+        if !self.has(index)? {
+            return Err("Don't have that chunk".into());
+        }
+        // Get metadata about chunk (offset and length)
+        // Read chunk
+        unimplemented!()
+    }
+
+    fn append(&mut self, data: &[u8]) -> Result<u64> {
+        let mut data_file = if let Some(ref df) = self.data_file {
+            df
+        } else {
+            return Err("No data file in this register".into());
+        };
+        let index = self.len();
+        // 1. Hash data chunk
+        // 2. Append data to data file
+        data_file.seek(SeekFrom::End(0))?;
+        data_file.write_all(data)?;
+        // 3. Add hash to tree
+        // 4. Add signature to signature file
+        // 5. Update bitfile
+        unimplemented!()
+    }
+
+    fn len(&self) -> Result<u64> {
+        // Length in entry count.
+        let tree_len = self.tree_sleep.len()?;
+        if tree_len == 0 {
+            Ok(0)
+        } else if tree_len % 2 != 1 {
+            Err("Even number of tree file SLEEP entries".into())
+        } else {
+            Ok((self.tree_sleep.len()? / 2) + 1)
+        }
+    }
+
+    fn len_bytes(&self) -> Result<u64> {
+        // Total binary size of data file.
+        let mut data_file = if let Some(ref df) = self.data_file {
+            df
+        } else {
+            return Err("No data file in this register".into());
+        };
+        // Elaborate version will iterate through tree root nodes.
+        Ok(data_file.metadata()?.len())
+    }
+
+    fn verify(&self) -> Result<()> {
+        unimplemented!()
+    }
+
+    fn check(&self) -> Result<()> {
+        /* XXX:
+        let sign_len = self.sign_sleep.len()?;
+        let tree_len = self.tree_sleep.len()?;
+        if tree_len != sign_len * 2 {
+            return Err("Inconsistent SLEEP file sizes".into());
+        }
+        */
+        Ok(())
+    }
+
+    fn writable(&self) -> bool {
+        unimplemented!()
+        //self.sign_sleep.file.writable()
+    }
+}