diff options
| -rw-r--r-- | src/bin/geniza-net.rs | 53 | ||||
| -rw-r--r-- | src/bin/geniza-sleep.rs | 79 | ||||
| -rw-r--r-- | src/lib.rs | 12 | ||||
| -rw-r--r-- | src/protocol.rs | 88 | ||||
| -rw-r--r-- | src/sleep_file.rs | 57 | ||||
| -rw-r--r-- | src/sleep_register.rs | 151 |
6 files changed, 228 insertions, 212 deletions
diff --git a/src/bin/geniza-net.rs b/src/bin/geniza-net.rs index be4dba0..a3abc61 100644 --- a/src/bin/geniza-net.rs +++ b/src/bin/geniza-net.rs @@ -1,10 +1,10 @@ -// Free Software under GPL-3.0, see LICENSE +// Free Software under GPL-3.0, see LICENSE // Copyright 2017 Bryan Newbold -#[macro_use] -extern crate error_chain; extern crate clap; extern crate env_logger; +#[macro_use] +extern crate error_chain; extern crate geniza; // TODO: more careful import @@ -12,19 +12,22 @@ use geniza::*; use clap::{App, SubCommand}; fn run() -> Result<()> { - env_logger::init().unwrap(); - + let matches = App::new("geniza-net") .version(env!("CARGO_PKG_VERSION")) - .subcommand(SubCommand::with_name("connect") - .about("Connects to a peer and exchanges handshake") - .arg_from_usage("<host_port> 'peer host:port to connect to'") - .arg_from_usage("<dat_key> 'dat key (public key) to register with'")) - .subcommand(SubCommand::with_name("receive-all") - .about("Connects to a peer, pulls all metadata and content") - .arg_from_usage("<host_port> 'peer host:port to connect to'") - .arg_from_usage("<dat_key> 'dat key (public key) to register with'")) + .subcommand( + SubCommand::with_name("connect") + .about("Connects to a peer and exchanges handshake") + .arg_from_usage("<host_port> 'peer host:port to connect to'") + .arg_from_usage("<dat_key> 'dat key (public key) to register with'"), + ) + .subcommand( + SubCommand::with_name("receive-all") + .about("Connects to a peer, pulls all metadata and content") + .arg_from_usage("<host_port> 'peer host:port to connect to'") + .arg_from_usage("<dat_key> 'dat key (public key) to register with'"), + ) .get_matches(); @@ -32,51 +35,45 @@ fn run() -> Result<()> { ("connect", Some(subm)) => { let host_port = subm.value_of("host_port").unwrap(); let dat_key = subm.value_of("dat_key").unwrap(); - if dat_key.len() != 32*2 { + if dat_key.len() != 32 * 2 { bail!("dat key not correct length"); } let mut key_bytes = vec![]; for i in 0..32 { - let r = u8::from_str_radix(&dat_key[2*i .. 2*i+2], 16); + let r = u8::from_str_radix(&dat_key[2 * i..2 * i + 2], 16); match r { Ok(b) => key_bytes.push(b), Err(e) => bail!("Problem with hex: {}", e), }; } - DatConnection::connect( - host_port, - &key_bytes, - false)?; + DatConnection::connect(host_port, &key_bytes, false)?; println!("Done!"); - }, + } ("receive-all", Some(subm)) => { let host_port = subm.value_of("host_port").unwrap(); let dat_key = subm.value_of("dat_key").unwrap(); - if dat_key.len() != 32*2 { + if dat_key.len() != 32 * 2 { bail!("dat key not correct length"); } let mut key_bytes = vec![]; for i in 0..32 { - let r = u8::from_str_radix(&dat_key[2*i .. 2*i+2], 16); + let r = u8::from_str_radix(&dat_key[2 * i..2 * i + 2], 16); match r { Ok(b) => key_bytes.push(b), Err(e) => bail!("Problem with hex: {}", e), }; } - let mut dc = DatConnection::connect( - host_port, - &key_bytes, - false)?; + let mut dc = DatConnection::connect(host_port, &key_bytes, false)?; // XXX: number here totally arbitrary dc.receive_all(false, 10)?; dc.receive_all(true, 10)?; println!("Done!"); - }, + } _ => { println!("Missing or unimplemented command!"); println!("{}", matches.usage()); ::std::process::exit(-1); - }, + } } Ok(()) } diff --git a/src/bin/geniza-sleep.rs b/src/bin/geniza-sleep.rs index 461df0d..1a50d17 100644 --- a/src/bin/geniza-sleep.rs +++ b/src/bin/geniza-sleep.rs @@ -1,11 +1,11 @@ -// Free Software under GPL-3.0, see LICENSE +// Free Software under GPL-3.0, see LICENSE // Copyright 2017 Bryan Newbold #[macro_use] -extern crate error_chain; -#[macro_use] extern crate clap; extern crate env_logger; +#[macro_use] +extern crate error_chain; extern crate geniza; // TODO: more careful import @@ -14,31 +14,40 @@ use std::path::Path; use clap::{App, SubCommand}; fn run() -> Result<()> { - env_logger::init().unwrap(); - + let matches = App::new("geniza-sleep") .version(env!("CARGO_PKG_VERSION")) - .subcommand(SubCommand::with_name("info") - .about("Reads a SLEEP dir register and shows some basic metadata") - .arg_from_usage("<DIR> 'directory containing files'") - .arg_from_usage("<prefix> 'prefix for each data file'")) - .subcommand(SubCommand::with_name("create") - .about("Creates an SLEEP directory register (with header)") - .arg_from_usage("<DIR> 'directory containing files'") - .arg_from_usage("<prefix> 'prefix for each data file'")) - .subcommand(SubCommand::with_name("file-info") - .about("Reads a single SLEEP file and shows some basic metadata") - .arg_from_usage("<FILE> 'SLEEP file to read'")) - .subcommand(SubCommand::with_name("file-create") - .about("Creates an empty single SLEEP file (with header)") - .arg_from_usage("<FILE> 'SLEEP file to write (can't exist)'") - .arg_from_usage("<magic> 'Magic word to use (eg, 0x5025700)'") - .arg_from_usage("<entry_size> 'Size of each entry (bytes)'") - .arg_from_usage("<algo_name> 'Name of algorithm (empty string for none)'")) - .subcommand(SubCommand::with_name("read-all") - .about("Reads a single SLEEP file, iterates through all entries, prints raw bytes") - .arg_from_usage("<FILE> 'SLEEP file to read'")) + .subcommand( + SubCommand::with_name("info") + .about("Reads a SLEEP dir register and shows some basic metadata") + .arg_from_usage("<DIR> 'directory containing files'") + .arg_from_usage("<prefix> 'prefix for each data file'"), + ) + .subcommand( + SubCommand::with_name("create") + .about("Creates an SLEEP directory register (with header)") + .arg_from_usage("<DIR> 'directory containing files'") + .arg_from_usage("<prefix> 'prefix for each data file'"), + ) + .subcommand( + SubCommand::with_name("file-info") + .about("Reads a single SLEEP file and shows some basic metadata") + .arg_from_usage("<FILE> 'SLEEP file to read'"), + ) + .subcommand( + SubCommand::with_name("file-create") + .about("Creates an empty single SLEEP file (with header)") + .arg_from_usage("<FILE> 'SLEEP file to write (can't exist)'") + .arg_from_usage("<magic> 'Magic word to use (eg, 0x5025700)'") + .arg_from_usage("<entry_size> 'Size of each entry (bytes)'") + .arg_from_usage("<algo_name> 'Name of algorithm (empty string for none)'"), + ) + .subcommand( + SubCommand::with_name("read-all") + .about("Reads a single SLEEP file, iterates through all entries, prints raw bytes") + .arg_from_usage("<FILE> 'SLEEP file to read'"), + ) .get_matches(); @@ -50,22 +59,25 @@ fn run() -> Result<()> { //debug!(println!("{:?}", sdr)); println!("Entry count: {}", sdr.len()?); println!("Total size (bytes): {}", sdr.len_bytes()?); - }, + } ("create", Some(subm)) => { let dir = Path::new(subm.value_of("DIR").unwrap()); let prefix = subm.value_of("prefix").unwrap(); SleepDirRegister::create(dir, prefix)?; println!("Done!"); - }, + } ("file-info", Some(subm)) => { let path = Path::new(subm.value_of("FILE").unwrap()); let sf = SleepFile::open(path, false)?; //debug!(println!("{:?}", sf)); println!("Magic: 0x{:X}", sf.get_magic()); - println!("Algorithm: '{}'", sf.get_algorithm().or(Some("".to_string())).unwrap()); + println!( + "Algorithm: '{}'", + sf.get_algorithm().or(Some("".to_string())).unwrap() + ); println!("Entry Size (bytes): {}", sf.get_entry_size()); println!("Entry count: {}", sf.len()?); - }, + } ("file-create", Some(subm)) => { let path = Path::new(subm.value_of("FILE").unwrap()); let algo_name = subm.value_of("algo_name").unwrap(); @@ -78,21 +90,22 @@ fn run() -> Result<()> { path, value_t_or_exit!(subm, "magic", u32), value_t_or_exit!(subm, "entry_size", u16), - algo_name)?; + algo_name, + )?; println!("Done!"); - }, + } ("file-read-all", Some(subm)) => { let path = Path::new(subm.value_of("FILE").unwrap()); let mut sf = SleepFile::open(path, false)?; for i in 0..sf.len()? { println!("{}: {:?}", i, sf.read(i)); } - }, + } _ => { println!("Missing or unimplemented command!"); println!("{}", matches.usage()); ::std::process::exit(-1); - }, + } } Ok(()) } @@ -1,4 +1,4 @@ -// Free Software under GPL-3.0, see LICENSE +// Free Software under GPL-3.0, see LICENSE // Copyright 2017 Bryan Newbold //! This is a hobby/learning implementation of the dat distributed data synchronization system. @@ -13,16 +13,16 @@ //! Additional notes in the source code for this repo, under the 'notes' directory. Also, see //! README. +extern crate crypto; +extern crate env_logger; #[macro_use] extern crate error_chain; +extern crate integer_encoding; #[macro_use] extern crate log; -extern crate env_logger; -extern crate integer_encoding; -extern crate crypto; -extern crate sodiumoxide; -extern crate rand; extern crate protobuf; +extern crate rand; +extern crate sodiumoxide; #[cfg(test)] extern crate tempdir; diff --git a/src/protocol.rs b/src/protocol.rs index 7a64f88..aaad914 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -31,37 +31,36 @@ pub enum DatNetMessage { fn msg_code(msg: &DatNetMessage) -> u8 { match msg { - &DatNetMessage::Register(_) => 0, + &DatNetMessage::Register(_) => 0, &DatNetMessage::Handshake(_) => 1, - &DatNetMessage::Status(_) => 2, - &DatNetMessage::Have(_) => 3, - &DatNetMessage::Unhave(_) => 4, - &DatNetMessage::Want(_) => 5, - &DatNetMessage::Unwant(_) => 6, - &DatNetMessage::Request(_) => 7, - &DatNetMessage::Cancel(_) => 8, - &DatNetMessage::Data(_) => 9, + &DatNetMessage::Status(_) => 2, + &DatNetMessage::Have(_) => 3, + &DatNetMessage::Unhave(_) => 4, + &DatNetMessage::Want(_) => 5, + &DatNetMessage::Unwant(_) => 6, + &DatNetMessage::Request(_) => 7, + &DatNetMessage::Cancel(_) => 8, + &DatNetMessage::Data(_) => 9, } } fn msg_sugar(msg: &DatNetMessage) -> &Message { match msg { - &DatNetMessage::Register(ref m) => m, + &DatNetMessage::Register(ref m) => m, &DatNetMessage::Handshake(ref m) => m, - &DatNetMessage::Status(ref m) => m, - &DatNetMessage::Have(ref m) => m, - &DatNetMessage::Unhave(ref m) => m, - &DatNetMessage::Want(ref m) => m, - &DatNetMessage::Unwant(ref m) => m, - &DatNetMessage::Request(ref m) => m, - &DatNetMessage::Cancel(ref m) => m, - &DatNetMessage::Data(ref m) => m, + &DatNetMessage::Status(ref m) => m, + &DatNetMessage::Have(ref m) => m, + &DatNetMessage::Unhave(ref m) => m, + &DatNetMessage::Want(ref m) => m, + &DatNetMessage::Unwant(ref m) => m, + &DatNetMessage::Request(ref m) => m, + &DatNetMessage::Cancel(ref m) => m, + &DatNetMessage::Data(ref m) => m, } } /// This helper is pretty slow/inefficient; lots of copying memory fn bytewise_stream_xor_ic_inplace(buf: &mut [u8], byte_offset: u64, nonce: &Nonce, key: &Key) { - let mut offset = byte_offset; // We may have a partial-64-byte-block to finish encrypting first @@ -70,26 +69,25 @@ fn bytewise_stream_xor_ic_inplace(buf: &mut [u8], byte_offset: u64, nonce: &Nonc if partial_len != 0 { let mut partial = vec![0; 64]; for i in 0..partial_len { - partial[partial_offset+i] = buf[i]; + partial[partial_offset + i] = buf[i]; } let partial_enc = stream_xor_ic(&partial, offset / 64, &nonce, &key); offset += partial_len as u64; for i in 0..partial_len { - buf[i] = partial_enc[partial_offset+i]; + buf[i] = partial_enc[partial_offset + i]; } } if buf.len() > partial_len { let main_enc = stream_xor_ic(&buf[partial_len..], offset / 64, &nonce, &key); //offset += main_enc.len() as u64; for i in 0..main_enc.len() { - buf[partial_len+i] = main_enc[i]; + buf[partial_len + i] = main_enc[i]; } } } #[test] fn test_bsxii_short() { - let nonce = gen_nonce(); let key = gen_key(); @@ -110,7 +108,6 @@ fn test_bsxii_short() { #[test] fn test_bsxii_continued() { - let nonce = gen_nonce(); let key = gen_key(); @@ -161,7 +158,6 @@ pub struct DatConnection { } impl Read for DatConnection { - /// Encrypted TCP read (after connection initialized). Uses XOR of an XSalsa20 stream, using /// block offsets. fn read(&mut self, buf: &mut [u8]) -> ::std::io::Result<usize> { @@ -174,10 +170,8 @@ impl Read for DatConnection { } impl Write for DatConnection { - /// Encrypted write to complement `read()`. fn write(&mut self, buf: &[u8]) -> ::std::io::Result<usize> { - // Don't mutate what we've been passed let mut enc = vec![0; buf.len()]; enc.copy_from_slice(buf); @@ -194,9 +188,7 @@ impl Write for DatConnection { } impl DatConnection { - pub fn connect(host_port: &str, key: &[u8], live: bool) -> Result<DatConnection> { - let timeout = Duration::new(7, 0); let tx_nonce = gen_nonce(); let mut local_id = [0; 32]; @@ -224,13 +216,12 @@ impl DatConnection { data_discovery_key: [0; 32], tx_nonce: tx_nonce, tx_offset: 0, - rx_nonce: gen_nonce(), // dummy + rx_nonce: gen_nonce(), // dummy rx_offset: 0, }; // Exchange register/feed dc.tcp.set_nodelay(true)?; // Faster handshake - // send register let mut register_msg = Feed::new(); register_msg.set_discoveryKey(dc.discovery_key.to_vec()); register_msg.set_nonce((tx_nonce[0..24]).to_vec()); @@ -281,13 +272,17 @@ impl DatConnection { } fn send_msg(&mut self, dnm: &DatNetMessage, is_content: bool) -> Result<()> { - let header_int: u8 = (is_content as u8) << 4 | (msg_code(dnm) & 0x0F); let msg: &Message = msg_sugar(dnm); let total_message_size = (msg.compute_size() as usize) + 1; - trace!("SEND total_len={} header={} is_content={} type={:?}", - total_message_size, header_int, is_content, &dnm); + trace!( + "SEND total_len={} header={} is_content={} type={:?}", + total_message_size, + header_int, + is_content, + &dnm + ); // send both header varints, and data self.write_varint(total_message_size as u64)?; @@ -309,13 +304,17 @@ impl DatConnection { } fn recv_msg(&mut self) -> Result<(bool, DatNetMessage)> { - let total_len: u64 = self.read_varint()?; let header: u8 = self.read_varint()?; let is_content = (header & (1 << 4)) != 0; - trace!("RECV total_len={} header={} is_content={}", total_len, header, is_content); + trace!( + "RECV total_len={} header={} is_content={}", + total_len, + header, + is_content + ); if header > 0x1F { bail!("Invalid header received: {}", header); @@ -351,8 +350,12 @@ impl DatConnection { let header_int: u8 = 0; let total_message_size = (reg.compute_size() as usize) + 1; - trace!("SEND total_len={} header={} msg={:?}", - total_message_size, header_int, reg); + trace!( + "SEND total_len={} header={} msg={:?}", + total_message_size, + header_int, + reg + ); self.tcp.write_varint(total_message_size as u64)?; self.tcp.write_varint(header_int as u32)?; @@ -383,7 +386,6 @@ impl DatConnection { } pub fn get_data_key(&mut self) -> Result<()> { - // Status: downloading, not uploading let mut sm = Info::new(); sm.set_uploading(false); @@ -419,7 +421,7 @@ impl DatConnection { info!("Expected Have message, got: {:?}", &msg); continue; } - }; + } // Request let mut rm = Request::new(); @@ -443,7 +445,8 @@ impl DatConnection { } info!("Got data discovery key"); self.data_key.copy_from_slice(&data_key[0..32]); - self.data_discovery_key.copy_from_slice(&make_discovery_key(data_key)[0..32]); + self.data_discovery_key + .copy_from_slice(&make_discovery_key(data_key)[0..32]); return Ok(()); } else { unimplemented!("non-hyperdrive Index type: {}", index_msg.get_field_type()); @@ -457,7 +460,6 @@ impl DatConnection { } pub fn receive_all(&mut self, is_content: bool, length: u64) -> Result<()> { - // Status: downloading, not uploading let mut sm = Info::new(); sm.set_uploading(false); @@ -488,7 +490,7 @@ impl DatConnection { loop { let (was_content, msg) = self.recv_msg()?; - if was_content != is_content{ + if was_content != is_content { info!("Expected other message channel"); continue; } diff --git a/src/sleep_file.rs b/src/sleep_file.rs index 36bf7a9..71e95ba 100644 --- a/src/sleep_file.rs +++ b/src/sleep_file.rs @@ -12,7 +12,6 @@ use errors::*; /// /// Back-ends could be in RAM, on disk, remote HTTP, etc. pub trait SleepStorage { - /// Returns the 32-bit "magic word", indicating the content type, in native format (aka, not /// necessarily big-endian). fn get_magic(&self) -> u32; @@ -50,12 +49,10 @@ pub struct SleepFile { } impl SleepFile { - // TODO: 'from' pre-existing File // Something here to allow paths as references or actual Path... pub fn open(path: &Path, writable: bool) -> Result<SleepFile> { - let mut f = OpenOptions::new() .read(true) .write(writable) @@ -71,8 +68,10 @@ impl SleepFile { if algo_len > 24 { bail!("Invalid SLEEP header: can't have algo_len > 24"); } - let algorithm_name = if algo_len == 0 { None } else { - Some(String::from_utf8_lossy(&header[8..(8+(algo_len as usize))]).into_owned()) + let algorithm_name = if algo_len == 0 { + None + } else { + Some(String::from_utf8_lossy(&header[8..(8 + (algo_len as usize))]).into_owned()) }; let sf = SleepFile { file: f, @@ -86,8 +85,12 @@ impl SleepFile { } /// This function will *not* allow overwriting an existing file. - pub fn create(path: &Path, magic: u32, entry_size: u16, algo: Option<String>) -> Result<SleepFile> { - + pub fn create( + path: &Path, + magic: u32, + entry_size: u16, + algo: Option<String>, + ) -> Result<SleepFile> { let mut header = [0; 32]; u32::to_be(magic).encode_fixed(&mut header[0..4]); header[4] = 0; // version @@ -99,7 +102,7 @@ impl SleepFile { bail!("Algorithm name must be 24 bytes at most"); } header[7] = algo_len as u8; - header[8..(8+algo_len)].clone_from_slice(name); + header[8..(8 + algo_len)].clone_from_slice(name); } else { header[7] = 0; }; @@ -120,10 +123,15 @@ impl SleepFile { } impl SleepStorage for SleepFile { - - fn get_magic(&self) -> u32 { self.magic } - fn get_algorithm(&self) -> Option<String> { self.algorithm_name.clone() } - fn get_entry_size(&self) -> u16 { self.entry_size } + fn get_magic(&self) -> u32 { + self.magic + } + fn get_algorithm(&self) -> Option<String> { + self.algorithm_name.clone() + } + fn get_entry_size(&self) -> u16 { + self.entry_size + } fn read(&mut self, index: u64) -> Result<Vec<u8>> { let entry_size = self.entry_size as usize; @@ -131,7 +139,8 @@ impl SleepStorage for SleepFile { bail!("Tried to read beyond end of SLEEP file"); } let mut entry = vec![0; entry_size]; - self.file.seek(SeekFrom::Start(32 + (entry_size as u64) * index))?; + self.file + .seek(SeekFrom::Start(32 + (entry_size as u64) * index))?; self.file.read_exact(&mut entry)?; Ok(entry) } @@ -141,7 +150,8 @@ impl SleepStorage for SleepFile { if data.len() != self.entry_size as usize { bail!("Tried to write mis-sized data"); } - self.file.seek(SeekFrom::Start(32 + (self.entry_size as u64) * index))?; + self.file + .seek(SeekFrom::Start(32 + (self.entry_size as u64) * index))?; self.file.write_all(&data)?; Ok(()) } @@ -156,23 +166,20 @@ impl SleepStorage for SleepFile { if length < 32 || (length - 32) % (self.entry_size as u64) != 0 { bail!("Bad SLEEP file: missing header or not multiple of entry_size"); } - return Ok((length - 32) / (self.entry_size as u64)) + return Ok((length - 32) / (self.entry_size as u64)); } } #[test] fn test_sleep_open() { - - let sf = SleepFile::open( - Path::new("test-data/sleep/empty/empty.sleep"), false).unwrap(); + let sf = SleepFile::open(Path::new("test-data/sleep/empty/empty.sleep"), false).unwrap(); assert_eq!(sf.len().unwrap(), 0); assert_eq!(sf.get_magic(), 0x050257FF); assert_eq!(sf.get_algorithm(), None); assert_eq!(sf.get_entry_size(), 1); - let sf = SleepFile::open( - Path::new("test-data/dat/simple/.dat/metadata.tree"), false).unwrap(); + let sf = SleepFile::open(Path::new("test-data/dat/simple/.dat/metadata.tree"), false).unwrap(); // Calculated from 'dat log' assert_eq!(sf.len().unwrap(), 5); @@ -183,15 +190,10 @@ fn test_sleep_open() { #[test] fn test_sleep_create() { - use tempdir::TempDir; let tmp_dir = TempDir::new("geniza-test").unwrap(); - SleepFile::create( - &tmp_dir.path().join("empty2.sleep"), - 0x050257FF, - 1, - None).unwrap(); + SleepFile::create(&tmp_dir.path().join("empty2.sleep"), 0x050257FF, 1, None).unwrap(); // TODO: binary diff against 'test-data/sleep/empty/empty.sleep' @@ -199,5 +201,6 @@ fn test_sleep_create() { &tmp_dir.path().join("simple_metadata.sleep"), 0x05025702, 40, - Some("BLAKE2b".into())).unwrap(); + Some("BLAKE2b".into()), + ).unwrap(); } diff --git a/src/sleep_register.rs b/src/sleep_register.rs index e798cad..fa8229d 100644 --- a/src/sleep_register.rs +++ b/src/sleep_register.rs @@ -8,14 +8,13 @@ use std::fs::OpenOptions; use crypto::blake2b::Blake2b; use crypto::digest::Digest; use crypto::ed25519; -use rand::{Rng, OsRng}; +use rand::{OsRng, Rng}; use errors::*; use sleep_file::*; /// Abstract access to Hypercore register pub trait HyperRegister { - /// Whether the register store contains the given (data) entry fn has(&self, index: u64) -> Result<bool>; @@ -52,11 +51,9 @@ pub trait HyperRegister { } impl HyperRegister { - fn hash_leaf(data: &[u8]) -> [u8; 40] { let mut buf = [0; 40]; - u64::to_be(data.len() as u64) - .encode_fixed(&mut buf[32..40]); + u64::to_be(data.len() as u64).encode_fixed(&mut buf[32..40]); let mut hash = Blake2b::new(32); hash.input(&[0; 1]); hash.input(&buf[32..40]); @@ -68,10 +65,9 @@ impl HyperRegister { fn hash_parent(lhash: &[u8], rhash: &[u8]) -> [u8; 40] { let mut buf = [0; 40]; // TODO: check overflow - let sum_size = u64::from_be(FixedInt::decode_fixed(&lhash[32..40])) + - u64::from_be(FixedInt::decode_fixed(&rhash[32..40])); - u64::to_be(sum_size as u64) - .encode_fixed(&mut buf[32..40]); + let sum_size = u64::from_be(FixedInt::decode_fixed(&lhash[32..40])) + + u64::from_be(FixedInt::decode_fixed(&rhash[32..40])); + u64::to_be(sum_size as u64).encode_fixed(&mut buf[32..40]); let mut hash = Blake2b::new(32); hash.input(&[1; 1]); @@ -96,7 +92,6 @@ impl HyperRegister { } hash.result(&mut buf[0..32]); Ok(buf.to_vec()) - } fn tree_root_nodes(data_count: u64) -> Vec<u64> { @@ -124,7 +119,7 @@ impl HyperRegister { let mut roots = vec![]; for x in components { roots.push(accum + (x - 1)); - accum += 2*x; + accum += 2 * x; } roots } @@ -134,7 +129,7 @@ impl HyperRegister { // log(N) would go up previous parent nodes (eg, use root_nodes()) let mut sum: u64 = 0; for i in 0..index { - let leaf = reg.get_tree_entry(i*2)?; + let leaf = reg.get_tree_entry(i * 2)?; sum += u64::from_be(FixedInt::decode_fixed(&leaf[32..40])); } Ok(sum) @@ -147,14 +142,14 @@ impl HyperRegister { // find lowest-significant zero bit if (index & (1 << i)) == 0 { // set that bit and clear next higher - return ((index | (1 << i))) & !(1 << (i+1)); + return ((index | (1 << i))) & !(1 << (i + 1)); } } panic!("Parent lookup overflowed, huge index!"); } /// Calling this on a leaf node is an error, as is calling very high node numbers (> 2^62) - fn tree_child_indices(index: u64) -> Result<(u64,u64)> { + fn tree_child_indices(index: u64) -> Result<(u64, u64)> { if index % 2 == 0 { bail!("Leaf tree nodes have no children"); } @@ -162,9 +157,9 @@ impl HyperRegister { // find lowest-significant zero bit... if (index & (1 << i)) == 0 { // larger child has this bit high, next lower bit cleared - let right = ((index | (1 << i))) & !(1 << (i-1)); + let right = ((index | (1 << i))) & !(1 << (i - 1)); // smaller child has next lower bit cleared - let left = index & !(1 << (i-1)); + let left = index & !(1 << (i - 1)); return Ok((left, right)); } } @@ -177,11 +172,11 @@ fn test_tree_root_nodes() { assert_eq!(HyperRegister::tree_root_nodes(0), vec![]); assert_eq!(HyperRegister::tree_root_nodes(1), vec![0]); assert_eq!(HyperRegister::tree_root_nodes(2), vec![1]); - assert_eq!(HyperRegister::tree_root_nodes(3), vec![1,4]); + assert_eq!(HyperRegister::tree_root_nodes(3), vec![1, 4]); assert_eq!(HyperRegister::tree_root_nodes(4), vec![3]); - assert_eq!(HyperRegister::tree_root_nodes(5), vec![3,8]); - assert_eq!(HyperRegister::tree_root_nodes(6), vec![3,9]); - assert_eq!(HyperRegister::tree_root_nodes(7), vec![3,9,12]); + assert_eq!(HyperRegister::tree_root_nodes(5), vec![3, 8]); + assert_eq!(HyperRegister::tree_root_nodes(6), vec![3, 9]); + assert_eq!(HyperRegister::tree_root_nodes(7), vec![3, 9, 12]); assert_eq!(HyperRegister::tree_root_nodes(8), vec![7]); } @@ -205,12 +200,12 @@ fn test_tree_parent_index() { fn test_tree_child_indices() { assert!(HyperRegister::tree_child_indices(0).is_err()); assert!(HyperRegister::tree_child_indices(1024).is_err()); - assert_eq!(HyperRegister::tree_child_indices(1).unwrap(), (0, 2)); + assert_eq!(HyperRegister::tree_child_indices(1).unwrap(), (0, 2)); - assert_eq!(HyperRegister::tree_child_indices(3).unwrap(), (1, 5)); - assert_eq!(HyperRegister::tree_child_indices(5).unwrap(), (4, 6)); - assert_eq!(HyperRegister::tree_child_indices(7).unwrap(), (3, 11)); - assert_eq!(HyperRegister::tree_child_indices(9).unwrap(), (8, 10)); + assert_eq!(HyperRegister::tree_child_indices(3).unwrap(), (1, 5)); + assert_eq!(HyperRegister::tree_child_indices(5).unwrap(), (4, 6)); + assert_eq!(HyperRegister::tree_child_indices(7).unwrap(), (3, 11)); + assert_eq!(HyperRegister::tree_child_indices(9).unwrap(), (8, 10)); assert_eq!(HyperRegister::tree_child_indices(11).unwrap(), (9, 13)); assert_eq!(HyperRegister::tree_child_indices(13).unwrap(), (12, 14)); assert_eq!(HyperRegister::tree_child_indices(15).unwrap(), (7, 23)); @@ -230,46 +225,49 @@ pub struct SleepDirRegister { } fn read_key_file(path: &Path, is_secret: bool) -> Result<Vec<u8>> { - let expected = if is_secret { 64 } else { 32 }; let mut key = vec![]; - let mut key_file = OpenOptions::new() - .read(true) - .write(false) - .open(path)?; + let mut key_file = OpenOptions::new().read(true).write(false).open(path)?; key_file.read_to_end(&mut key)?; if key.len() != expected { - bail!("Bad key file (len {} != {}): {:?}", key.len(), expected, path); + bail!( + "Bad key file (len {} != {}): {:?}", + key.len(), + expected, + path + ); } Ok(key) } fn write_key_file(path: &Path, key: &[u8], is_secret: bool) -> Result<()> { - let expected = if is_secret { 64 } else { 32 }; if key.len() != expected { - bail!("Bad key file (len {} != {}): {:?}", key.len(), expected, path); + bail!( + "Bad key file (len {} != {}): {:?}", + key.len(), + expected, + path + ); } - let mut key_file = OpenOptions::new() - .write(true) - .create_new(true) - .open(path)?; + let mut key_file = OpenOptions::new().write(true).create_new(true).open(path)?; key_file.write_all(&key)?; Ok(()) } impl SleepDirRegister { - pub fn open(directory: &Path, prefix: &str, writable: bool) -> Result<SleepDirRegister> { // read public key from disk let pub_key: Vec<u8> = read_key_file( &directory.join(Path::new(&(prefix.to_owned() + ".key"))), - false)?; + false, + )?; let mut secret_key = None; if writable { secret_key = Some(read_key_file( &directory.join(Path::new(&(prefix.to_owned() + ".secret_key"))), - true)?); + true, + )?); } let data_path = &(prefix.to_owned() + ".data"); let data_path = Path::new(data_path); @@ -282,11 +280,17 @@ impl SleepDirRegister { None }; let tree_sleep = SleepFile::open( - &directory.join(Path::new(&(prefix.to_owned() + ".tree"))), writable)?; + &directory.join(Path::new(&(prefix.to_owned() + ".tree"))), + writable, + )?; let sign_sleep = SleepFile::open( - &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))), writable)?; + &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))), + writable, + )?; let bitfield_sleep = SleepFile::open( - &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))), writable)?; + &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))), + writable, + )?; let mut sf = SleepDirRegister { tree_sleep, sign_sleep, @@ -308,11 +312,13 @@ impl SleepDirRegister { write_key_file( &directory.join(Path::new(&(prefix.to_owned() + ".key"))), &pub_key, - false)?; + false, + )?; write_key_file( &directory.join(Path::new(&(prefix.to_owned() + ".secret_key"))), &secret_key, - true)?; + true, + )?; let data_file = OpenOptions::new() .read(true) .write(true) @@ -322,17 +328,20 @@ impl SleepDirRegister { &directory.join(Path::new(&(prefix.to_owned() + ".tree"))), 0x05025702, 40, - Some("BLAKE2b".to_string()))?; + Some("BLAKE2b".to_string()), + )?; let sign_sleep = SleepFile::create( &directory.join(Path::new(&(prefix.to_owned() + ".signatures"))), 0x05025701, 64, - Some("Ed25519".to_string()))?; + Some("Ed25519".to_string()), + )?; let bitfield_sleep = SleepFile::create( &directory.join(Path::new(&(prefix.to_owned() + ".bitfield"))), 0x05025700, 3328, - None)?; + None, + )?; let mut sf = SleepDirRegister { tree_sleep, sign_sleep, @@ -347,7 +356,6 @@ impl SleepDirRegister { } impl HyperRegister for SleepDirRegister { - /// TODO: this version only works for "dense" registers: it just checks if the index is in the /// total length, instead of using the bitfield. fn has(&self, index: u64) -> Result<bool> { @@ -370,7 +378,6 @@ impl HyperRegister for SleepDirRegister { } fn get_data_entry(&mut self, index: u64) -> Result<Vec<u8>> { - // Get metadata about chunk (offset and length) let offset = HyperRegister::get_data_offset(self, index)?; @@ -384,7 +391,7 @@ impl HyperRegister for SleepDirRegister { } else { bail!("No data file in this register"); }; - let leaf = self.tree_sleep.read(index*2)?; + let leaf = self.tree_sleep.read(index * 2)?; let data_len = u64::from_be(FixedInt::decode_fixed(&leaf[32..40])); // avoid foot-gun in development: cap at ~1 billion bytes assert!(data_len < 2u64.pow(29)); @@ -403,7 +410,6 @@ impl HyperRegister for SleepDirRegister { } fn append(&mut self, data: &[u8]) -> Result<u64> { - if !self.data_file.is_some() { bail!("No data file in this register"); }; @@ -420,22 +426,21 @@ impl HyperRegister for SleepDirRegister { } // 3. Add hash to tree file, update merkel tree - self.tree_sleep.write(index*2, &leaf_hash)?; - let mut parent = HyperRegister::tree_parent_index(index*2); - while parent < index*2 { + self.tree_sleep.write(index * 2, &leaf_hash)?; + let mut parent = HyperRegister::tree_parent_index(index * 2); + while parent < index * 2 { let (left, right) = HyperRegister::tree_child_indices(parent)?; - let (left, right) = (self.tree_sleep.read(left)?, - self.tree_sleep.read(right)?); + let (left, right) = (self.tree_sleep.read(left)?, self.tree_sleep.read(right)?); let parent_hash = HyperRegister::hash_parent(&left[0..40], &right[0..40]); self.tree_sleep.write(parent, &parent_hash[0..40])?; parent = HyperRegister::tree_parent_index(parent); } - + // 4. Add signature to signature file - let root_hash = HyperRegister::hash_roots(self, index+1)?; + let root_hash = HyperRegister::hash_roots(self, index + 1)?; let root_sig = ed25519::signature(&root_hash, &self.secret_key.clone().unwrap()); self.sign_sleep.append(&root_sig)?; - + // 5. Update bitfile Ok(index) } @@ -457,7 +462,7 @@ impl HyperRegister for SleepDirRegister { // log(N) would go up previous parent nodes (eg, use tree_root_nodes()) let mut sum: u64 = 0; for i in 0..self.len()? { - let leaf = self.get_tree_entry(i*2)?; + let leaf = self.get_tree_entry(i * 2)?; sum += u64::from_be(FixedInt::decode_fixed(&leaf[32..40])); } Ok(sum) @@ -471,7 +476,7 @@ impl HyperRegister for SleepDirRegister { let sign_len = self.sign_sleep.len()?; let tree_len = self.tree_sleep.len()?; if (tree_len == 0) && (sign_len == 0) { - return Ok(()) + return Ok(()); } if tree_len != (sign_len * 2) - 1 { bail!("Inconsistent SLEEP signature/tree file sizes"); @@ -488,22 +493,21 @@ impl HyperRegister for SleepDirRegister { /// Checks if we have the secret key (such that we can append to this register) fn writable(&self) -> bool { - return self.secret_key.is_some() + return self.secret_key.is_some(); } } #[test] fn test_sdr_open() { - - let mut sdr = SleepDirRegister::open( - Path::new("test-data/dat/simple/.dat/"), "metadata", false).unwrap(); + let mut sdr = + SleepDirRegister::open(Path::new("test-data/dat/simple/.dat/"), "metadata", false).unwrap(); // Values from 'dat log' assert_eq!(sdr.len().unwrap(), 3); assert_eq!(sdr.len_bytes().unwrap(), 145); - let mut sdr = SleepDirRegister::open( - Path::new("test-data/dat/simple/.dat/"), "content", false).unwrap(); + let mut sdr = + SleepDirRegister::open(Path::new("test-data/dat/simple/.dat/"), "content", false).unwrap(); // Values from 'dat log' assert_eq!(sdr.len().unwrap(), 2); @@ -512,7 +516,6 @@ fn test_sdr_open() { #[test] fn test_sdr_create() { - use tempdir::TempDir; let tmp_dir = TempDir::new("geniza-test").unwrap(); let mut sdr = SleepDirRegister::create(tmp_dir.path(), "dummy").unwrap(); @@ -523,7 +526,6 @@ fn test_sdr_create() { #[test] fn test_sdr_append() { - use tempdir::TempDir; let tmp_dir = TempDir::new("geniza-test").unwrap(); let mut sdr = SleepDirRegister::create(tmp_dir.path(), "dummy").unwrap(); @@ -532,18 +534,17 @@ fn test_sdr_append() { sdr.check().unwrap(); assert_eq!(sdr.len().unwrap(), 1); assert_eq!(sdr.len_bytes().unwrap(), 12); - let count = 100; // TODO: make this >1000 when things are faster + let count = 100; // TODO: make this >1000 when things are faster for _ in 0..count { - sdr.append(&[1,2,3,4,5]).unwrap(); + sdr.append(&[1, 2, 3, 4, 5]).unwrap(); } sdr.check().unwrap(); - assert_eq!(sdr.len().unwrap(), 1+count); - assert_eq!(sdr.len_bytes().unwrap(), 12 + (count*5)); + assert_eq!(sdr.len().unwrap(), 1 + count); + assert_eq!(sdr.len_bytes().unwrap(), 12 + (count * 5)); } #[test] fn test_sdr_has() { - use tempdir::TempDir; let tmp_dir = TempDir::new("geniza-test").unwrap(); let mut sdr = SleepDirRegister::create(tmp_dir.path(), "dummy").unwrap(); |