diff options
| -rw-r--r-- | README.md | 7 | ||||
| -rw-r--r-- | TODO | 7 | ||||
| -rw-r--r-- | src/bitfield.rs | 55 | ||||
| -rw-r--r-- | src/helpers.rs | 61 | ||||
| -rw-r--r-- | src/lib.rs | 70 | ||||
| -rw-r--r-- | src/peer.rs | 12 | ||||
| -rw-r--r-- | src/synchronizer.rs (renamed from src/node.rs) | 57 |
7 files changed, 156 insertions, 113 deletions
@@ -30,15 +30,16 @@ it will eat-your-data! - [x] import file to register - [x] export file from register - [x] import/export directories recursively -- [ ] Networking +- [ ] Protocol - [x] send/receive encrypted messages to a known host - - [ ] receive entire register from a known host - - [ ] share (upload) register to a known host - [ ] bitfields - [ ] Discovery - [x] centralized DNS - [ ] mDNS (local DNS) - [ ] DHT (distributed hash table) +- [ ] Peer Synchronization + - [ ] receive entire register from a known host + - [ ] share (upload) register to a known host - [ ] Wrapper commands - [ ] clone - [ ] share @@ -1,11 +1,10 @@ next: +- DatPeer wrapper API +- Synchronizer API - clean up node bitfield implementation - "insert" API for HyperRegister -- crude single-thread drive+network clone - sparse register clone -- "needed data chunks" and progress function? -- crude single-thread drive+network broadcast - 'geniza clone' command - 'geniza checkout' command - 'geniza init', 'geniza commit' commands @@ -28,7 +27,7 @@ Backburner: - API to link and run from, eg, python - uTP transport - in-memory storage backend -- WASM +- compile to WASM... play in browser? - multi-connection network sync (per-connection state, etc) - duplicate file/chunk optimizations - secret_key-in-home helpers (read/write) diff --git a/src/bitfield.rs b/src/bitfield.rs new file mode 100644 index 0000000..11640ea --- /dev/null +++ b/src/bitfield.rs @@ -0,0 +1,55 @@ + +use errors::*; +use integer_encoding::VarInt; +use bit_field::BitArray; + + +// WrappedBitfield +// +// uses vec of u64 internally? +// +// fn from_message() +// fn get(u64) + +pub fn decode_bitfield(raw_bf: &[u8]) -> Result<Vec<u8>> { + let mut offset = 0; // byte offset that we have read up to + if raw_bf.len() < 1 { + bail!("Expected (varint-encoded) bitfield to have len>=1"); + } + let mut bit_array: Vec<u8> = vec![]; + while offset < raw_bf.len() { + let (header, inc): (u64, usize) = VarInt::decode_var(&raw_bf[offset..]); + offset += inc; + + if (header & 0x01) == 0x01 { + // compressed + let bit = (header & 0x02) == 0x02; + let run_len = header >> 2; + if bit { + bit_array.append(&mut vec![0xFF; run_len as usize]); + } else { + bit_array.append(&mut vec![0x00; run_len as usize]); + } + } else { + // uncompressed + let byte_count = header >> 1; + let mut data = raw_bf[offset..(offset + byte_count as usize)].to_vec(); + bit_array.append(&mut data); + offset += byte_count as usize; + } + } + // XXX: HACK + bit_array.reverse(); + return Ok(bit_array); +} + +/// Finds the index of the lowest bit +pub fn max_high_bit(bf: &[u8]) -> u64 { + // XXX: HACK, going backwards + for i in 0..bf.bit_length() { + if bf.get_bit(i) { + return (bf.bit_length() - i - 1) as u64; + } + } + return 0; +} diff --git a/src/helpers.rs b/src/helpers.rs new file mode 100644 index 0000000..32f0e25 --- /dev/null +++ b/src/helpers.rs @@ -0,0 +1,61 @@ + +use errors::*; +use crypto::digest::Digest; +use crypto::blake2b::Blake2b; + +/// Helper to calculate a discovery key from a public key. 'key' should be 32 bytes; the returned +/// array will also be 32 bytes long. +/// +/// dat discovery keys are calculated as a BLAKE2b "keyed hash" (using the passed key) of the string +/// "hypercore" (with no trailing null byte). +pub fn make_discovery_key(key: &[u8]) -> Vec<u8> { + let mut discovery_key = [0; 32]; + let mut hash = Blake2b::new_keyed(32, key); + hash.input(&"hypercore".as_bytes()); + hash.result(&mut discovery_key); + discovery_key.to_vec() +} + +/// Helper to parse a dat address (aka, public key) in string format. +/// +/// Address can start with 'dat://'. It should contain 64 hexadecimal characters. +pub fn parse_dat_address(input: &str) -> Result<Vec<u8>> { + + let raw_key = if input.starts_with("dat://") { + &input[6..] + } else { + input + }; + if raw_key.len() != 32 * 2 { + bail!("dat key not correct length"); + } + let mut key_bytes = vec![]; + for i in 0..32 { + let r = u8::from_str_radix(&raw_key[2 * i..2 * i + 2], 16); + match r { + Ok(b) => key_bytes.push(b), + Err(e) => bail!("Problem with hex: {}", e), + }; + } + Ok(key_bytes) +} + +#[test] +fn test_parse_dat_address() { + + assert!(parse_dat_address( + "c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_ok()); + assert!(parse_dat_address( + "C7638882870ABD4044D6467B0738F15E3A36F57C3A7F7F3417FD7E4E0841D597").is_ok()); + assert!(parse_dat_address( + "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_ok()); + + assert!(parse_dat_address( + "c7638882870ab").is_err()); + assert!(parse_dat_address( + "g7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_err()); + assert!(parse_dat_address( + "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d5970").is_err()); + assert!(parse_dat_address( + "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d59").is_err()); +} @@ -46,6 +46,10 @@ mod errors { pub use errors::*; // Organize code internally (files, modules), but pull it all into a flat namespace to export. +mod helpers; +pub use helpers::*; +mod bitfield; +pub use bitfield::*; mod sleep_file; pub use sleep_file::*; mod sleep_register; @@ -56,68 +60,10 @@ mod protocol; pub use protocol::*; pub mod network_msgs; pub mod metadata_msgs; -mod node; -pub use node::*; mod discovery; pub use discovery::*; +mod peer; +pub use peer::*; +mod synchronizer; +pub use synchronizer::*; -// Shared functions -use crypto::digest::Digest; -use crypto::blake2b::Blake2b; - -/// Helper to calculate a discovery key from a public key. 'key' should be 32 bytes; the returned -/// array will also be 32 bytes long. -/// -/// dat discovery keys are calculated as a BLAKE2b "keyed hash" (using the passed key) of the string -/// "hypercore" (with no trailing null byte). -pub fn make_discovery_key(key: &[u8]) -> Vec<u8> { - let mut discovery_key = [0; 32]; - let mut hash = Blake2b::new_keyed(32, key); - hash.input(&"hypercore".as_bytes()); - hash.result(&mut discovery_key); - discovery_key.to_vec() -} - -/// Helper to parse a dat address (aka, public key) in string format. -/// -/// Address can start with 'dat://'. It should contain 64 hexadecimal characters. -pub fn parse_dat_address(input: &str) -> Result<Vec<u8>> { - - let raw_key = if input.starts_with("dat://") { - &input[6..] - } else { - input - }; - if raw_key.len() != 32 * 2 { - bail!("dat key not correct length"); - } - let mut key_bytes = vec![]; - for i in 0..32 { - let r = u8::from_str_radix(&raw_key[2 * i..2 * i + 2], 16); - match r { - Ok(b) => key_bytes.push(b), - Err(e) => bail!("Problem with hex: {}", e), - }; - } - Ok(key_bytes) -} - -#[test] -fn test_parse_dat_address() { - - assert!(parse_dat_address( - "c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_ok()); - assert!(parse_dat_address( - "C7638882870ABD4044D6467B0738F15E3A36F57C3A7F7F3417FD7E4E0841D597").is_ok()); - assert!(parse_dat_address( - "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_ok()); - - assert!(parse_dat_address( - "c7638882870ab").is_err()); - assert!(parse_dat_address( - "g7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_err()); - assert!(parse_dat_address( - "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d5970").is_err()); - assert!(parse_dat_address( - "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d59").is_err()); -} diff --git a/src/peer.rs b/src/peer.rs new file mode 100644 index 0000000..5535b09 --- /dev/null +++ b/src/peer.rs @@ -0,0 +1,12 @@ + +//use errors::*; + +// DatPeer +// connection (thread?) +// last_sent_ts +// last_received_ts +// has_messages +// registers: vec? map? +// +// fn has(reg, u64) +// fn has_intersection(reg, bitfield) diff --git a/src/node.rs b/src/synchronizer.rs index 582bf3d..216df6e 100644 --- a/src/node.rs +++ b/src/synchronizer.rs @@ -1,60 +1,29 @@ use errors::*; use network_msgs::*; +use bitfield::*; use protocol::{DatNetMessage, DatConnection}; -use integer_encoding::VarInt; -use bit_field::BitArray; use sleep_register::HyperRegister; -fn decode_bitfiled(raw_bf: &[u8]) -> Result<Vec<u8>> { - let mut offset = 0; // byte offset that we have read up to - if raw_bf.len() < 1 { - bail!("Expected (varint-encoded) bitfield to have len>=1"); - } - let mut bit_array: Vec<u8> = vec![]; - while offset < raw_bf.len() { - let (header, inc): (u64, usize) = VarInt::decode_var(&raw_bf[offset..]); - offset += inc; - - if (header & 0x01) == 0x01 { - // compressed - let bit = (header & 0x02) == 0x02; - let run_len = header >> 2; - if bit { - bit_array.append(&mut vec![0xFF; run_len as usize]); - } else { - bit_array.append(&mut vec![0x00; run_len as usize]); - } - } else { - // uncompressed - let byte_count = header >> 1; - let mut data = raw_bf[offset..(offset + byte_count as usize)].to_vec(); - bit_array.append(&mut data); - offset += byte_count as usize; - } - } - // XXX: HACK - bit_array.reverse(); - return Ok(bit_array); -} +// Synchronizer +// register_keys +// peers: vec +// registers: HyperRegisters +// mode: enum +// state: enum +// wanted: bitfield +// requested: vec +// +// fn next_wanted() -> Option((reg, u64)) +// fn tick() -/// Finds the index of the lowest bit -fn max_high_bit(bf: &[u8]) -> u64 { - // XXX: HACK, going backwards - for i in 0..bf.bit_length() { - if bf.get_bit(i) { - return (bf.bit_length() - i - 1) as u64; - } - } - return 0; -} fn max_index(have_msg: &Have) -> Result<u64> { if have_msg.has_length() { return Ok(have_msg.get_start() + have_msg.get_length()); } else if have_msg.has_bitfield() { let raw_bf = have_msg.get_bitfield(); - let bf = decode_bitfiled(raw_bf)?; + let bf = decode_bitfield(raw_bf)?; trace!("decoded bitfield: {:?}", bf); return Ok(max_high_bit(&bf)); } else { |