aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--README.md7
-rw-r--r--TODO7
-rw-r--r--src/bitfield.rs55
-rw-r--r--src/helpers.rs61
-rw-r--r--src/lib.rs70
-rw-r--r--src/peer.rs12
-rw-r--r--src/synchronizer.rs (renamed from src/node.rs)57
7 files changed, 156 insertions, 113 deletions
diff --git a/README.md b/README.md
index e6a5e46..654614a 100644
--- a/README.md
+++ b/README.md
@@ -30,15 +30,16 @@ it will eat-your-data!
- [x] import file to register
- [x] export file from register
- [x] import/export directories recursively
-- [ ] Networking
+- [ ] Protocol
- [x] send/receive encrypted messages to a known host
- - [ ] receive entire register from a known host
- - [ ] share (upload) register to a known host
- [ ] bitfields
- [ ] Discovery
- [x] centralized DNS
- [ ] mDNS (local DNS)
- [ ] DHT (distributed hash table)
+- [ ] Peer Synchronization
+ - [ ] receive entire register from a known host
+ - [ ] share (upload) register to a known host
- [ ] Wrapper commands
- [ ] clone
- [ ] share
diff --git a/TODO b/TODO
index f1ccb08..c702002 100644
--- a/TODO
+++ b/TODO
@@ -1,11 +1,10 @@
next:
+- DatPeer wrapper API
+- Synchronizer API
- clean up node bitfield implementation
- "insert" API for HyperRegister
-- crude single-thread drive+network clone
- sparse register clone
-- "needed data chunks" and progress function?
-- crude single-thread drive+network broadcast
- 'geniza clone' command
- 'geniza checkout' command
- 'geniza init', 'geniza commit' commands
@@ -28,7 +27,7 @@ Backburner:
- API to link and run from, eg, python
- uTP transport
- in-memory storage backend
-- WASM
+- compile to WASM... play in browser?
- multi-connection network sync (per-connection state, etc)
- duplicate file/chunk optimizations
- secret_key-in-home helpers (read/write)
diff --git a/src/bitfield.rs b/src/bitfield.rs
new file mode 100644
index 0000000..11640ea
--- /dev/null
+++ b/src/bitfield.rs
@@ -0,0 +1,55 @@
+
+use errors::*;
+use integer_encoding::VarInt;
+use bit_field::BitArray;
+
+
+// WrappedBitfield
+//
+// uses vec of u64 internally?
+//
+// fn from_message()
+// fn get(u64)
+
+pub fn decode_bitfield(raw_bf: &[u8]) -> Result<Vec<u8>> {
+ let mut offset = 0; // byte offset that we have read up to
+ if raw_bf.len() < 1 {
+ bail!("Expected (varint-encoded) bitfield to have len>=1");
+ }
+ let mut bit_array: Vec<u8> = vec![];
+ while offset < raw_bf.len() {
+ let (header, inc): (u64, usize) = VarInt::decode_var(&raw_bf[offset..]);
+ offset += inc;
+
+ if (header & 0x01) == 0x01 {
+ // compressed
+ let bit = (header & 0x02) == 0x02;
+ let run_len = header >> 2;
+ if bit {
+ bit_array.append(&mut vec![0xFF; run_len as usize]);
+ } else {
+ bit_array.append(&mut vec![0x00; run_len as usize]);
+ }
+ } else {
+ // uncompressed
+ let byte_count = header >> 1;
+ let mut data = raw_bf[offset..(offset + byte_count as usize)].to_vec();
+ bit_array.append(&mut data);
+ offset += byte_count as usize;
+ }
+ }
+ // XXX: HACK
+ bit_array.reverse();
+ return Ok(bit_array);
+}
+
+/// Finds the index of the lowest bit
+pub fn max_high_bit(bf: &[u8]) -> u64 {
+ // XXX: HACK, going backwards
+ for i in 0..bf.bit_length() {
+ if bf.get_bit(i) {
+ return (bf.bit_length() - i - 1) as u64;
+ }
+ }
+ return 0;
+}
diff --git a/src/helpers.rs b/src/helpers.rs
new file mode 100644
index 0000000..32f0e25
--- /dev/null
+++ b/src/helpers.rs
@@ -0,0 +1,61 @@
+
+use errors::*;
+use crypto::digest::Digest;
+use crypto::blake2b::Blake2b;
+
+/// Helper to calculate a discovery key from a public key. 'key' should be 32 bytes; the returned
+/// array will also be 32 bytes long.
+///
+/// dat discovery keys are calculated as a BLAKE2b "keyed hash" (using the passed key) of the string
+/// "hypercore" (with no trailing null byte).
+pub fn make_discovery_key(key: &[u8]) -> Vec<u8> {
+ let mut discovery_key = [0; 32];
+ let mut hash = Blake2b::new_keyed(32, key);
+ hash.input(&"hypercore".as_bytes());
+ hash.result(&mut discovery_key);
+ discovery_key.to_vec()
+}
+
+/// Helper to parse a dat address (aka, public key) in string format.
+///
+/// Address can start with 'dat://'. It should contain 64 hexadecimal characters.
+pub fn parse_dat_address(input: &str) -> Result<Vec<u8>> {
+
+ let raw_key = if input.starts_with("dat://") {
+ &input[6..]
+ } else {
+ input
+ };
+ if raw_key.len() != 32 * 2 {
+ bail!("dat key not correct length");
+ }
+ let mut key_bytes = vec![];
+ for i in 0..32 {
+ let r = u8::from_str_radix(&raw_key[2 * i..2 * i + 2], 16);
+ match r {
+ Ok(b) => key_bytes.push(b),
+ Err(e) => bail!("Problem with hex: {}", e),
+ };
+ }
+ Ok(key_bytes)
+}
+
+#[test]
+fn test_parse_dat_address() {
+
+ assert!(parse_dat_address(
+ "c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_ok());
+ assert!(parse_dat_address(
+ "C7638882870ABD4044D6467B0738F15E3A36F57C3A7F7F3417FD7E4E0841D597").is_ok());
+ assert!(parse_dat_address(
+ "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_ok());
+
+ assert!(parse_dat_address(
+ "c7638882870ab").is_err());
+ assert!(parse_dat_address(
+ "g7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_err());
+ assert!(parse_dat_address(
+ "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d5970").is_err());
+ assert!(parse_dat_address(
+ "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d59").is_err());
+}
diff --git a/src/lib.rs b/src/lib.rs
index 63000f7..e3ad197 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -46,6 +46,10 @@ mod errors {
pub use errors::*;
// Organize code internally (files, modules), but pull it all into a flat namespace to export.
+mod helpers;
+pub use helpers::*;
+mod bitfield;
+pub use bitfield::*;
mod sleep_file;
pub use sleep_file::*;
mod sleep_register;
@@ -56,68 +60,10 @@ mod protocol;
pub use protocol::*;
pub mod network_msgs;
pub mod metadata_msgs;
-mod node;
-pub use node::*;
mod discovery;
pub use discovery::*;
+mod peer;
+pub use peer::*;
+mod synchronizer;
+pub use synchronizer::*;
-// Shared functions
-use crypto::digest::Digest;
-use crypto::blake2b::Blake2b;
-
-/// Helper to calculate a discovery key from a public key. 'key' should be 32 bytes; the returned
-/// array will also be 32 bytes long.
-///
-/// dat discovery keys are calculated as a BLAKE2b "keyed hash" (using the passed key) of the string
-/// "hypercore" (with no trailing null byte).
-pub fn make_discovery_key(key: &[u8]) -> Vec<u8> {
- let mut discovery_key = [0; 32];
- let mut hash = Blake2b::new_keyed(32, key);
- hash.input(&"hypercore".as_bytes());
- hash.result(&mut discovery_key);
- discovery_key.to_vec()
-}
-
-/// Helper to parse a dat address (aka, public key) in string format.
-///
-/// Address can start with 'dat://'. It should contain 64 hexadecimal characters.
-pub fn parse_dat_address(input: &str) -> Result<Vec<u8>> {
-
- let raw_key = if input.starts_with("dat://") {
- &input[6..]
- } else {
- input
- };
- if raw_key.len() != 32 * 2 {
- bail!("dat key not correct length");
- }
- let mut key_bytes = vec![];
- for i in 0..32 {
- let r = u8::from_str_radix(&raw_key[2 * i..2 * i + 2], 16);
- match r {
- Ok(b) => key_bytes.push(b),
- Err(e) => bail!("Problem with hex: {}", e),
- };
- }
- Ok(key_bytes)
-}
-
-#[test]
-fn test_parse_dat_address() {
-
- assert!(parse_dat_address(
- "c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_ok());
- assert!(parse_dat_address(
- "C7638882870ABD4044D6467B0738F15E3A36F57C3A7F7F3417FD7E4E0841D597").is_ok());
- assert!(parse_dat_address(
- "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_ok());
-
- assert!(parse_dat_address(
- "c7638882870ab").is_err());
- assert!(parse_dat_address(
- "g7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d597").is_err());
- assert!(parse_dat_address(
- "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d5970").is_err());
- assert!(parse_dat_address(
- "dat://c7638882870abd4044d6467b0738f15e3a36f57c3a7f7f3417fd7e4e0841d59").is_err());
-}
diff --git a/src/peer.rs b/src/peer.rs
new file mode 100644
index 0000000..5535b09
--- /dev/null
+++ b/src/peer.rs
@@ -0,0 +1,12 @@
+
+//use errors::*;
+
+// DatPeer
+// connection (thread?)
+// last_sent_ts
+// last_received_ts
+// has_messages
+// registers: vec? map?
+//
+// fn has(reg, u64)
+// fn has_intersection(reg, bitfield)
diff --git a/src/node.rs b/src/synchronizer.rs
index 582bf3d..216df6e 100644
--- a/src/node.rs
+++ b/src/synchronizer.rs
@@ -1,60 +1,29 @@
use errors::*;
use network_msgs::*;
+use bitfield::*;
use protocol::{DatNetMessage, DatConnection};
-use integer_encoding::VarInt;
-use bit_field::BitArray;
use sleep_register::HyperRegister;
-fn decode_bitfiled(raw_bf: &[u8]) -> Result<Vec<u8>> {
- let mut offset = 0; // byte offset that we have read up to
- if raw_bf.len() < 1 {
- bail!("Expected (varint-encoded) bitfield to have len>=1");
- }
- let mut bit_array: Vec<u8> = vec![];
- while offset < raw_bf.len() {
- let (header, inc): (u64, usize) = VarInt::decode_var(&raw_bf[offset..]);
- offset += inc;
-
- if (header & 0x01) == 0x01 {
- // compressed
- let bit = (header & 0x02) == 0x02;
- let run_len = header >> 2;
- if bit {
- bit_array.append(&mut vec![0xFF; run_len as usize]);
- } else {
- bit_array.append(&mut vec![0x00; run_len as usize]);
- }
- } else {
- // uncompressed
- let byte_count = header >> 1;
- let mut data = raw_bf[offset..(offset + byte_count as usize)].to_vec();
- bit_array.append(&mut data);
- offset += byte_count as usize;
- }
- }
- // XXX: HACK
- bit_array.reverse();
- return Ok(bit_array);
-}
+// Synchronizer
+// register_keys
+// peers: vec
+// registers: HyperRegisters
+// mode: enum
+// state: enum
+// wanted: bitfield
+// requested: vec
+//
+// fn next_wanted() -> Option((reg, u64))
+// fn tick()
-/// Finds the index of the lowest bit
-fn max_high_bit(bf: &[u8]) -> u64 {
- // XXX: HACK, going backwards
- for i in 0..bf.bit_length() {
- if bf.get_bit(i) {
- return (bf.bit_length() - i - 1) as u64;
- }
- }
- return 0;
-}
fn max_index(have_msg: &Have) -> Result<u64> {
if have_msg.has_length() {
return Ok(have_msg.get_start() + have_msg.get_length());
} else if have_msg.has_bitfield() {
let raw_bf = have_msg.get_bitfield();
- let bf = decode_bitfiled(raw_bf)?;
+ let bf = decode_bitfield(raw_bf)?;
trace!("decoded bitfield: {:?}", bf);
return Ok(max_high_bit(&bf));
} else {