DHT Discovery

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     'colmeia-hypercore',
     'colmeia-hyperdrive',
     'colmeia-hyperstack',
+    'colmeia-hyperswarm-dht',
     'colmeia-hyperswarm-mdns',
     'colmeiad',
 ]

README.md

@@ -41,7 +41,10 @@ Uses NOISE protocol and a different handshake.
   - [x] `Mdns`: announces and find dat in the network
   - [ ] Tests
   - [ ] All protocol 1:1
-- [ ] `colmeia-dht`: Interop with hypwerswarm dht infrastructure (:eyes: <https://github.com/mattsse/hyperswarm-dht>)
+- [x] `colmeia-dht`: Interop with hypwerswarm dht infrastructure (:eyes: <https://github.com/mattsse/hyperswarm-dht>)
+  - [x] `Locator`: stream to find dat members in the network
+  - [x] `Announcer`: stream that announces a dat in the network
+  - [x] `Dht`: announces and find dat in the network
   - [ ] Tests
   - [ ] All protocol 1:1
 
@@ -80,24 +83,48 @@ Uses NOISE protocol and a different handshake.
 
 ## CLI binaries
 
+### Server
+
+Start the web service using `colmeiad`:
+
+```sh
+RUST_LOG=debug cargo run -p colmeiad -- b45eca0cdf33195821b94dc3836460065864ebd621c9a05aa8c54698c8b388b6
+```
+
+You be able to get the information about the blocks on `/`
+
+```sh
+curl -v http://localhost:8080
+```
+
+### Dump hash metadata
+
+```sh
+cargo run --bin colmeia-url-meta -- b45eca0cdf33195821b94dc3836460065864ebd621c9a05aa8c54698c8b388b6
+```
+
 ```txt
-Local dat2: be41e3d43d054982e14dfc60281d9d4425ab5d4b0b280a355b7927869ca08fc5
+Valid public key.
+Public Hexa     b45eca0cdf33195821b94dc3836460065864ebd621c9a05aa8c54698c8b388b6
+Public Bytes    [180, 94, 202, 12, 223, 51, 25, 88, 33, 185, 77, 195, 131, 100, 96, 6, 88, 100, 235, 214, 33, 201, 160, 90, 168, 197, 70, 152, 200, 179, 136, 182]
+Discovery Hexa  b172d7304b7df9d539b67ca100a1930ae102111fc253cf784e649231956a9b9c
+Discovery Bytes [177, 114, 215, 48, 75, 125, 249, 213, 57, 182, 124, 161, 0, 161, 147, 10, 225, 2, 17, 31, 194, 83, 207, 120, 78, 100, 146, 49, 149, 106, 155, 156]
 ```
 
 ### Find a local peer
 
 [colmeia-mdns](./colmeia-bins/src/bin/colmeia-hyperswarm-mdns.rs)
 
 ```sh
-RUST_LOG=debug cargo run --bin colmeia-hyperswarm-mdns -- 7e5998407b3d9dbb94db21ff50ad6f1b1d2c79e476fbaf9856c342eb4382e7f5 8000
+RUST_LOG=debug cargo run --bin colmeia-hyperswarm-mdns -- b45eca0cdf33195821b94dc3836460065864ebd621c9a05aa8c54698c8b388b6 8000
 ```
 
 ### Connect and troubleshoot a peer
 
 [colmeia-nc](./colmeia-bins/src/bin/colmeia-nc.rs)
 
 ```sh
-RUST_LOG=debug cargo run --bin colmeia-nc -- 7e5998407b3d9dbb94db21ff50ad6f1b1d2c79e476fbaf9856c342eb4382e7f5 127.0.0.1:3282
+RUST_LOG=debug cargo run --bin colmeia-nc -- b45eca0cdf33195821b94dc3836460065864ebd621c9a05aa8c54698c8b388b6 127.0.0.1:3282
 ```
 
 ### Clone from a single host
@@ -106,6 +133,20 @@ RUST_LOG=debug cargo run --bin colmeia-nc -- 7e5998407b3d9dbb94db21ff50ad6f1b1d2
 RUST_LOG=debug cargo run --bin colmeia-clone -- 192.168.15.173:3282 dat://6268b99fbacacea49c6bc3d4776b606db2aeadb3fa831342ba9f70d55c98929f
 ```
 
+### Test discovery mechanisms
+
+#### MDNS
+
+```sh
+RUST_LOG=debug cargo run --bin colmeia-hyperswarm-mdns -- b45eca0cdf33195821b94dc3836460065864ebd621c9a05aa8c54698c8b388b6 8000
+```
+
+#### DHT
+
+```sh
+RUST_LOG=debug cargo run --bin colmeia-hyperswarm-dht -- b45eca0cdf33195821b94dc3836460065864ebd621c9a05aa8c54698c8b388b6
+```
+
 ## Platforms
 
 :warning: **TODO**: redo support as part of dat -> hypercore migration
@@ -193,3 +234,8 @@ Run and `scp` the binaries on `target/mips-unknown-linux-musl`:
 ```sh
  cross build --target mips-unknown-linux-musl
  ```
+
+## Other hypercores to test
+
+- `94f0cab7f60fcc2a711df11c85db5e0594d11e8a3efd04a06f46a3c34d03c418`
+- `b282d5efe484143816362d33b3f9b3ea45ecfb8a6ada97e278fdfdc6a725e22f`

colmeia-bins/Cargo.toml

@@ -8,16 +8,24 @@ edition = '2018'
 env_logger = '*'
 log = '*'
 hex = '*'
+ed25519-dalek = '*'
 
 [dependencies.async-std]
-version = '1.6.2'
+version = '1.9.0'
 features = ['unstable']
 
 [dependencies.hypercore-protocol]
 git = 'https://github.com/Frando/hypercore-protocol-rs'
 
+[dependencies.hyperswarm-dht]
+git = 'https://github.com/bltavares/hyperswarm-dht'
+branch = 'wip'
+
 [dependencies.colmeia-hyperswarm-mdns]
 path = '../colmeia-hyperswarm-mdns'
 
+[dependencies.colmeia-hyperswarm-dht]
+path = '../colmeia-hyperswarm-dht'
+
 [dependencies.colmeia-hyperstack]
 path = '../colmeia-hyperstack'

colmeia-bins/src/bin/colmeia-clone.rs

@@ -24,7 +24,7 @@ fn address() -> SocketAddr {
 // }
 
 fn main() {
-    env_logger::init();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
 
     let key = name();
     let address = address();

colmeia-bins/src/bin/colmeia-hyperswarm-dht.rs

@@ -0,0 +1,74 @@
+use async_std::{prelude::StreamExt, task};
+use colmeia_hyperstack::utils::PublicKeyExt;
+use colmeia_hyperswarm_dht::{dht::Config, DHTDiscovery};
+use std::{env, time::Duration};
+
+// 7e5998407b3d9dbb94db21ff50ad6f1b1d2c79e476fbaf9856c342eb4382e7f5
+// pra
+// b62ef6792a0e2f2c5be328593532415d80c0f52c // 32 bytes
+// Derivacao igual ao dat1
+fn main() {
+    log::debug!("Starting up");
+    let hyper_hash = env::args()
+        .nth(1)
+        .expect("provide a hyper hash as argument");
+
+    let bootstrap_servers = env::args()
+        .nth(2)
+        .map(|x| x.split(',').map(str::to_owned).collect())
+        .unwrap_or_else(|| {
+            hyperswarm_dht::DEFAULT_BOOTSTRAP
+                .iter()
+                .map(|&x| x.to_owned())
+                .collect()
+        });
+
+    let duration = env::args()
+        .nth(3)
+        .unwrap_or_else(|| "10".into())
+        .parse::<u64>()
+        .expect("Could not parse duration into a number");
+    let duration = Duration::from_secs(duration);
+
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+
+    let key = hyper_hash.parse_from_hash().expect("could not parse hash");
+
+    let config = Config {
+        bootstrap_servers,
+        ..Default::default()
+    };
+
+    task::block_on(async move {
+        log::info!("Starting up");
+
+        let topic = hypercore_protocol::discovery_key(key.as_bytes());
+
+        let swarm = DHTDiscovery::listen(config).await;
+        let mut swarm = swarm.expect("Could not bind socket");
+        swarm
+            // .with_announcer(2323, duration)
+            .with_locator(duration);
+
+        log::info!("Adding topic");
+        swarm
+            .add_topic(&topic)
+            .await
+            .expect("could not write topic");
+
+        log::info!("Finding topic");
+        let result = swarm.next().await;
+        log::info!("Found peer: {:?}", result);
+
+        log::info!("Remove topic");
+        swarm
+            .remove_topic(&topic)
+            .await
+            .expect("could not remove topic");
+
+        log::info!("Sleep");
+        task::sleep(Duration::from_secs(60)).await;
+
+        log::info!("Done");
+    });
+}

colmeia-bins/src/bin/colmeia-hyperswarm-mdns.rs

@@ -25,7 +25,7 @@ fn main() {
         .parse::<u64>()
         .expect("Could not parse duration into a number");
 
-    env_logger::init();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
 
     let key = hyper_hash.parse_from_hash().expect("could not parse hash");
 

colmeia-bins/src/bin/colmeia-nc.rs

@@ -20,7 +20,7 @@ fn main() {
     // 1. receive ip:port as argument (eg: hashs and 192.168.137.1:58906)
     // 2. try to connect and hadhsake (with hypercore-protocol) (eg: 7e5998407b3d9dbb94db21ff50ad6f1b1d2c79e476fbaf9856c342eb4382e7f5)
     // 3. Finalize with all working
-    env_logger::init();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
 
     let key = name();
     let address = address();

colmeia-bins/src/bin/colmeia-sync.rs

@@ -1,4 +1,4 @@
-use async_std::task;
+use async_std::{prelude::StreamExt, task};
 use colmeia_hyperstack::{utils::PublicKeyExt, Hyperstack};
 
 fn name() -> String {
@@ -14,7 +14,7 @@ fn name() -> String {
 // }
 
 fn main() {
-    env_logger::init();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
 
     let key = name();
     let hash = key.parse_from_hash().expect("invalid dat argument");
@@ -25,7 +25,8 @@ fn main() {
             .await
             .expect("Could not start hyperdrive on the stack");
         let mdns = hyperstack.lan().await.expect("could not configure mdns");
-        hyperstack.with_discovery(mdns);
+        let dht = hyperstack.dht().await.expect("could not start dht");
+        hyperstack.with_discovery(mdns.merge(dht));
         hyperstack.replicate().await;
     });
 }

colmeia-bins/src/bin/colmeia-url-meta.rs

@@ -0,0 +1,44 @@
+use std::env;
+
+use colmeia_hyperstack::utils::{HashParserError, PublicKeyExt};
+use ed25519_dalek::PublicKey;
+
+fn main() {
+    let hyper_hash = env::args()
+        .nth(1)
+        .expect("provide a hyper hash as argument");
+
+    let key = hyper_hash.parse_from_hash();
+
+    match key {
+        Ok(public) => {
+            valid_public_key(&public);
+        }
+        Err(e) => {
+            invalid_public_key(e);
+        }
+    }
+}
+
+fn invalid_public_key(e: HashParserError) {
+    match e {
+        HashParserError::InvalidHexHash(_) => {
+            eprintln!("Invalid hyper url");
+        }
+        HashParserError::InvalidPublicKey(_) => {
+            eprintln!("Invalid public key. This is likely the discovery key.");
+        }
+    }
+}
+
+fn valid_public_key(public: &PublicKey) {
+    println!("Valid public key.");
+    let bytes = public.as_bytes();
+    println!("Public Hexa\t{}", hex::encode(bytes));
+    println!("Public Bytes\t{:?}", bytes);
+
+    let topic = hypercore_protocol::discovery_key(bytes);
+
+    println!("Discovery Hexa\t{}", hex::encode(&topic));
+    println!("Discovery Bytes\t{:?}", topic);
+}

colmeia-bins/src/bin/dht-bootstrap.rs

@@ -0,0 +1,22 @@
+use async_std::{prelude::StreamExt, task};
+use hyperswarm_dht::{DhtConfig, HyperDht};
+
+fn main() {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+
+    task::block_on(async move {
+        let config = DhtConfig::default().ephemeral().empty_bootstrap_nodes();
+        let mut bootstrap = HyperDht::with_config(config)
+            .await
+            .expect("Could not start service");
+
+        log::info!(
+            "Bootstrap server running on: {}",
+            bootstrap.local_addr().expect("could not get local address")
+        );
+
+        while let Some(event) = bootstrap.next().await {
+            log::debug!("Event: {:?}", event);
+        }
+    });
+}

colmeia-hypercore/Cargo.toml

@@ -5,16 +5,19 @@ authors = ['Bruno Tavares <[email protected]>']
 edition = '2018'
 
 [dependencies]
-futures = '0.3.5'
-hypercore = '0.11.1-beta.9'
-anyhow = '1.0.34'
+futures = '0.3.12'
+anyhow = '1.0.38'
 random-access-storage = '4.0.0'
-log = '0.4.8'
+log = '0.4.14'
 bitfield-rle = '0.2.0'
 
 [dependencies.async-std]
-version = '1.6.2'
+version = '1.9.0'
 features = ['unstable']
 
 [dependencies.hypercore-protocol]
 git = 'https://github.com/Frando/hypercore-protocol-rs'
+
+[dependencies.hypercore]
+git = 'https://github.com/bltavares/hypercore'
+branch = 'update-ed25519-dalek'