From 79580699516c2094107a7c76131133dcff365b27 Mon Sep 17 00:00:00 2001 From: Alexander Simmerl Date: Tue, 14 Sep 2021 23:23:23 +1000 Subject: [PATCH 1/6] librad: Implement Display for Network This comes in handy in places where `Network` is used as input for parameterisation and therefore needs to be displayed or otherwise stringly represented. Signed-off-by: Alexander Simmerl --- librad/src/net.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/librad/src/net.rs b/librad/src/net.rs index 830d1c85f..c68a284ee 100644 --- a/librad/src/net.rs +++ b/librad/src/net.rs @@ -3,7 +3,7 @@ // This file is part of radicle-link, distributed under the GPLv3 with Radicle // Linking Exception. For full terms see the included LICENSE file. -use std::{borrow::Cow, str::FromStr}; +use std::{borrow::Cow, fmt::Display, str::FromStr}; pub mod codec; pub mod connection; @@ -43,7 +43,7 @@ pub const PROTOCOL_VERSION: u8 = 2; /// should be kept short. /// /// [ALPN]: https://tools.ietf.org/html/rfc7301 -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum Network { Main, Custom(Cow<'static, [u8]>), @@ -55,6 +55,12 @@ impl Default for Network { } } +impl Display for Network { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + impl FromStr for Network { type Err = &'static str; From 624f29e533f722e683ce6ab4603e17055a956647 Mon Sep 17 00:00:00 2001 From: Alexander Simmerl Date: Tue, 14 Sep 2021 23:25:55 +1000 Subject: [PATCH 2/6] librad: Replace RadHome's new with default It expresses the behaviour more correctly as the implementation of new is really about finding a sensible default in the presenece of env vars and other factors. Signed-off-by: Alexander Simmerl --- librad/src/profile.rs | 20 ++++++++++++-------- rad-profile/src/lib.rs | 14 +++++++------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/librad/src/profile.rs b/librad/src/profile.rs index 5470e8e81..dba1a22dc 100644 --- a/librad/src/profile.rs +++ b/librad/src/profile.rs @@ -5,6 +5,7 @@ use std::{ env, + fmt, io, path::{Path, PathBuf}, }; @@ -37,6 +38,7 @@ pub struct Profile { } /// An enumeration of where the root directory for a `Profile` lives. +#[derive(Clone, Debug, Eq, PartialEq)] pub enum RadHome { /// The system specific directories given by [`directories::ProjectDirs`]. ProjectDirs, @@ -45,23 +47,25 @@ pub enum RadHome { } impl Default for RadHome { - fn default() -> Self { - Self::new() - } -} - -impl RadHome { /// If `RAD_HOME` is defined then the path supplied there is used and /// [`RadHome::Root`] is constructed. Otherwise, [`RadHome::ProjectDirs`] is /// constructed. - pub fn new() -> Self { + fn default() -> Self { if let Ok(root) = env::var(RAD_HOME) { Self::Root(Path::new(&root).to_path_buf()) } else { Self::ProjectDirs } } +} +impl fmt::Display for RadHome { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl RadHome { fn config(&self) -> Result { Ok(match self { Self::ProjectDirs => project_dirs()?.config_dir().to_path_buf(), @@ -151,7 +155,7 @@ impl Profile { /// `ProjectDirs_DATA_HOME/radicle-link/`. pub fn load() -> Result { let env_profile_id = ProfileId::from_env()?; - let home = RadHome::new(); + let home = RadHome::default(); Self::from_home(&home, env_profile_id) } diff --git a/rad-profile/src/lib.rs b/rad-profile/src/lib.rs index c240e0de8..4d0a990c1 100644 --- a/rad-profile/src/lib.rs +++ b/rad-profile/src/lib.rs @@ -69,7 +69,7 @@ where C::Error: fmt::Debug + fmt::Display + Send + Sync + 'static, C::SecretBox: Serialize + DeserializeOwned, { - let home = RadHome::new(); + let home = RadHome::default(); let profile = Profile::new(&home)?; Profile::set(&home, profile.id().clone())?; let key = SecretKey::new(); @@ -82,7 +82,7 @@ where /// Get the current active `ProfileId`. pub fn get(id: Option) -> Result, Error> { - let home = RadHome::new(); + let home = RadHome::default(); match id { Some(id) => Profile::get(&home, id).map_err(Error::from), None => Profile::active(&home).map_err(Error::from), @@ -91,13 +91,13 @@ pub fn get(id: Option) -> Result, Error> { /// Set the active profile to the given `ProfileId`. pub fn set(id: ProfileId) -> Result<(), Error> { - let home = RadHome::new(); + let home = RadHome::default(); Profile::set(&home, id).map_err(Error::from).map(|_| ()) } /// List the set of active profiles that exist. pub fn list() -> Result, Error> { - let home = RadHome::new(); + let home = RadHome::default(); Profile::list(&home).map_err(Error::from) } @@ -106,7 +106,7 @@ pub fn peer_id

(id: P) -> Result where P: Into>, { - let home = RadHome::new(); + let home = RadHome::default(); let profile = get_or_active(&home, id)?; let read = ReadOnly::open(profile.paths())?; Ok(*read.peer_id()) @@ -116,7 +116,7 @@ pub fn paths

(id: P) -> Result where P: Into>, { - let home = RadHome::new(); + let home = RadHome::default(); get_or_active(&home, id).map(|p| p.paths().clone()) } @@ -133,7 +133,7 @@ where P: Into>, S: ClientStream + Unpin + 'static, { - let home = RadHome::new(); + let home = RadHome::default(); let profile = get_or_active(&home, id)?; let store = keys::file_storage(&profile, crypto); let key = store.get_key()?; From 4ef6f640bc89d5310d9f7eb5ad0852bacbc19919 Mon Sep 17 00:00:00 2001 From: Alexander Simmerl Date: Tue, 14 Sep 2021 23:27:40 +1000 Subject: [PATCH 3/6] rad-clib: Avoid unwrap with transparent error Will give callers more control over the handling of that error case, the author expects this to be an oversight of the original implementation. Signed-off-by: Alexander Simmerl --- rad-clib/src/keys.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rad-clib/src/keys.rs b/rad-clib/src/keys.rs index 6b59aa3df..6a4256d91 100644 --- a/rad-clib/src/keys.rs +++ b/rad-clib/src/keys.rs @@ -22,7 +22,7 @@ use librad::{ IntoSecretKeyError, SomeSigner, }, - git::storage::ReadOnly, + git::storage::{self, ReadOnly}, profile::Profile, PublicKey, SecretKey, @@ -37,6 +37,8 @@ pub enum Error { File(#[from] file::Error, IntoSecretKeyError>), #[error(transparent)] SshConnect(#[from] ssh::error::Connect), + #[error(transparent)] + StorageInit(#[from] storage::read::error::Init), } /// Create a [`Prompt`] for unlocking the key storage. @@ -79,7 +81,7 @@ pub async fn signer_ssh(profile: &Profile) -> Result where S: ClientStream + Unpin + 'static, { - let storage = ReadOnly::open(profile.paths()).unwrap(); + let storage = ReadOnly::open(profile.paths())?; let peer_id = storage.peer_id(); let agent = SshAgent::new((**peer_id).into()); let signer = agent.connect::().await?; From b9eeadd66adfc5ea0156cd9f568cea6a011024bc Mon Sep 17 00:00:00 2001 From: Alexander Simmerl Date: Tue, 14 Sep 2021 23:30:23 +1000 Subject: [PATCH 4/6] node: Add library to wire up a running p2p node First iterative implementation of RFC 0696 focusing on driving the core peer protocol and establishing the harness for configuration of a running node. Rite of passage for it will be replacement of the ephemeral peer in the e2e networkss, as that requires a well behaved peer with a set of knobs exposed. Structurally the majority of the implementation lives in a library crate node-lib. The code is rather Mario-esque as it is primarily plumbing of code that either existed in other crates backing binaries or newish code doing the same thing for core pieces. There are still large pieces missing, which a tracked in #722 and will be piled on-top as patches to keep this already big delta focused. The declared initial goal is to get linkd into a state where it can run as bootstrap/seed node. Signed-off-by: Alexander Simmerl --- Cargo.toml | 1 + node-lib/Cargo.toml | 40 ++ node-lib/src/args.rs | 323 +++++++++++++++ node-lib/src/cfg.rs | 205 ++++++++++ node-lib/src/cfg/seed.rs | 75 ++++ node-lib/src/lib.rs | 18 + node-lib/src/logging.rs | 49 +++ node-lib/src/metrics.rs | 6 + node-lib/src/metrics/graphite.rs | 60 +++ node-lib/src/node.rs | 82 ++++ node-lib/src/protocol.rs | 72 ++++ node-lib/src/signals.rs | 46 +++ node-lib/src/socket_activation.rs | 29 ++ node-lib/src/socket_activation/macos.rs | 12 + node-lib/src/socket_activation/unix.rs | 67 ++++ test/Cargo.toml | 6 + test/examples/socket_activation.rs | 18 + test/examples/socket_activation_wrapper.rs | 34 ++ test/src/test/integration.rs | 1 + test/src/test/integration/node_lib.rs | 6 + .../integration/node_lib/socket_activation.rs | 10 + .../node_lib/socket_activation/macos.rs | 4 + .../node_lib/socket_activation/unix.rs | 22 ++ test/src/test/unit.rs | 1 + test/src/test/unit/node_lib.rs | 7 + test/src/test/unit/node_lib/args.rs | 373 ++++++++++++++++++ test/src/test/unit/node_lib/cfg.rs | 35 ++ 27 files changed, 1602 insertions(+) create mode 100644 node-lib/Cargo.toml create mode 100644 node-lib/src/args.rs create mode 100644 node-lib/src/cfg.rs create mode 100644 node-lib/src/cfg/seed.rs create mode 100644 node-lib/src/lib.rs create mode 100644 node-lib/src/logging.rs create mode 100644 node-lib/src/metrics.rs create mode 100644 node-lib/src/metrics/graphite.rs create mode 100644 node-lib/src/node.rs create mode 100644 node-lib/src/protocol.rs create mode 100644 node-lib/src/signals.rs create mode 100644 node-lib/src/socket_activation.rs create mode 100644 node-lib/src/socket_activation/macos.rs create mode 100644 node-lib/src/socket_activation/unix.rs create mode 100644 test/examples/socket_activation.rs create mode 100644 test/examples/socket_activation_wrapper.rs create mode 100644 test/src/test/integration/node_lib.rs create mode 100644 test/src/test/integration/node_lib/socket_activation.rs create mode 100644 test/src/test/integration/node_lib/socket_activation/macos.rs create mode 100644 test/src/test/integration/node_lib/socket_activation/unix.rs create mode 100644 test/src/test/unit/node_lib.rs create mode 100644 test/src/test/unit/node_lib/args.rs create mode 100644 test/src/test/unit/node_lib/cfg.rs diff --git a/Cargo.toml b/Cargo.toml index e2cfad616..15414818e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "link-git-protocol", "link-identities", "macros", + "node-lib", "rad-clib", "rad-exe", "rad-profile", diff --git a/node-lib/Cargo.toml b/node-lib/Cargo.toml new file mode 100644 index 000000000..35d6596bb --- /dev/null +++ b/node-lib/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "node-lib" +version = "0.1.0" +edition = "2018" +license = "GPL-3.0-or-later" +authors = [ + "xla ", +] + +[lib] +doctest = true +test = false + +[dependencies] +anyhow = "1.0" +base64 = "0.13" +env_logger = "0.9" +futures = "0.3" +lazy_static = "1.4" +log = "0.4" +nix = "0.22" +structopt = { version = "0.3", default-features = false } +thiserror = "1.0" +tempfile = "3.2" +tokio = { version = "1.10", default-features = false, features = [ "fs", "io-std", "macros", "process", "rt-multi-thread", "signal" ] } +tracing = { version = "0.1", default-features = false, features = [ "attributes", "std" ] } +tracing-subscriber = "0.2" + +[dependencies.rad-clib] +path = "../rad-clib" +version = "0.1.0" + +[dependencies.librad] +path = "../librad" +version = "0.1.0" + +[dependencies.thrussh-agent] +git = "https://github.com/FintanH/thrussh" +branch = "generic-agent" +features = [ "tokio-agent" ] diff --git a/node-lib/src/args.rs b/node-lib/src/args.rs new file mode 100644 index 000000000..8b7fb31cc --- /dev/null +++ b/node-lib/src/args.rs @@ -0,0 +1,323 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +// TODO(xla): Expose discovery args. +// TODO(xla): Expose storage args. +// TODO(xla): Expose logging args. + +use std::{fmt, net::SocketAddr, path::PathBuf, str::FromStr}; + +use structopt::StructOpt; + +use librad::{ + crypto, + net::Network, + profile::{ProfileId, RadHome}, + PeerId, +}; + +#[derive(Debug, Default, Eq, PartialEq, StructOpt)] +pub struct Args { + /// List of bootstrap nodes for initial discovery. + #[structopt(long = "bootstrap", name = "bootstrap")] + pub bootstraps: Vec, + + /// Identifier of the profile the daemon will run for. This value determines + /// which monorepo (if existing) on disk will be the backing storage. + #[structopt(long)] + pub profile_id: Option, + + /// Home of the profile data, if not provided is read from the environment + /// and falls back to project dirs. + #[structopt(long, default_value, parse(from_str = parse_rad_home))] + pub rad_home: RadHome, + + /// Configures the type of signer used to get access to the storage. + #[structopt(long, default_value)] + pub signer: Signer, + + #[structopt(flatten)] + pub key: KeyArgs, + + #[structopt(flatten)] + pub metrics: MetricsArgs, + + #[structopt(flatten)] + pub protocol: ProtocolArgs, + + /// Forces the creation of a temporary root for the local state, should be + /// used for debug and testing only. + #[structopt(long)] + pub tmp_root: bool, +} + +#[derive(Debug, Eq, PartialEq)] +pub struct Bootstrap { + pub addr: String, + pub peer_id: PeerId, +} + +impl fmt::Display for Bootstrap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}@{}", self.peer_id, self.addr) + } +} + +impl FromStr for Bootstrap { + type Err = String; + + fn from_str(src: &str) -> Result { + match src.split_once('@') { + Some((peer_id, addr)) => { + let peer_id = peer_id + .parse() + .map_err(|e: crypto::peer::conversion::Error| e.to_string())?; + + Ok(Self { + addr: addr.to_string(), + peer_id, + }) + }, + None => Err("missing peer id".to_string()), + } + } +} + +#[derive(Debug, Eq, PartialEq, StructOpt)] +pub enum Signer { + /// Construct signer from a secret key. + Key, + /// Connect to ssh-agent for delegated signing. + SshAgent, +} + +impl Default for Signer { + fn default() -> Self { + Self::SshAgent + } +} + +impl fmt::Display for Signer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let ty = match self { + Self::Key => "key", + Self::SshAgent => "ssh-agent", + }; + + write!(f, "{}", ty) + } +} + +impl FromStr for Signer { + type Err = String; + + fn from_str(input: &str) -> Result { + match input { + "key" => Ok(Self::Key), + "ssh-agent" => Ok(Self::SshAgent), + _ => Err(format!("unsupported signer `{}`", input)), + } + } +} + +#[derive(Debug, Default, Eq, PartialEq, StructOpt)] +pub struct KeyArgs { + /// Location of the key file on disk. + #[structopt( + long = "key-file-path", + name = "key-file-path", + parse(from_str), + required_if("key-source", "file") + )] + pub file_path: Option, + /// Format of the key input data. + #[structopt( + long = "key-format", + name = "key-format", + default_value, + required_if("signer", "key") + )] + pub format: KeyFormat, + /// Specifies from which source the secret should be read. + #[structopt( + long = "key-source", + name = "key-source", + default_value, + required_if("signer", "key") + )] + pub source: KeySource, +} + +#[derive(Debug, Eq, PartialEq, StructOpt)] +pub enum KeyFormat { + Base64, + Binary, +} + +impl Default for KeyFormat { + fn default() -> Self { + Self::Binary + } +} + +impl fmt::Display for KeyFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let source = match self { + Self::Base64 => "base64", + Self::Binary => "binary", + }; + write!(f, "{}", source) + } +} + +impl FromStr for KeyFormat { + type Err = String; + + fn from_str(input: &str) -> Result { + match input { + "base64" => Ok(Self::Base64), + "binary" => Ok(Self::Binary), + _ => Err(format!("unsupported key format `{}`", input)), + } + } +} + +#[derive(Debug, Eq, PartialEq, StructOpt)] +pub enum KeySource { + Ephemeral, + File, + Stdin, +} + +impl Default for KeySource { + fn default() -> Self { + Self::Stdin + } +} + +impl fmt::Display for KeySource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { + let source = match self { + Self::Ephemeral => "in-memory", + Self::File => "file", + Self::Stdin => "stdin", + }; + write!(f, "{}", source) + } +} + +impl FromStr for KeySource { + type Err = String; + + fn from_str(input: &str) -> Result { + match input { + "ephemeral" => Ok(Self::Ephemeral), + "file" => Ok(Self::File), + "stdin" => Ok(Self::Stdin), + _ => Err(format!("unsupported key source `{}`", input)), + } + } +} + +fn parse_rad_home(src: &str) -> RadHome { + match src { + dirs if dirs == RadHome::ProjectDirs.to_string() => RadHome::ProjectDirs, + _ => RadHome::Root(PathBuf::from(src)), + } +} + +#[derive(Debug, Eq, PartialEq, StructOpt)] +pub struct MetricsArgs { + /// Provider for metrics collection. + #[structopt(long = "metrics-provider", name = "metrics-provider")] + pub provider: Option, + + /// Address of the graphite collector to send stats to. + #[structopt( + long, + default_value = "localhost:2003", + required_if("metrics-provider", "graphite") + )] + pub graphite_addr: String, +} + +impl Default for MetricsArgs { + fn default() -> Self { + Self { + provider: None, + graphite_addr: "localhost:2003".to_string(), + } + } +} + +#[derive(Debug, Eq, PartialEq, StructOpt)] +pub enum MetricsProvider { + Graphite, +} + +impl FromStr for MetricsProvider { + type Err = String; + + fn from_str(input: &str) -> Result { + match input { + "graphite" => Ok(Self::Graphite), + _ => Err(format!("unsupported key source `{}`", input)), + } + } +} + +#[derive(Debug, Default, Eq, PartialEq, StructOpt)] +pub struct ProtocolArgs { + /// Address to bind to for the protocol to accept connections. Must be + /// provided, shortcuts for any (0.0.0.0:0) and localhost (127.0.0.1:0) + /// are valid values. + #[structopt(long = "protocol-listen", name = "protocol-listen", parse(try_from_str = ProtocolListen::parse))] + pub listen: ProtocolListen, + + /// Network name to be used during handshake, if 'main' is passed the + /// default main network is used. + #[structopt( + long = "protocol-network", + name = "protocol-network", + default_value, + parse(try_from_str = parse_protocol_network)) + ] + pub network: Network, + // TODO(xla): Expose protocol args (membership, replication, etc.). +} + +#[derive(Debug, Eq, PartialEq, StructOpt)] +pub enum ProtocolListen { + Any, + Localhost, + Provided { addr: SocketAddr }, +} + +impl Default for ProtocolListen { + fn default() -> Self { + Self::Localhost + } +} + +impl ProtocolListen { + fn parse(src: &str) -> Result { + match src { + "any" => Ok(Self::Any), + "localhost" => Ok(Self::Localhost), + addr if !addr.is_empty() => Ok(Self::Provided { + addr: SocketAddr::from_str(addr).map_err(|err| err.to_string())?, + }), + _ => Err("protocol listen must be set".to_string()), + } + } +} + +fn parse_protocol_network(src: &str) -> Result { + match src { + _main if src.to_lowercase() == "main" => Ok(Network::Main), + custom if !src.is_empty() => Ok(Network::from_str(custom)?), + _ => Err("custom network can't be empty".to_string()), + } +} diff --git a/node-lib/src/cfg.rs b/node-lib/src/cfg.rs new file mode 100644 index 000000000..d005dd855 --- /dev/null +++ b/node-lib/src/cfg.rs @@ -0,0 +1,205 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::{ + convert::TryFrom, + io, + net::{Ipv4Addr, SocketAddr, SocketAddrV4, ToSocketAddrs as _}, + time::Duration, +}; + +use anyhow::{bail, Context, Result}; +use thrussh_agent::client::ClientStream; +use tokio::{ + fs::File, + io::{stdin, AsyncReadExt as _}, + time::{error::Elapsed, timeout}, +}; +use tracing::warn; + +use librad::{ + crypto::{BoxedSigner, IntoSecretKeyError}, + git::storage, + keystore::SecretKeyExt as _, + net, + net::{discovery, peer::Config as PeerConfig}, + profile::{Profile, RadHome}, + SecretKey, +}; +use rad_clib::keys; + +use crate::args; + +mod seed; +pub use seed::{Seed, Seeds}; + +lazy_static::lazy_static! { + /// General binding to any available port, i.e. `0.0.0.0:0`. + pub static ref ANY: SocketAddr = + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(0, 0, 0, 0), 0)); + + /// Localhost binding to any available port, i.e. `127.0.0.1:0`. + pub static ref LOCALHOST: SocketAddr = + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 0)); +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("decoding base64 key")] + Base64(#[from] base64::DecodeError), + + #[error(transparent)] + Io(#[from] io::Error), + + #[error(transparent)] + Init(#[from] storage::error::Init), + + #[error(transparent)] + Keys(#[from] keys::Error), + + #[error(transparent)] + Other(#[from] anyhow::Error), + + #[error(transparent)] + Profile(#[from] librad::profile::Error), + + #[error(transparent)] + SecretKey(#[from] IntoSecretKeyError), + + #[error("resolving seed nodes")] + Seed(#[from] seed::Error), + + #[error(transparent)] + Timeout(#[from] Elapsed), +} + +pub struct Cfg { + pub disco: Disco, + pub metrics: Option, + pub peer: PeerConfig, +} + +impl Cfg { + pub async fn from_args(args: &args::Args) -> Result + where + S: ClientStream + Unpin + 'static, + { + let seeds = Seeds::resolve(&args.bootstraps).await?; + let disco = discovery::Static::try_from(seeds)?; + let profile = Profile::try_from(args)?; + let signer = construct_signer::(args, &profile).await?; + + // Ensure the storage is accessible for the created profile and signer. + storage::Storage::init(profile.paths(), signer.clone())?; + + let listen_addr = match args.protocol.listen { + args::ProtocolListen::Any => *ANY, + args::ProtocolListen::Localhost => *LOCALHOST, + args::ProtocolListen::Provided { addr } => addr, + }; + + let metrics = match args.metrics.provider { + Some(args::MetricsProvider::Graphite) => Some(Metrics::Graphite( + args.metrics + .graphite_addr + .to_socket_addrs()? + .next() + .unwrap(), + )), + None => None, + }; + + Ok(Self { + disco, + metrics, + peer: PeerConfig { + signer, + protocol: net::protocol::Config { + paths: profile.paths().clone(), + listen_addr, + advertised_addrs: None, + membership: Default::default(), + network: args.protocol.network.clone(), + replication: Default::default(), + fetch: Default::default(), + rate_limits: Default::default(), + }, + storage: Default::default(), + }, + }) + } +} + +pub enum Metrics { + Graphite(SocketAddr), +} + +impl TryFrom<&args::Args> for Profile { + type Error = Error; + + fn try_from(args: &args::Args) -> Result { + let home = if args.tmp_root { + warn!("creating temporary root which is ephemeral and should only be used for debug and testing"); + RadHome::Root(tempfile::tempdir()?.path().to_path_buf()) + } else { + args.rad_home.clone() + }; + + Profile::from_home(&home, args.profile_id.clone()).map_err(Error::from) + } +} + +async fn construct_signer(args: &args::Args, profile: &Profile) -> anyhow::Result +where + S: ClientStream + Unpin + 'static, +{ + match args.signer { + args::Signer::SshAgent => keys::signer_ssh::(profile) + .await + .map_err(anyhow::Error::from), + args::Signer::Key => { + let bytes = match args.key.source { + args::KeySource::Ephemeral => { + warn!("generating key in-memory which is ephemeral and should only be used for debug and testing"); + + SecretKey::new().as_ref().to_vec() + }, + args::KeySource::File => { + if args.key.file_path.is_none() { + bail!("file path must be present when file source is set"); + } + + let mut file = File::open(args.key.file_path.clone().unwrap()) + .await + .context("opening key file")?; + let mut bytes = vec![]; + + timeout(Duration::from_secs(5), file.read_to_end(&mut bytes)) + .await? + .context("reading key file")?; + + bytes + }, + args::KeySource::Stdin => { + let mut bytes = vec![]; + timeout(Duration::from_secs(5), stdin().read_to_end(&mut bytes)) + .await? + .context("reading stdin")?; + bytes + }, + }; + + let key = match args.key.format { + args::KeyFormat::Base64 => { + let bs = base64::decode(&bytes)?; + SecretKey::from_bytes_and_meta(bs.into(), &())? + }, + args::KeyFormat::Binary => SecretKey::from_bytes_and_meta(bytes.into(), &())?, + }; + + Ok(BoxedSigner::from(key)) + }, + } +} diff --git a/node-lib/src/cfg/seed.rs b/node-lib/src/cfg/seed.rs new file mode 100644 index 000000000..5038ea624 --- /dev/null +++ b/node-lib/src/cfg/seed.rs @@ -0,0 +1,75 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::{convert::TryFrom, io, net::SocketAddr}; + +use librad::net::discovery; +use tokio::net::lookup_host; + +use librad::PeerId; + +use crate::args::Bootstrap; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("the seed `{0}` failed to resolve to an address")] + DnsLookupFailed(String), + + #[error(transparent)] + Io(#[from] io::Error), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Seed { + pub addrs: Vec, + pub peer_id: PeerId, +} + +impl Seed { + /// Create a [`Seed`] from a [`Bootstrap`]. + /// + /// # Errors + /// + /// * If the supplied address cannot be resolved. + async fn try_from_bootstrap(bootstrap: &Bootstrap) -> Result { + let addrs: Vec = lookup_host(bootstrap.addr.clone()).await?.collect(); + if !addrs.is_empty() { + Ok(Self { + addrs, + peer_id: bootstrap.peer_id, + }) + } else { + Err(Error::DnsLookupFailed(bootstrap.to_string())) + } + } +} + +pub struct Seeds(pub Vec); + +impl Seeds { + pub async fn resolve(bootstraps: &[Bootstrap]) -> Result { + let mut resolved = Vec::with_capacity(bootstraps.len()); + + for bootstrap in bootstraps.iter() { + resolved.push(Seed::try_from_bootstrap(bootstrap).await?); + } + + Ok(Self(resolved)) + } +} + +impl TryFrom for discovery::Static { + type Error = Error; + + fn try_from(seeds: Seeds) -> Result { + discovery::Static::resolve( + seeds + .0 + .iter() + .map(|seed| (seed.peer_id, seed.addrs.as_slice())), + ) + .map_err(Error::from) + } +} diff --git a/node-lib/src/lib.rs b/node-lib/src/lib.rs new file mode 100644 index 000000000..bbea36b9b --- /dev/null +++ b/node-lib/src/lib.rs @@ -0,0 +1,18 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +pub mod args; + +mod cfg; +pub use cfg::{Seed, Seeds}; + +mod logging; +mod metrics; +pub mod node; +mod protocol; +mod signals; + +#[cfg(unix)] +pub mod socket_activation; diff --git a/node-lib/src/logging.rs b/node-lib/src/logging.rs new file mode 100644 index 000000000..ec263ead8 --- /dev/null +++ b/node-lib/src/logging.rs @@ -0,0 +1,49 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::env; + +use log::{log_enabled, Level}; +use tracing::subscriber::set_global_default as set_subscriber; +use tracing_subscriber::{EnvFilter, FmtSubscriber}; + +/// Initialise logging / tracing +/// +/// The `TRACING_FMT` environment variable can be used to control the log +/// formatting. Supported values: +/// +/// * "pretty": [`tracing_subscriber::fmt::format::Pretty`] +/// * "compact": [`tracing_subscriber::fmt::format::Compact`] +/// * "json": [`tracing_subscriber::fmt::format::Json`] +/// +/// If the variable is not set, or set to any other value, the +/// [`tracing_subscriber::fmt::format::Full`] format is used. +pub fn init() { + if env_logger::builder().try_init().is_ok() { + let mut builder = FmtSubscriber::builder() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("debug")), + ) + .with_test_writer(); + if log_enabled!(target: "librad", Level::Trace) { + builder = builder.with_thread_ids(true); + } else if env::var("TRACING_FMT").is_err() { + let default_format = if env::var("CI").is_ok() { + "compact" + } else { + "pretty" + }; + env::set_var("TRACING_FMT", default_format); + } + + match env::var("TRACING_FMT").ok().as_deref() { + Some("pretty") => set_subscriber(builder.pretty().finish()), + Some("compact") => set_subscriber(builder.compact().finish()), + Some("json") => set_subscriber(builder.json().flatten_event(true).finish()), + _ => set_subscriber(builder.finish()), + } + .expect("setting tracing subscriber failed") + } +} diff --git a/node-lib/src/metrics.rs b/node-lib/src/metrics.rs new file mode 100644 index 000000000..0e5019dfe --- /dev/null +++ b/node-lib/src/metrics.rs @@ -0,0 +1,6 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +pub mod graphite; diff --git a/node-lib/src/metrics/graphite.rs b/node-lib/src/metrics/graphite.rs new file mode 100644 index 000000000..717638f9c --- /dev/null +++ b/node-lib/src/metrics/graphite.rs @@ -0,0 +1,60 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::{ + net::SocketAddr, + time::{Duration, SystemTime}, +}; + +use tokio::{net::UdpSocket, time}; +use tracing::{debug, info, instrument}; + +use librad::{net::peer::Peer, Signer}; + +const CONNECTIONS_TOTAL: &str = "connections_total"; +const CONNECTED_PEERS: &str = "connected_peers"; +const MEMBERSHIP_ACTIVE: &str = "membership_active"; +const MEMBERSHIP_PASSIVE: &str = "membership_passive"; + +#[instrument(name = "graphite subroutine", skip(peer))] +pub async fn routine(peer: Peer, graphite_addr: SocketAddr) -> anyhow::Result<()> +where + S: Signer + Clone, +{ + info!("starting graphite stats routine"); + + debug!("connecting to graphite at {}", graphite_addr); + let sock = UdpSocket::bind("0.0.0.0:0").await?; + sock.connect(graphite_addr).await?; + debug!("connected to graphite at {}", graphite_addr); + + let peer_id = peer.peer_id().to_string(); + loop { + time::sleep(Duration::from_secs(10)).await; + + let stats = time::timeout(Duration::from_secs(5), peer.stats()).await?; + let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?; + + for (metric, value) in &[ + (CONNECTED_PEERS, stats.connected_peers.len()), + (CONNECTIONS_TOTAL, stats.connections_total), + (MEMBERSHIP_ACTIVE, stats.membership_active), + (MEMBERSHIP_PASSIVE, stats.membership_passive), + ] { + sock.send(line(peer_id.clone(), metric, *value as f32, now).as_bytes()) + .await?; + } + } +} + +fn line(peer_id: String, metric: &str, value: f32, time: Duration) -> String { + format!( + "linkd_{};peer={} {:?} {}", + metric, + peer_id, + value, + time.as_secs() + ) +} diff --git a/node-lib/src/node.rs b/node-lib/src/node.rs new file mode 100644 index 000000000..34e6255d1 --- /dev/null +++ b/node-lib/src/node.rs @@ -0,0 +1,82 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::panic; + +use futures::future::{select_all, FutureExt as _}; +use structopt::StructOpt as _; +use tokio::{spawn, sync::mpsc}; +use tracing::info; + +use librad::{ + crypto::BoxedSigner, + net::{discovery, peer::Peer}, +}; + +use crate::{ + args::Args, + cfg::{self, Cfg}, + logging, + metrics::graphite, + protocol, + signals, +}; + +pub async fn run() -> anyhow::Result<()> { + logging::init(); + + let args = Args::from_args(); + let cfg: Cfg = cfg(&args).await?; + + let (shutdown_tx, shutdown_rx) = mpsc::channel(1); + let signals_task = tokio::spawn(signals::routine(shutdown_tx)); + + let mut coalesced = vec![]; + let peer = Peer::new(cfg.peer)?; + let peer_task = spawn(protocol::routine(peer.clone(), cfg.disco, shutdown_rx)).fuse(); + coalesced.push(peer_task); + + if let Some(cfg::Metrics::Graphite(addr)) = cfg.metrics { + let graphite_task = spawn(graphite::routine(peer, addr)).fuse(); + coalesced.push(graphite_task); + } + + // if let Some(_listener) = socket_activation::env()? { + // TODO(xla): Schedule listen loop. + // } else { + // TODO(xla): Bind to configured/default socket path, constructed from + // profile info. + // TODO(xla): Schedule listen loop. + // } + + // TODO(xla): Setup subroutines. + // - Public API + // - Anncouncemnets + // - Replication Requests + // - Tracking + + info!("starting node"); + let (res, _idx, _rest) = select_all(coalesced).await; + + if let Err(e) = res { + if e.is_panic() { + panic::resume_unwind(e.into_panic()); + } + } + + signals_task.await??; + + Ok(()) +} + +#[cfg(unix)] +async fn cfg(args: &Args) -> anyhow::Result> { + Ok(Cfg::from_args::(args).await?) +} + +#[cfg(windows)] +async fn cfg(args: &Args) -> anyhow::Result> { + Ok(Cfg::from_args::(args).await?) +} diff --git a/node-lib/src/protocol.rs b/node-lib/src/protocol.rs new file mode 100644 index 000000000..d023f309e --- /dev/null +++ b/node-lib/src/protocol.rs @@ -0,0 +1,72 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::{net::SocketAddr, panic, time::Duration}; + +use futures::{future::FutureExt as _, pin_mut, select}; +use tokio::{sync::mpsc, time::sleep}; +use tracing::{error, info, instrument}; + +use librad::{ + net::{self, discovery::Discovery, peer::Peer}, + Signer, +}; + +#[instrument(name = "peer subroutine", skip(disco, peer, shutdown_rx))] +pub async fn routine( + peer: Peer, + disco: D, + mut shutdown_rx: mpsc::Receiver<()>, +) -> anyhow::Result<()> +where + D: Discovery + Clone + 'static, + S: Signer + Clone, +{ + let shutdown = shutdown_rx.recv().fuse(); + futures::pin_mut!(shutdown); + + loop { + match peer.bind().await { + Ok(bound) => { + let (stop, run) = bound.accept(disco.clone().discover()); + let run = run.fuse(); + pin_mut!(run); + + let res = select! { + _ = shutdown => { + stop(); + run.await + } + res = run => res + }; + + match res { + Err(net::protocol::io::error::Accept::Done) => { + info!("network endpoint shut down"); + break; + }, + Err(err) => { + error!(?err, "accept error"); + }, + Ok(never) => never, + } + }, + Err(err) => { + error!(?err, "bind error"); + + let sleep = sleep(Duration::from_secs(2)).fuse(); + pin_mut!(sleep); + select! { + _ = sleep => {}, + _ = shutdown => { + break; + } + } + }, + } + } + + Ok(()) +} diff --git a/node-lib/src/signals.rs b/node-lib/src/signals.rs new file mode 100644 index 000000000..700b94d8d --- /dev/null +++ b/node-lib/src/signals.rs @@ -0,0 +1,46 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use tokio::{select, sync::mpsc}; +use tracing::{info, instrument}; + +#[cfg(unix)] +#[instrument(name = "signals subroutine", skip(shutdown_tx))] +pub async fn routine(shutdown_tx: mpsc::Sender<()>) -> anyhow::Result<()> { + use tokio::signal::unix::*; + + let mut int = signal(SignalKind::interrupt())?; + let mut quit = signal(SignalKind::quit())?; + let mut term = signal(SignalKind::terminate())?; + + let signal = select! { + _ = int.recv() => SignalKind::interrupt(), + _ = quit.recv() => SignalKind::quit(), + _ = term.recv() => SignalKind::terminate(), + }; + + info!(?signal, "received termination signal"); + let _ = shutdown_tx.try_send(()); + + Ok(()) +} + +#[cfg(windows)] +#[instrument(name = "signals subroutine", skip(shutdown_tx))] +pub async fn routine(shutdown_tx: mpsc::Sender<()>) -> anyhow::Result<()> { + use tokio::signal::windows::*; + + let mut br = ctrl_break()?; + let mut c = ctrl_c()?; + + select! { + _ = br.recv() => info!("received Break signal"), + _ = c.recv() => info!("recieved CtrlC signal"), + }; + + let _ = shutdown_tx.try_send(()); + + Ok(()) +} diff --git a/node-lib/src/socket_activation.rs b/node-lib/src/socket_activation.rs new file mode 100644 index 000000000..aa8d4c0f5 --- /dev/null +++ b/node-lib/src/socket_activation.rs @@ -0,0 +1,29 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::os::unix::net::UnixListener; + +use anyhow::Result; + +#[cfg(all(unix, target_os = "macos"))] +mod macos; +#[cfg(all(unix, target_os = "macos"))] +use macos as imp; + +#[cfg(all(unix, not(target_os = "macos")))] +mod unix; +#[cfg(all(unix, not(target_os = "macos")))] +use unix as imp; + +/// Constructs a Unix socket from the file descriptor passed through the +/// environemnt. The returned listener will be `None` if there are no +/// environment variables set that are applicable for the current platform or no +/// suitable implementations are activated/supported: +/// +/// * systemd under unix systems with an OS other than macos: https://www.freedesktop.org/software/systemd/man/systemd.socket.html +/// * launchd under macos: https://en.wikipedia.org/wiki/Launchd#Socket_activation_protocol +pub fn env() -> Result> { + imp::env() +} diff --git a/node-lib/src/socket_activation/macos.rs b/node-lib/src/socket_activation/macos.rs new file mode 100644 index 000000000..8719a016d --- /dev/null +++ b/node-lib/src/socket_activation/macos.rs @@ -0,0 +1,12 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::os::unix::net::UnixListener; + +use anyhow::Result; + +pub fn env() -> Result> { + todo!() +} diff --git a/node-lib/src/socket_activation/unix.rs b/node-lib/src/socket_activation/unix.rs new file mode 100644 index 000000000..a99318df8 --- /dev/null +++ b/node-lib/src/socket_activation/unix.rs @@ -0,0 +1,67 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +//! Implementation of the systemd socket activation protocol. +//! http://0pointer.de/blog/projects/socket-activation.html +//! +//! TODO +//! * support FDs beyond 3 +//! * support mapping from listen names + +use std::{ + env, + os::unix::{io::RawFd, net::UnixListener, prelude::FromRawFd}, +}; + +use anyhow::{bail, Result}; +use nix::{ + fcntl::{fcntl, FcntlArg::F_SETFD, FdFlag}, + sys::socket::SockAddr, + unistd::Pid, +}; + +/// Environemnt variable which carries the amount of file descriptors passed +/// down. +const LISTEN_FDS: &str = "LISTEN_FDS"; +/// Environment variable containing colon-separated list of names corresponding +/// to the `FileDescriptorName` option in the service file. +const _LISTEN_NAMES: &str = "LISTEN_NAMES"; +/// Environemnt variable when present should match PID of the current process. +const LISTEN_PID: &str = "LISTEN_PID"; + +pub fn env() -> Result> { + // TODO(xla): Enable usage of more than the first fd. For now the assumption + // should be safe as long as the service files are defined in accordance. + if let Some(fd) = fds().and_then(|fds| fds.first().cloned()) { + if !matches!(nix::sys::socket::getsockname(fd)?, SockAddr::Unix(_)) { + bail!( + "file descriptor {} taken from env is not a valid unix socket", + fd + ); + } + + // Set FD_CLOEXEC to avoid further inheritance to children. + fcntl(fd, F_SETFD(FdFlag::FD_CLOEXEC))?; + + return Ok(Some(unsafe { FromRawFd::from_raw_fd(fd) })); + } + + Ok(None) +} + +fn fds() -> Option> { + if let Some(count) = env::var(LISTEN_FDS).ok().and_then(|x| x.parse().ok()) { + if env::var(LISTEN_PID).ok() == Some(Pid::this().to_string()) { + env::remove_var(LISTEN_FDS); + env::remove_var(LISTEN_PID); + + // Magic number to start counting FDs from, as 0, 1 and 2 are + // reserved for stdin, stdout and stderr respectively. + return Some((0..count).map(|offset| 3 + offset as RawFd).collect()); + } + } + + None +} diff --git a/test/Cargo.toml b/test/Cargo.toml index 0a3f7409b..f2df81e46 100644 --- a/test/Cargo.toml +++ b/test/Cargo.toml @@ -10,6 +10,7 @@ license = "GPL-3.0-or-later" test = true [dependencies] +assert_cmd = "2" assert_matches = "1" anyhow = "1" async-stream = "0.3" @@ -29,6 +30,7 @@ log = "0.4" minicbor = "0.9.1" multibase = "0.9" multihash = "0.11" +nix = "0.22" nonempty = "0.6" nonzero_ext = "0.2" once_cell = "1" @@ -40,6 +42,7 @@ serde = "1" serde_json = "1" sha-1 = "0.9" sized-vec = "0.3" +structopt = { version = "0.3", default-features = false } tempfile = "3" typenum = "1.13" tokio = "1.1" @@ -67,6 +70,9 @@ path = "../link-canonical" path = "../link-git-protocol" features = ["git2"] +[dependencies.node-lib] +path = "../node-lib" + [dependencies.rad-exe] path = "../rad-exe" diff --git a/test/examples/socket_activation.rs b/test/examples/socket_activation.rs new file mode 100644 index 000000000..9ed983963 --- /dev/null +++ b/test/examples/socket_activation.rs @@ -0,0 +1,18 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::process::exit; + +use anyhow::Result; + +use node_lib::socket_activation; + +fn main() -> Result<()> { + if let Some(_listener) = socket_activation::env()? { + exit(0) + } else { + exit(1); + } +} diff --git a/test/examples/socket_activation_wrapper.rs b/test/examples/socket_activation_wrapper.rs new file mode 100644 index 000000000..28fd27b45 --- /dev/null +++ b/test/examples/socket_activation_wrapper.rs @@ -0,0 +1,34 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use anyhow::Result; +use nix::{sys::socket, unistd::Pid}; +use std::{fs::remove_file, os::unix::process::CommandExt as _, process::Command}; + +fn main() -> Result<()> { + remove_file("/tmp/test-linkd-socket-activation.sock").ok(); + + let sock = socket::socket( + socket::AddressFamily::Unix, + socket::SockType::Stream, + socket::SockFlag::empty(), + None, + )?; + let addr = socket::SockAddr::new_unix("/tmp/test-linkd-socket-activation.sock")?; + socket::bind(sock, &addr)?; + socket::listen(sock, 1)?; + + let mut cmd = Command::new("cargo"); + cmd.arg("run") + .arg("-p") + .arg("radicle-link-test") + .arg("--example") + .arg("socket_activation"); + cmd.env("LISTEN_FDS", "1"); + cmd.env("LISTEN_PID", Pid::this().to_string()); + cmd.exec(); + + Ok(()) +} diff --git a/test/src/test/integration.rs b/test/src/test/integration.rs index 3e534249e..25efb94cc 100644 --- a/test/src/test/integration.rs +++ b/test/src/test/integration.rs @@ -7,3 +7,4 @@ mod daemon; mod git_helpers; mod librad; mod link_git_protocol; +mod node_lib; diff --git a/test/src/test/integration/node_lib.rs b/test/src/test/integration/node_lib.rs new file mode 100644 index 000000000..c67813df7 --- /dev/null +++ b/test/src/test/integration/node_lib.rs @@ -0,0 +1,6 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +mod socket_activation; diff --git a/test/src/test/integration/node_lib/socket_activation.rs b/test/src/test/integration/node_lib/socket_activation.rs new file mode 100644 index 000000000..918dadd42 --- /dev/null +++ b/test/src/test/integration/node_lib/socket_activation.rs @@ -0,0 +1,10 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +#[cfg(all(unix, target_os = "macos"))] +mod macos; + +#[cfg(all(unix, not(target_os = "macos")))] +mod unix; diff --git a/test/src/test/integration/node_lib/socket_activation/macos.rs b/test/src/test/integration/node_lib/socket_activation/macos.rs new file mode 100644 index 000000000..0634e8173 --- /dev/null +++ b/test/src/test/integration/node_lib/socket_activation/macos.rs @@ -0,0 +1,4 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. diff --git a/test/src/test/integration/node_lib/socket_activation/unix.rs b/test/src/test/integration/node_lib/socket_activation/unix.rs new file mode 100644 index 000000000..f51d8483b --- /dev/null +++ b/test/src/test/integration/node_lib/socket_activation/unix.rs @@ -0,0 +1,22 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::process::Command; + +use anyhow::Result; + +#[test] +fn construct_listener_from_env() -> Result<()> { + let mut cmd = Command::new("cargo"); + cmd.arg("run") + .arg("-p") + .arg("radicle-link-test") + .arg("--example") + .arg("socket_activation_wrapper"); + let mut cmd = assert_cmd::cmd::Command::from_std(cmd); + cmd.assert().success(); + + Ok(()) +} diff --git a/test/src/test/unit.rs b/test/src/test/unit.rs index 36a4e2c46..4f09f8c62 100644 --- a/test/src/test/unit.rs +++ b/test/src/test/unit.rs @@ -7,4 +7,5 @@ mod git_ext; mod git_trailers; mod librad; mod link_git_protocol; +mod node_lib; mod rad_exe; diff --git a/test/src/test/unit/node_lib.rs b/test/src/test/unit/node_lib.rs new file mode 100644 index 000000000..46738f120 --- /dev/null +++ b/test/src/test/unit/node_lib.rs @@ -0,0 +1,7 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +mod args; +mod cfg; diff --git a/test/src/test/unit/node_lib/args.rs b/test/src/test/unit/node_lib/args.rs new file mode 100644 index 000000000..74be27085 --- /dev/null +++ b/test/src/test/unit/node_lib/args.rs @@ -0,0 +1,373 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::{ + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + path::PathBuf, + str::FromStr, +}; + +use anyhow::Result; +use structopt::StructOpt as _; + +use librad::{ + net::Network, + profile::{ProfileId, RadHome}, +}; + +use node_lib::args::{ + self, + Args, + Bootstrap, + KeyArgs, + MetricsArgs, + MetricsProvider, + ProtocolArgs, + ProtocolListen, + Signer, +}; + +#[test] +fn defaults() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_matches!( + parsed, + Args { + rad_home: RadHome::ProjectDirs, + .. + } + ); + assert_eq!( + parsed, + Args { + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn bootstraps() -> Result<()> { + let bootstraps = vec![ + Bootstrap { + addr: "sprout.radicle.xyz:12345".to_string(), + peer_id: "hynkyndc6w3p8urucakobzna7sxwgcqny7xxtw88dtx3pkf7m3nrzc".parse()?, + }, + Bootstrap { + addr: "setzling.radicle.xyz:12345".to_string(), + peer_id: "hybz9gfgtd9d4pd14a6r66j5hz6f77fed4jdu7pana4fxaxbt369kg".parse()?, + }, + ]; + + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--bootstrap", "hynkyndc6w3p8urucakobzna7sxwgcqny7xxtw88dtx3pkf7m3nrzc@sprout.radicle.xyz:12345", + "--bootstrap", "hybz9gfgtd9d4pd14a6r66j5hz6f77fed4jdu7pana4fxaxbt369kg@setzling.radicle.xyz:12345", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + bootstraps, + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn metrics_graphite() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--metrics-provider", "graphite", + "--graphite-addr", "graphite:9108", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + metrics: MetricsArgs { + provider: Some(MetricsProvider::Graphite), + graphite_addr: "graphite:9108".to_string(), + }, + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn profile_id() -> Result<()> { + let id = ProfileId::new(); + + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--profile-id", id.as_str() + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + profile_id: Some(id), + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn protocol_listen() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "127.0.0.1:12345", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + protocol: ProtocolArgs { + listen: ProtocolListen::Provided { + addr: SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 12345)) + }, + ..Default::default() + }, + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn protocol_network() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--protocol-network", "testnet", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + protocol: ProtocolArgs { + network: Network::from_str("testnet").unwrap(), + ..Default::default() + }, + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn rad_home() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--rad-home", "/tmp/linkd", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + rad_home: RadHome::Root(PathBuf::from("/tmp/linkd")), + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn signer_key_file() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--signer", "key", + "--key-source", "file", + "--key-file-path", "~/.config/radicle/secret.key", + ]; + let parsed = Args::from_iter_safe(iter)?; + assert_eq!( + parsed, + Args { + signer: args::Signer::Key, + key: KeyArgs { + source: args::KeySource::File, + file_path: Some(PathBuf::from("~/.config/radicle/secret.key")), + ..Default::default() + }, + ..Default::default() + } + ); + + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--signer", "key", + "--key-format", "base64", + "--key-source", "file", + "--key-file-path", "~/.config/radicle/secret.seed", + ]; + let parsed = Args::from_iter_safe(iter)?; + assert_eq!( + parsed, + Args { + signer: args::Signer::Key, + key: KeyArgs { + format: args::KeyFormat::Base64, + source: args::KeySource::File, + file_path: Some(PathBuf::from("~/.config/radicle/secret.seed")), + }, + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn signer_key_ephemeral() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--signer", "key", + "--key-source", "ephemeral", + ]; + let parsed = Args::from_iter_safe(iter)?; + assert_eq!( + parsed, + Args { + signer: args::Signer::Key, + key: KeyArgs { + source: args::KeySource::Ephemeral, + ..Default::default() + }, + ..Default::default() + } + ); + + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--signer", "key", + "--key-format", "base64", + "--key-source", "file", + "--key-file-path", "~/.config/radicle/secret.seed", + ]; + let parsed = Args::from_iter_safe(iter)?; + assert_eq!( + parsed, + Args { + signer: args::Signer::Key, + key: KeyArgs { + format: args::KeyFormat::Base64, + source: args::KeySource::File, + file_path: Some(PathBuf::from("~/.config/radicle/secret.seed")), + }, + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn signer_key_stdin() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--signer", "key", + "--key-source", "stdin", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + signer: args::Signer::Key, + key: KeyArgs { + source: args::KeySource::Stdin, + ..Default::default() + }, + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn signer_ssh_agent() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--signer", "ssh-agent", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + signer: Signer::SshAgent, + ..Default::default() + } + ); + + Ok(()) +} + +#[test] +fn tmp_root() -> Result<()> { + #[rustfmt::skip] + let iter = vec![ + "linkd", + "--protocol-listen", "localhost", + "--tmp-root", + ]; + let parsed = Args::from_iter_safe(iter)?; + + assert_eq!( + parsed, + Args { + tmp_root: true, + ..Default::default() + } + ); + + Ok(()) +} diff --git a/test/src/test/unit/node_lib/cfg.rs b/test/src/test/unit/node_lib/cfg.rs new file mode 100644 index 000000000..89bc3806a --- /dev/null +++ b/test/src/test/unit/node_lib/cfg.rs @@ -0,0 +1,35 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use std::net; + +use anyhow::Result; +use pretty_assertions::assert_eq; + +use node_lib::{Seed, Seeds}; + +#[tokio::test(flavor = "multi_thread")] +async fn test_resolve_seeds() -> Result<()> { + let seeds = Seeds::resolve(&[ + "hydsst3z3d5bc6pxq4gz1g4cu6sgbx38czwf3bmmk3ouz4ibjbbtds@localhost:9999" + .parse() + .unwrap(), + ]) + .await?; + + assert!(!seeds.0.is_empty(), "seeds should not be empty"); + + if let Some(Seed { addrs, .. }) = seeds.0.first() { + let addr = addrs.first().unwrap(); + let expected: net::SocketAddr = match *addr { + net::SocketAddr::V4(_addr) => ([127, 0, 0, 1], 9999).into(), + net::SocketAddr::V6(_addr) => "[::1]:9999".parse().expect("valid ivp6 address"), + }; + + assert_eq!(expected, *addr); + } + + Ok(()) +} From 1e0dd589f986cf978ee7bdc2bcf4c9c291a3fb2b Mon Sep 17 00:00:00 2001 From: Alexander Simmerl Date: Tue, 14 Sep 2021 23:34:10 +1000 Subject: [PATCH 5/6] linkd: Add binary A very thin and light wrapper around the `node::run` exposed from the node lib crate, so `linkd` exists as a deploy target. Bootstrapping a whole new workspace which is excluded from root, allows for a checked in `Cargo.lock` without affecting the lib crates. Historically this used to be the role of radicle-bins. Signed-off-by: Alexander Simmerl --- Cargo.toml | 3 +++ bins/Cargo.toml | 8 ++++++++ bins/linkd.key | 1 + bins/linkd/Cargo.toml | 15 +++++++++++++++ bins/linkd/src/main.rs | 13 +++++++++++++ 5 files changed, 40 insertions(+) create mode 100644 bins/Cargo.toml create mode 100644 bins/linkd.key create mode 100644 bins/linkd/Cargo.toml create mode 100644 bins/linkd/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 15414818e..7dd4dc79e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,9 @@ members = [ "std-ext", "test", ] +exclude = [ + "bins" +] [patch.crates-io.git2] git = "https://github.com/radicle-dev/git2-rs.git" diff --git a/bins/Cargo.toml b/bins/Cargo.toml new file mode 100644 index 000000000..c3f795f07 --- /dev/null +++ b/bins/Cargo.toml @@ -0,0 +1,8 @@ +[workspace] +members = [ + "linkd" +] + +[patch.crates-io.thrussh-encoding] +git = "https://github.com/FintanH/thrussh.git" +branch = "generic-agent" diff --git a/bins/linkd.key b/bins/linkd.key new file mode 100644 index 000000000..78d46b499 --- /dev/null +++ b/bins/linkd.key @@ -0,0 +1 @@ +:5-B|dkGkH2(ܞI \ No newline at end of file diff --git a/bins/linkd/Cargo.toml b/bins/linkd/Cargo.toml new file mode 100644 index 000000000..5b2909518 --- /dev/null +++ b/bins/linkd/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "linkd" +version = "0.1.0" +edition = "2018" +license = "GPL-3.0-or-later" +authors = [ + "xla ", +] + +[dependencies] +tokio = { version = "1.10", default-features = false, features = [ "macros", "process", "rt-multi-thread" ] } + +[dependencies.node-lib] +path = "../../node-lib" +version = "0.1.0" diff --git a/bins/linkd/src/main.rs b/bins/linkd/src/main.rs new file mode 100644 index 000000000..b9badde1f --- /dev/null +++ b/bins/linkd/src/main.rs @@ -0,0 +1,13 @@ +// Copyright © 2021 The Radicle Link Contributors +// +// This file is part of radicle-link, distributed under the GPLv3 with Radicle +// Linking Exception. For full terms see the included LICENSE file. + +use node_lib::node::run; + +#[tokio::main] +async fn main() { + if let Err(e) = run().await { + eprintln!("linkd failed: {:?}", e); + } +} From 05164e446c91a4e1f8db11f2a916e3573fdc2959 Mon Sep 17 00:00:00 2001 From: Alexander Simmerl Date: Tue, 14 Sep 2021 23:36:29 +1000 Subject: [PATCH 6/6] e2e: Replace ephemeral peer with linkd Now that a binary exists which drives the core protocol and has all the knobs exposed that the e2e network tests want to turn there is no need anymore for the ad-hocish epehmeral peer implementation. This concludes the first chapter of linkd's development as the declared goal of gaining some confidence through at the integration layer for it succeeded. Signed-off-by: Alexander Simmerl --- e2e/Procfile | 4 +-- e2e/compose.yaml | 29 ++++++++++++------- ...meral-peer.dockerfile => linkd.dockerfile} | 11 +++---- 3 files changed, 26 insertions(+), 18 deletions(-) rename e2e/{ephemeral-peer.dockerfile => linkd.dockerfile} (72%) diff --git a/e2e/Procfile b/e2e/Procfile index 5d7d5ea5f..4c718ad42 100644 --- a/e2e/Procfile +++ b/e2e/Procfile @@ -1,2 +1,2 @@ -bootstrap: cargo run --bin ephemeral-peer -- --secret-key hIfobTmxKMemyXPOC8EmUNdufwi2MsKucEB9EikOyDE --listen 127.0.0.1:54321 -peer: sleep 5; cargo run --bin ephemeral-peer -- --bootstrap hyne66jefcpkobg91qzdy6ysetr8fn3p3d6myce61uwf7s67g3i79e@127.0.0.1:54321 +bootstrap: cd ../bins && echo -n "hIfobTmxKMemyXPOC8EmUNdufwi2MsKucEB9EikOyDE" | cargo run -p linkd -- --signer key --key-format base64 --protocol-listen 127.0.0.1:54321 --tmp-root +peer: sleep 5; cd ../bins && cargo run -p linkd -- --signer key --key-source ephemeral --tmp-root --protocol-listen localhost --bootstrap hyne66jefcpkobg91qzdy6ysetr8fn3p3d6myce61uwf7s67g3i79e@127.0.0.1:54321 diff --git a/e2e/compose.yaml b/e2e/compose.yaml index 7d88649aa..438644eb6 100644 --- a/e2e/compose.yaml +++ b/e2e/compose.yaml @@ -33,16 +33,19 @@ services: - 'graphite-exporter' build: context: ../ - dockerfile: e2e/ephemeral-peer.dockerfile - image: ephemeral-peer + dockerfile: e2e/linkd.dockerfile + image: linkd init: true ports: - '12345:12345/udp' - command: | - ephemeral-peer - --secret-key hIfobTmxKMemyXPOC8EmUNdufwi2MsKucEB9EikOyDE - --listen 0.0.0.0:12345 - --graphite graphite:9109 + command: bash -c 'echo -n "hIfobTmxKMemyXPOC8EmUNdufwi2MsKucEB9EikOyDE" | + linkd + --signer key + --key-format base64 + --protocol-listen 0.0.0.0:12345 + --tmp-root + --metrics-provider graphite + --graphite-addr graphite:9109' environment: - 'RUST_LOG=${RUST_LOG:-debug}' - 'TRACING_FMT=${TRACING_FMT:-compact}' @@ -52,15 +55,19 @@ services: passive-peer: depends_on: - 'bootstrap-peer' - image: ephemeral-peer + image: linkd init: true ports: - '12346/udp' command: | - ephemeral-peer - --listen 0.0.0.0:12346 + linkd + --signer key + --key-source ephemeral + --protocol-listen 0.0.0.0:12346 + --tmp-root --bootstrap hyne66jefcpkobg91qzdy6ysetr8fn3p3d6myce61uwf7s67g3i79e@bootstrap:12345 - --graphite graphite:9109 + --metrics-provider graphite + --graphite-addr graphite:9109 environment: - 'RUST_LOG=${RUST_LOG:-debug}' - 'TRACING_FMT=${TRACING_FMT:-compact}' diff --git a/e2e/ephemeral-peer.dockerfile b/e2e/linkd.dockerfile similarity index 72% rename from e2e/ephemeral-peer.dockerfile rename to e2e/linkd.dockerfile index aa6931915..0fd92f904 100644 --- a/e2e/ephemeral-peer.dockerfile +++ b/e2e/linkd.dockerfile @@ -7,9 +7,10 @@ RUN --mount=type=bind,source=.,target=/build,rw \ --mount=type=cache,target=/cache \ set -eux pipefail; \ mkdir -p /cache/target; \ - ln -s /cache/target target ; \ - cargo build --release --package radicle-link-e2e --bin ephemeral-peer; \ - mv target/release/ephemeral-peer /ephemeral-peer + ln -s /cache/target target; \ + cd bins; \ + cargo build --release --package linkd; \ + mv target/release/linkd /linkd FROM debian:buster-slim RUN set -eux; \ @@ -21,5 +22,5 @@ RUN set -eux; \ ; \ apt-get autoremove; \ rm -rf /var/lib/apt/lists/* -COPY --from=build /ephemeral-peer /usr/local/bin/ephemeral-peer -CMD ["ephemeral-peer"] +COPY --from=build /linkd /usr/local/bin/linkd +CMD ["linkd"]