From e3d5f90dd2910b99a7ee6caaefced0161f20e083 Mon Sep 17 00:00:00 2001 From: Kitsu Date: Fri, 30 Jul 2021 15:47:29 +0300 Subject: [PATCH] Allow to specify a key for private registries --- Cargo.lock | 9 +++--- Cargo.toml | 4 +-- dockerfiles/entrypoint.sh | 3 +- src/bin/cratesfyi.rs | 11 ++++++-- src/build_queue.rs | 5 +++- src/config.rs | 10 +++++++ src/docbuilder/rustwide_builder.rs | 17 ++++++++--- src/index/mod.rs | 45 ++++++++++++++++++++++++------ 8 files changed, 79 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5329afdcb..4e071896e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -473,9 +473,9 @@ dependencies = [ [[package]] name = "crates-index-diff" -version = "7.1.2" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64af39a9a6805d715f8b72307d70815ed1ee38ef84e9de250fcdd56fe75a0e19" +checksum = "9f04ef23e6e584a3f8f7860274e0daf1c46e859d387cebdad1de54091d7cea08" dependencies = [ "git2", "serde", @@ -3106,9 +3106,8 @@ dependencies = [ [[package]] name = "rustwide" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "689b0858f1118c80702a03f9cee1ab09955b55b4166b72457859212790afe129" +version = "0.14.0" +source = "git+https://github.com/l4l/rustwide.git?branch=registry-key#bd25fcf067ad47c42bc2431a83a4306de005f85a" dependencies = [ "attohttpc", "base64 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 7066606cc..bc50d0727 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ log = "0.4" regex = "1" structopt = "0.3" crates-index = { version = "0.15.1", optional = true } -crates-index-diff = "7.1.1" +crates-index-diff = "8.0.0" reqwest = { version = "0.11", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready semver = { version = "0.9", features = ["serde"] } slug = "=0.1.1" @@ -48,7 +48,7 @@ schemamama = "0.3" schemamama_postgres = "0.3" systemstat = "0.1.4" prometheus = { version = "0.10.0", default-features = false } -rustwide = "0.13" +rustwide = { git = "https://github.com/l4l/rustwide.git", branch = "registry-key" } mime_guess = "2" dotenv = "0.15" zstd = "0.5" diff --git a/dockerfiles/entrypoint.sh b/dockerfiles/entrypoint.sh index 8d16175c9..a1c62cb13 100755 --- a/dockerfiles/entrypoint.sh +++ b/dockerfiles/entrypoint.sh @@ -6,6 +6,7 @@ export DOCSRS_PREFIX=/opt/docsrs/prefix export DOCSRS_DOCKER=true export DOCSRS_LOG=${DOCSRS_LOG-"docs-rs,rustwide=info"} export PATH="$PATH:/build/target/release" +export REGISTRY_URL=${REGISTRY_URL:-https://github.com/rust-lang/crates.io-index} # Try migrating the database multiple times if it fails # This avoids the docker container crashing the first time it's started with @@ -28,7 +29,7 @@ done set -e if ! [ -d "${DOCSRS_PREFIX}/crates.io-index/.git" ]; then - git clone https://github.com/rust-lang/crates.io-index "${DOCSRS_PREFIX}/crates.io-index" + git clone ${REGISTRY_URL} "${DOCSRS_PREFIX}/crates.io-index" # Prevent new crates built before the container creation to be built git --git-dir="$DOCSRS_PREFIX/crates.io-index/.git" branch crates-index-diff_last-seen fi diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index c28a7f70a..f422db385 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -334,7 +334,9 @@ impl BuildSubcommand { .build_local_package(&path) .context("Building documentation failed")?; } else { - let registry_url = ctx.config()?.registry_url.clone(); + let config = ctx.config()?; + let registry_url = config.registry_url.clone(); + let registry_key = config.registry_key.clone(); builder .build_package( &crate_name @@ -343,7 +345,10 @@ impl BuildSubcommand { .with_context(|| anyhow!("must specify version if not local"))?, registry_url .as_ref() - .map(|s| PackageKind::Registry(s.as_str())) + .map(|s| PackageKind::Registry { + url: s.as_str(), + key: registry_key, + }) .unwrap_or(PackageKind::CratesIo), ) .context("Building documentation failed")?; @@ -598,7 +603,7 @@ impl Context for BinContext { let config = self.config()?; let path = config.registry_index_path.clone(); if let Some(registry_url) = config.registry_url.clone() { - Index::from_url(path, registry_url) + Index::from_url(path, registry_url, config.registry_key.clone()) } else { Index::new(path) }? diff --git a/src/build_queue.rs b/src/build_queue.rs index 447d03569..a3eaa030f 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -257,7 +257,10 @@ impl BuildQueue { let kind = krate .registry .as_ref() - .map(|r| PackageKind::Registry(r.as_str())) + .map(|r| PackageKind::Registry { + url: r.as_str(), + key: self.config.registry_key.clone(), + }) .unwrap_or(PackageKind::CratesIo); if let Err(err) = builder diff --git a/src/config.rs b/src/config.rs index 16dea210d..351c4580f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -11,6 +11,7 @@ pub struct Config { pub prefix: PathBuf, pub registry_index_path: PathBuf, pub registry_url: Option, + pub registry_key: Option, // Database connection params pub(crate) database_url: String, @@ -98,6 +99,15 @@ impl Config { registry_index_path: env("REGISTRY_INDEX_PATH", prefix.join("crates.io-index"))?, registry_url: maybe_env("REGISTRY_URL")?, + registry_key: maybe_env::("REGISTRY_KEY").and_then(|key| { + Ok(if let Some(key) = key { + Some(key) + } else { + maybe_env::("REGISTRY_KEY_PATH")? + .map(std::fs::read_to_string) + .transpose()? + }) + })?, prefix: prefix.clone(), database_url: require_env("DOCSRS_DATABASE_URL")?, diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index caadd4380..9080b395b 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -19,7 +19,7 @@ use postgres::Client; use rustwide::cmd::{Command, CommandError, SandboxBuilder, SandboxImage}; use rustwide::logging::{self, LogStorage}; use rustwide::toolchain::ToolchainError; -use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; +use rustwide::{AlternativeRegistry, Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; use serde_json::Value; use std::collections::{HashMap, HashSet}; use std::path::Path; @@ -32,7 +32,7 @@ const DUMMY_CRATE_VERSION: &str = "1.0.0"; pub enum PackageKind<'a> { Local(&'a Path), CratesIo, - Registry(&'a str), + Registry { url: &'a str, key: Option }, } pub struct RustwideBuilder { @@ -242,7 +242,10 @@ impl RustwideBuilder { let registry_url = self.config.registry_url.clone(); let package_kind = registry_url .as_ref() - .map(|r| PackageKind::Registry(r.as_str())) + .map(|r| PackageKind::Registry { + url: r.as_str(), + key: self.config.registry_key.clone(), + }) .unwrap_or(PackageKind::CratesIo); if let Err(err) = self.build_package(name, version, package_kind) { warn!("failed to build package {} {}: {}", name, version, err); @@ -309,7 +312,13 @@ impl RustwideBuilder { let krate = match kind { PackageKind::Local(path) => Crate::local(path), PackageKind::CratesIo => Crate::crates_io(name, version), - PackageKind::Registry(registry) => Crate::registry(registry, name, version), + PackageKind::Registry { url, key } => { + let mut registry = AlternativeRegistry::new(url); + if let Some(key) = key { + registry.authenticate_with_ssh_key(key); + } + Crate::registry(registry, name, version) + } }; krate.fetch(&self.workspace).map_err(FailureError::compat)?; diff --git a/src/index/mod.rs b/src/index/mod.rs index dc6931e1b..613a2332f 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -15,6 +15,7 @@ pub struct Index { path: PathBuf, api: Api, repository_url: Option, + registry_key: Option, } #[derive(Debug, serde::Deserialize, Clone)] @@ -39,12 +40,31 @@ fn load_config(repo: &git2::Repository) -> Result { Ok(config) } +fn fetch_options(key: &str) -> git2::FetchOptions<'_> { + let mut fo = git2::FetchOptions::new(); + fo.remote_callbacks({ + let mut callbacks = git2::RemoteCallbacks::new(); + callbacks.credentials(move |_url, username_from_url, _allowed_types| { + git2::Cred::ssh_key_from_memory(username_from_url.unwrap(), None, key, None) + }); + callbacks + }); + fo +} + impl Index { - pub fn from_url(path: PathBuf, repository_url: String) -> Result { + pub fn from_url( + path: PathBuf, + repository_url: String, + registry_key: Option, + ) -> Result { let url = repository_url.clone(); let diff = crates_index_diff::Index::from_path_or_cloned_with_options( &path, - crates_index_diff::CloneOptions { repository_url }, + crates_index_diff::CloneOptions { + repository_url, + fetch_options: registry_key.as_ref().map(|s| fetch_options(s.as_str())), + }, ) .context("initialising registry index repository")?; @@ -54,6 +74,7 @@ impl Index { path, api, repository_url: Some(url), + registry_key, }) } @@ -68,17 +89,23 @@ impl Index { path, api, repository_url: None, + registry_key: None, }) } pub(crate) fn diff(&self) -> Result { - let options = self - .repository_url - .clone() - .map(|repository_url| crates_index_diff::CloneOptions { repository_url }) - .unwrap_or_default(); - let diff = crates_index_diff::Index::from_path_or_cloned_with_options(&self.path, options) - .context("re-opening registry index for diff")?; + let fetch_options = self.registry_key.as_deref().map(fetch_options); + let diff = crates_index_diff::Index::from_path_or_cloned_with_options( + &self.path, + self.repository_url + .clone() + .map(move |repository_url| crates_index_diff::CloneOptions { + repository_url, + fetch_options, + }) + .unwrap_or_default(), + ) + .context("re-opening registry index for diff")?; Ok(diff) }