From 34ae07e16388fb077c989fe291fd13366f475e0c Mon Sep 17 00:00:00 2001 From: Gus Caplan Date: Tue, 24 Nov 2020 14:08:36 -0600 Subject: [PATCH 1/4] Add custom docker image support --- crates/metadata/lib.rs | 9 +++++++++ src/docbuilder/rustwide_builder.rs | 28 ++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/crates/metadata/lib.rs b/crates/metadata/lib.rs index 195e3df13..78e0cece2 100644 --- a/crates/metadata/lib.rs +++ b/crates/metadata/lib.rs @@ -95,6 +95,7 @@ pub enum MetadataError { /// targets = [ "x86_64-apple-darwin", "x86_64-pc-windows-msvc" ] /// rustc-args = [ "--example-rustc-arg" ] /// rustdoc-args = [ "--example-rustdoc-arg" ] +/// docker-image = "rustops/crates-build-env" /// ``` /// /// You can define one or more fields in your `Cargo.toml`. @@ -128,6 +129,9 @@ pub struct Metadata { /// List of command line arguments for `rustdoc`. #[serde(default)] rustdoc_args: Vec, + + /// Custom docker image. + docker_image: Option, } /// The targets that should be built for a crate. @@ -277,6 +281,11 @@ impl Metadata { map.insert("DOCS_RS", "1".into()); map } + + /// Return the custom docker image, if provided. + pub fn docker_image(&self) -> Option { + self.docker_image.clone() + } } impl std::str::FromStr for Metadata { diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 04fefeabb..b6d11331a 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -106,11 +106,15 @@ impl RustwideBuilder { self.skip_build_if_exists = should; } - fn prepare_sandbox(&self, limits: &Limits) -> SandboxBuilder { - SandboxBuilder::new() + fn prepare_sandbox(&self, metadata: &Metadata, limits: &Limits) -> Result { + let mut builder = SandboxBuilder::new() .cpu_limit(self.config.build_cpu_limit.map(|limit| limit as f32)) .memory_limit(Some(limits.memory())) - .enable_networking(limits.networking()) + .enable_networking(limits.networking()); + if let Some(image) = metadata.docker_image() { + builder = builder.image(SandboxImage::remote(&image)?) + } + Ok(builder) } pub fn update_toolchain(&mut self) -> Result<()> { @@ -208,11 +212,15 @@ impl RustwideBuilder { let krate = Crate::crates_io(DUMMY_CRATE_NAME, DUMMY_CRATE_VERSION); krate.fetch(&self.workspace)?; + let metadata = Metadata::from_crate_root(&build_dir.build_dir())?; + build_dir - .build(&self.toolchain, &krate, self.prepare_sandbox(&limits)) + .build( + &self.toolchain, + &krate, + self.prepare_sandbox(&metadata, &limits)?, + ) .run(|build| { - let metadata = Metadata::from_crate_root(&build.host_source_dir())?; - let res = self.execute_build(HOST_TARGET, true, build, &limits, &metadata)?; if !res.result.successful { failure::bail!("failed to build dummy crate for {}", self.rustc_version); @@ -322,14 +330,18 @@ impl RustwideBuilder { let local_storage = tempfile::Builder::new().prefix("docsrs-docs").tempdir()?; + let metadata = Metadata::from_crate_root(&build_dir.build_dir())?; let res = build_dir - .build(&self.toolchain, &krate, self.prepare_sandbox(&limits)) + .build( + &self.toolchain, + &krate, + self.prepare_sandbox(&metadata, &limits)?, + ) .run(|build| { use docsrs_metadata::BuildTargets; let mut has_docs = false; let mut successful_targets = Vec::new(); - let metadata = Metadata::from_crate_root(&build.host_source_dir())?; let BuildTargets { default_target, other_targets, From 947bda1260d06c45d7940f824d193df168172927 Mon Sep 17 00:00:00 2001 From: Gus Caplan Date: Tue, 24 Nov 2020 16:41:48 -0600 Subject: [PATCH 2/4] patch dep --- Cargo.lock | 27 ++++++++++++++++++++++++--- Cargo.toml | 2 +- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd65ffba9..9981ec609 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,21 @@ dependencies = [ "syn", ] +[[package]] +name = "attohttpc" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1082810677916862c7704351dfe4696a837aaf34da0dd6431abc60783e71ee8f" +dependencies = [ + "flate2", + "http", + "log 0.4.8", + "native-tls", + "openssl", + "url 2.1.1", + "wildmatch", +] + [[package]] name = "atty" version = "0.2.14" @@ -2872,9 +2887,9 @@ dependencies = [ [[package]] name = "rustwide" version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17a9c66cf835ece6742443f3a2c2874df15db3dfc060ced53feeb210a463fd93" +source = "git+https://github.com/devsnek/rustwide.git?branch=sandbox-image-override#5536645973149cc81217917862a9ea8957cda68e" dependencies = [ + "attohttpc", "base64 0.12.1", "failure", "flate2", @@ -2882,12 +2897,12 @@ dependencies = [ "futures-util", "getrandom", "git2", + "http", "lazy_static", "log 0.4.8", "nix", "percent-encoding 2.1.0", "remove_dir_all", - "reqwest", "scopeguard", "serde", "serde_json", @@ -4062,6 +4077,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wildmatch" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79346daa5ca66c72db46ee4dac6e605811478ae0807b385d18328be3f5c0eb74" + [[package]] name = "winapi" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index 669fca1ad..bb1b5547e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ schemamama = "0.3" schemamama_postgres = "0.3" systemstat = "0.1.4" prometheus = { version = "0.10.0", default-features = false } -rustwide = "0.11" +rustwide = { git = "https://github.com/devsnek/rustwide.git", branch = "sandbox-image-override" } mime_guess = "2" dotenv = "0.15" zstd = "0.5" From 0c7d7e4423dccfebcd4c9cebddf789bd09489cb1 Mon Sep 17 00:00:00 2001 From: Gus Caplan Date: Tue, 24 Nov 2020 16:50:30 -0600 Subject: [PATCH 3/4] squash! fix dir --- Cargo.lock | 2 +- src/docbuilder/rustwide_builder.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9981ec609..02bfc232b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2887,7 +2887,7 @@ dependencies = [ [[package]] name = "rustwide" version = "0.11.0" -source = "git+https://github.com/devsnek/rustwide.git?branch=sandbox-image-override#5536645973149cc81217917862a9ea8957cda68e" +source = "git+https://github.com/devsnek/rustwide.git?branch=sandbox-image-override#b0eba0018368bdde373a93f647d022c34d056807" dependencies = [ "attohttpc", "base64 0.12.1", diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index b6d11331a..c943bc6ad 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -212,7 +212,7 @@ impl RustwideBuilder { let krate = Crate::crates_io(DUMMY_CRATE_NAME, DUMMY_CRATE_VERSION); krate.fetch(&self.workspace)?; - let metadata = Metadata::from_crate_root(&build_dir.build_dir())?; + let metadata = Metadata::from_crate_root(&build_dir.get_source_dir(&krate)?)?; build_dir .build( @@ -330,7 +330,7 @@ impl RustwideBuilder { let local_storage = tempfile::Builder::new().prefix("docsrs-docs").tempdir()?; - let metadata = Metadata::from_crate_root(&build_dir.build_dir())?; + let metadata = Metadata::from_crate_root(&build_dir.get_source_dir(&krate)?)?; let res = build_dir .build( &self.toolchain, From 52e606561c1dc396860887188fa803aaa5dec2c3 Mon Sep 17 00:00:00 2001 From: Gus Caplan Date: Thu, 26 Nov 2020 20:32:22 -0600 Subject: [PATCH 4/4] delete image after build --- crates/metadata/lib.rs | 4 ++-- src/docbuilder/rustwide_builder.rs | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/metadata/lib.rs b/crates/metadata/lib.rs index 78e0cece2..84f115d9c 100644 --- a/crates/metadata/lib.rs +++ b/crates/metadata/lib.rs @@ -283,8 +283,8 @@ impl Metadata { } /// Return the custom docker image, if provided. - pub fn docker_image(&self) -> Option { - self.docker_image.clone() + pub fn docker_image(&self) -> Option<&String> { + self.docker_image.as_ref() } } diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index c943bc6ad..007487ab9 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -436,6 +436,11 @@ impl RustwideBuilder { build_dir.purge()?; krate.purge_from_cache(&self.workspace)?; local_storage.close()?; + if let Some(image) = metadata.docker_image() { + std::process::Command::new("docker") + .args(&["image", "rm", image]) + .status()?; + } Ok(res.result.successful) }