feat: implement changes to source parsing (#454)

docs/recipe_file.md

@@ -227,15 +227,15 @@ Patches may optionally be applied to the source.
 #### Destination path
 
 Within boa's work directory, you may specify a particular folder to place source
-into. Boa will always drop you into the same folder (build folder/work), but
+into. `rattler-build` will always drop you into the same folder (build folder/work), but
 it's up to you whether you want your source extracted into that folder, or
 nested deeper. This feature is particularly useful when dealing with multiple
 sources, but can apply to recipes with single sources as well.
 
 ```yaml
 source:
   #[source information here]
-  folder: my-destination/folder
+  target_directory: my-destination/folder
 ```
 
 #### Source from multiple sources
@@ -250,11 +250,11 @@ Example:
 ```yaml
 source:
   - url: https://package1.com/a.tar.bz2
-    folder: stuff
+    target_directory: stuff
   - url: https://package1.com/b.tar.bz2
-    folder: stuff
+    target_directory: stuff
   - git: https://github.com/mamba-org/boa
-    folder: boa
+    target_directory: boa
 ```
 
 Here, the two URL tarballs will go into one folder, and the git repo is checked
@@ -276,7 +276,7 @@ Recursive globbing using `**` is also supported.
 
 The build number should be incremented for new builds of the same version. The
 number defaults to `0`. The build string cannot contain "-". The string defaults
-to the default boa build string plus the build number.
+to the default rattler-build build string plus the build number.
 
 ```yaml
 build:
@@ -320,7 +320,7 @@ build:
 
 ### Script
 
-By default, boa uses a `build.sh` file on Unix (macOS and Linux) and a
+By default, rattler-build uses a `build.sh` file on Unix (macOS and Linux) and a
 `build.bat` file on Linux, if they exist in the same folder as the `recipe.yaml`
 file. With the script parameter you can either supply a different filename or
 write out short build scripts. You may need to use selectors to use different

examples/ros-humble-turtlebot4-msgs/recipe.yaml

@@ -5,9 +5,9 @@ package:
   version: 1.0.3
 
 source:
-  git_url: https://github.com/ros2-gbp/turtlebot4-release.git
-  git_rev: release/humble/turtlebot4_msgs/1.0.3-1
-  folder: ros-humble-turtlebot4-msgs/src/work
+  git: https://github.com/ros2-gbp/turtlebot4-release.git
+  rev: release/humble/turtlebot4_msgs/1.0.3-1
+  target_directory: ros-humble-turtlebot4-msgs/src/work
 
 build:
   script:

src/build.rs

@@ -216,37 +216,51 @@ pub async fn run_build(
     // Add the local channel to the list of channels
     let mut channels = vec![directories.output_dir.to_string_lossy().to_string()];
     channels.extend(output.build_configuration.channels.clone());
-
-    if !output.recipe.sources().is_empty() {
+    let output = if let Some(finalized_sources) = &output.finalized_sources {
         fetch_sources(
+            finalized_sources,
+            &directories.work_dir,
+            &directories.recipe_dir,
+            &directories.output_dir,
+        )
+        .await
+        .into_diagnostic()?;
+
+        output.clone()
+    } else {
+        let rendered_sources = fetch_sources(
             output.recipe.sources(),
             &directories.work_dir,
             &directories.recipe_dir,
             &directories.output_dir,
         )
         .await
         .into_diagnostic()?;
-    }
+
+        Output {
+            finalized_sources: Some(rendered_sources),
+            ..output.clone()
+        }
+    };
 
     let output = if output.finalized_dependencies.is_some() {
         tracing::info!("Using finalized dependencies");
 
         // The output already has the finalized dependencies, so we can just use it as-is
-        install_environments(output, tool_configuration.clone())
+        install_environments(&output, tool_configuration.clone())
             .await
             .into_diagnostic()?;
         output.clone()
     } else {
         let finalized_dependencies =
-            resolve_dependencies(output, &channels, tool_configuration.clone())
+            resolve_dependencies(&output, &channels, tool_configuration.clone())
                 .await
                 .into_diagnostic()?;
 
         // The output with the resolved dependencies
         Output {
             finalized_dependencies: Some(finalized_dependencies),
-            recipe: output.recipe.clone(),
-            build_configuration: output.build_configuration.clone(),
+            ..output.clone()
         }
     };
 

src/main.rs

@@ -555,6 +555,7 @@ async fn run_build_from_args(args: BuildOpts, multi_progress: MultiProgress) ->
                 force_colors: !args.no_force_colors,
             },
             finalized_dependencies: None,
+            finalized_sources: None,
         };
 
         run_build(&output, tool_config.clone()).await?;

src/metadata.rs

@@ -12,8 +12,11 @@ use fs_err as fs;
 use rattler_conda_types::{package::ArchiveType, PackageName, Platform};
 use serde::{Deserialize, Serialize};
 
-use crate::{hash::HashInfo, render::resolved_dependencies::FinalizedDependencies};
-
+use crate::{
+    hash::HashInfo,
+    recipe::parser::{Recipe, Source},
+    render::resolved_dependencies::FinalizedDependencies,
+};
 /// A Git revision
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct GitRev(String);
@@ -25,11 +28,13 @@ impl FromStr for GitRev {
         Ok(GitRev(s.to_string()))
     }
 }
+
 impl Default for GitRev {
     fn default() -> Self {
         Self(String::from("HEAD"))
     }
 }
+
 impl Display for GitRev {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}", self.0)
@@ -206,12 +211,15 @@ pub struct PackageIdentifier {
 #[derive(Clone, Serialize, Deserialize)]
 pub struct Output {
     /// The rendered recipe that is used to build this output
-    pub recipe: crate::recipe::parser::Recipe,
+    pub recipe: Recipe,
     /// The build configuration for this output (e.g. target_platform, channels, and other settings)
     pub build_configuration: BuildConfiguration,
     /// The finalized dependencies for this output. If this is `None`, the dependencies have not been resolved yet.
     /// During the `run_build` functions, the dependencies are resolved and this field is filled.
     pub finalized_dependencies: Option<FinalizedDependencies>,
+    /// The finalized sources for this output. Contain the exact git hashes for the sources that are used
+    /// to build this output.
+    pub finalized_sources: Option<Vec<Source>>,
 }
 
 impl Output {
@@ -446,4 +454,15 @@ mod test {
         let output_curl: Output = serde_yaml::from_str(&recipe_2).unwrap();
         assert_yaml_snapshot!(output_curl);
     }
+
+    #[test]
+    fn read_recipe_with_sources() {
+        let test_data_dir =
+            std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("test-data/rendered_recipes");
+        let recipe_1 = test_data_dir.join("git_source.yaml");
+        let recipe_1 = std::fs::read_to_string(recipe_1).unwrap();
+
+        let git_source_output: Output = serde_yaml::from_str(&recipe_1).unwrap();
+        assert_yaml_snapshot!(git_source_output);
+    }
 }

src/recipe/parser.rs

@@ -34,7 +34,7 @@ pub use self::{
         Compiler, Dependency, IgnoreRunExports, PinSubpackage, Requirements, RunExports,
     },
     script::{Script, ScriptContent},
-    source::{Checksum, GitSource, GitUrl, PathSource, Source, UrlSource},
+    source::{Checksum, GitRev, GitSource, GitUrl, PathSource, Source, UrlSource},
     test::{PackageContent, Test},
 };
 
@@ -43,15 +43,21 @@ use super::custom_yaml::Node;
 /// A recipe that has been parsed and validated.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Recipe {
-    package: Package,
+    /// The package information
+    pub package: Package,
+    /// The information about where to obtain the sources
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    source: Vec<Source>,
-    build: Build,
-    requirements: Requirements,
+    pub source: Vec<Source>,
+    /// The information about how to build the package
+    pub build: Build,
+    /// The information about the requirements
+    pub requirements: Requirements,
+    /// The information about how to test the package
     #[serde(default, skip_serializing_if = "Test::is_default")]
-    test: Test,
+    pub test: Test,
+    /// The information about the package
     #[serde(default, skip_serializing_if = "About::is_default")]
-    about: About,
+    pub about: About,
 }
 pub(crate) trait CollectErrors<K, V>: Iterator<Item = Result<K, V>> + Sized {
     fn collect_errors(self) -> Result<(), Vec<V>> {

src/recipe/parser/snapshots/rattler_build__recipe__parser__source__tests__git_serialization.snap

@@ -0,0 +1,7 @@
+---
+source: src/recipe/parser/source.rs
+expression: yaml
+---
+git: https://test.com/test.git
+rev: refs/heads/master
+