From 93d34dee923a8b829ef12f3943a4fdf78e947468 Mon Sep 17 00:00:00 2001 From: Jonathan Pallant Date: Tue, 30 Jul 2024 19:39:06 +0100 Subject: [PATCH] generate-copyright: use rinja to format the output I can't find a way to derive rinja::Template for Node - I think because it is a recursive type. So I rendered it manually using html_escape. --- src/tools/generate-copyright/Cargo.toml | 6 +- .../generate-copyright/src/cargo_metadata.rs | 12 +- src/tools/generate-copyright/src/main.rs | 226 +++++++----------- .../templates/COPYRIGHT.html | 54 +++++ 4 files changed, 146 insertions(+), 152 deletions(-) create mode 100644 src/tools/generate-copyright/templates/COPYRIGHT.html diff --git a/src/tools/generate-copyright/Cargo.toml b/src/tools/generate-copyright/Cargo.toml index c94cc35fb5036..c00292cf33108 100644 --- a/src/tools/generate-copyright/Cargo.toml +++ b/src/tools/generate-copyright/Cargo.toml @@ -8,8 +8,10 @@ description = "Produces a manifest of all the copyrighted materials in the Rust [dependencies] anyhow = "1.0.65" +cargo_metadata = "0.18.1" +html-escape = "0.2.13" +rinja = "0.2.0" serde = { version = "1.0.147", features = ["derive"] } serde_json = "1.0.85" -thiserror = "1" tempfile = "3" -cargo_metadata = "0.18.1" +thiserror = "1" diff --git a/src/tools/generate-copyright/src/cargo_metadata.rs b/src/tools/generate-copyright/src/cargo_metadata.rs index 655d73715e036..d02b9eeb6f922 100644 --- a/src/tools/generate-copyright/src/cargo_metadata.rs +++ b/src/tools/generate-copyright/src/cargo_metadata.rs @@ -1,7 +1,7 @@ //! Gets metadata about a workspace from Cargo use std::collections::BTreeMap; -use std::ffi::{OsStr, OsString}; +use std::ffi::OsStr; use std::path::Path; /// Describes how this module can fail @@ -36,7 +36,9 @@ pub struct PackageMetadata { /// A list of important files from the package, with their contents. /// /// This includes *COPYRIGHT*, *NOTICE*, *AUTHOR*, *LICENSE*, and *LICENCE* files, case-insensitive. - pub notices: BTreeMap, + pub notices: BTreeMap, + /// If this is true, this dep is in the Rust Standard Library + pub is_in_libstd: Option, } /// Use `cargo metadata` and `cargo vendor` to get a list of dependencies and their license data. @@ -101,6 +103,7 @@ pub fn get_metadata( license: package.license.unwrap_or_else(|| String::from("Unspecified")), authors: package.authors, notices: BTreeMap::new(), + is_in_libstd: None, }, ); } @@ -161,8 +164,9 @@ fn load_important_files( if metadata.is_dir() { // scoop up whole directory } else if metadata.is_file() { - println!("Scraping {}", filename.to_string_lossy()); - dep.notices.insert(filename.to_owned(), std::fs::read_to_string(path)?); + let filename = filename.to_string_lossy(); + println!("Scraping {}", filename); + dep.notices.insert(filename.to_string(), std::fs::read_to_string(path)?); } } } diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs index af69ab8c8bf36..03b789b739298 100644 --- a/src/tools/generate-copyright/src/main.rs +++ b/src/tools/generate-copyright/src/main.rs @@ -1,37 +1,17 @@ use std::collections::BTreeMap; -use std::io::Write; use std::path::{Path, PathBuf}; use anyhow::Error; +use rinja::Template; mod cargo_metadata; -static TOP_BOILERPLATE: &str = r##" - - - - - Copyright notices for The Rust Toolchain - - - -

Copyright notices for The Rust Toolchain

- -

This file describes the copyright and licensing information for the source -code within The Rust Project git tree, and the third-party dependencies used -when building the Rust toolchain (including the Rust Standard Library).

- -

Table of Contents

- -"##; - -static BOTTOM_BOILERPLATE: &str = r#" - - -"#; +#[derive(Template)] +#[template(path = "COPYRIGHT.html")] +struct CopyrightTemplate { + in_tree: Node, + dependencies: BTreeMap, +} /// The entry point to the binary. /// @@ -53,150 +33,114 @@ fn main() -> Result<(), Error> { Path::new("./src/tools/cargo/Cargo.toml"), Path::new("./library/std/Cargo.toml"), ]; - let collected_cargo_metadata = + let mut collected_cargo_metadata = cargo_metadata::get_metadata_and_notices(&cargo, &out_dir, &root_path, &workspace_paths)?; let stdlib_set = cargo_metadata::get_metadata(&cargo, &root_path, &[Path::new("./library/std/Cargo.toml")])?; - let mut buffer = Vec::new(); + for (key, value) in collected_cargo_metadata.iter_mut() { + value.is_in_libstd = Some(stdlib_set.contains_key(key)); + } - writeln!(buffer, "{}", TOP_BOILERPLATE)?; + let template = CopyrightTemplate { + in_tree: collected_tree_metadata.files, + dependencies: collected_cargo_metadata, + }; - writeln!( - buffer, - r#"

In-tree files

The following licenses cover the in-tree source files that were used in this release:

"# - )?; - render_tree_recursive(&collected_tree_metadata.files, &mut buffer)?; + let output = template.render()?; - writeln!( - buffer, - r#"

Out-of-tree dependencies

The following licenses cover the out-of-tree crates that were used in this release:

"# - )?; - render_deps(&collected_cargo_metadata, &stdlib_set, &mut buffer)?; + std::fs::write(&dest_file, output)?; - writeln!(buffer, "{}", BOTTOM_BOILERPLATE)?; + Ok(()) +} - std::fs::write(&dest_file, &buffer)?; +/// Describes a tree of metadata for our filesystem tree +#[derive(serde::Deserialize)] +struct Metadata { + files: Node, +} +/// Describes one node in our metadata tree +#[derive(serde::Deserialize)] +#[serde(rename_all = "kebab-case", tag = "type")] +pub(crate) enum Node { + Root { children: Vec }, + Directory { name: String, children: Vec, license: Option }, + File { name: String, license: License }, + Group { files: Vec, directories: Vec, license: License }, +} + +fn with_box(fmt: &mut std::fmt::Formatter<'_>, inner: F) -> std::fmt::Result +where + F: FnOnce(&mut std::fmt::Formatter<'_>) -> std::fmt::Result, +{ + writeln!(fmt, r#"
"#)?; + inner(fmt)?; + writeln!(fmt, "
")?; Ok(()) } -/// Recursively draw the tree of files/folders we found on disk and their licenses, as -/// markdown, into the given Vec. -fn render_tree_recursive(node: &Node, buffer: &mut Vec) -> Result<(), Error> { - writeln!(buffer, r#"
"#)?; - match node { - Node::Root { children } => { - for child in children { - render_tree_recursive(child, buffer)?; +impl std::fmt::Display for Node { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Node::Root { children } => { + if children.len() > 1 { + with_box(fmt, |f| { + for child in children { + writeln!(f, "{child}")?; + } + Ok(()) + }) + } else { + for child in children { + writeln!(fmt, "{child}")?; + } + Ok(()) + } } - } - Node::Directory { name, children, license } => { - render_tree_license(std::iter::once(name), license.as_ref(), buffer)?; - if !children.is_empty() { - writeln!(buffer, "

Exceptions:

")?; - for child in children { - render_tree_recursive(child, buffer)?; + Node::Directory { name, children, license } => with_box(fmt, |f| { + render_tree_license(std::iter::once(name), license.as_ref(), f)?; + if !children.is_empty() { + writeln!(f, "

Exceptions:

")?; + for child in children { + writeln!(f, "{child}")?; + } } + Ok(()) + }), + Node::Group { files, directories, license } => with_box(fmt, |f| { + render_tree_license(directories.iter().chain(files.iter()), Some(license), f) + }), + Node::File { name, license } => { + with_box(fmt, |f| render_tree_license(std::iter::once(name), Some(license), f)) } } - Node::Group { files, directories, license } => { - render_tree_license(directories.iter().chain(files.iter()), Some(license), buffer)?; - } - Node::File { name, license } => { - render_tree_license(std::iter::once(name), Some(license), buffer)?; - } } - writeln!(buffer, "
")?; - - Ok(()) } -/// Draw a series of sibling files/folders, as markdown, into the given Vec. +/// Draw a series of sibling files/folders, as HTML, into the given formatter. fn render_tree_license<'a>( names: impl Iterator, license: Option<&License>, - buffer: &mut Vec, -) -> Result<(), Error> { - writeln!(buffer, "

File/Directory: ")?; + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + writeln!(f, "

File/Directory: ")?; for name in names { - writeln!(buffer, "{name}")?; + writeln!(f, "{}", html_escape::encode_text(&name))?; } - writeln!(buffer, "

")?; + writeln!(f, "

")?; if let Some(license) = license { - writeln!(buffer, "

License: {}

", license.spdx)?; + writeln!(f, "

License: {}

", html_escape::encode_text(&license.spdx))?; for copyright in license.copyright.iter() { - writeln!(buffer, "

Copyright: {copyright}

")?; + writeln!(f, "

Copyright: {}

", html_escape::encode_text(©right))?; } } Ok(()) } -/// Render a list of out-of-tree dependencies as markdown into the given Vec. -fn render_deps( - all_deps: &BTreeMap, - stdlib_set: &BTreeMap, - buffer: &mut Vec, -) -> Result<(), Error> { - for (package, metadata) in all_deps { - let authors_list = if metadata.authors.is_empty() { - "None Specified".to_owned() - } else { - metadata.authors.join(", ") - }; - let url = format!("https://crates.io/crates/{}/{}", package.name, package.version); - writeln!(buffer)?; - writeln!( - buffer, - r#"

📦 {name}-{version}

"#, - name = package.name, - version = package.version, - )?; - writeln!(buffer, r#"

URL: {url}

"#,)?; - writeln!( - buffer, - "

In libstd: {}

", - if stdlib_set.contains_key(package) { "Yes" } else { "No" } - )?; - writeln!(buffer, "

Authors: {}

", escape_html(&authors_list))?; - writeln!(buffer, "

License: {}

", escape_html(&metadata.license))?; - writeln!(buffer, "

Notices: ")?; - if metadata.notices.is_empty() { - writeln!(buffer, "None")?; - } else { - for (name, contents) in &metadata.notices { - writeln!( - buffer, - "

{}", - name.to_string_lossy() - )?; - writeln!(buffer, "
\n{}\n
", contents)?; - writeln!(buffer, "
")?; - } - } - writeln!(buffer, "

")?; - } - Ok(()) -} -/// Describes a tree of metadata for our filesystem tree -#[derive(serde::Deserialize)] -struct Metadata { - files: Node, -} - -/// Describes one node in our metadata tree -#[derive(serde::Deserialize)] -#[serde(rename_all = "kebab-case", tag = "type")] -pub(crate) enum Node { - Root { children: Vec }, - Directory { name: String, children: Vec, license: Option }, - File { name: String, license: License }, - Group { files: Vec, directories: Vec, license: License }, -} - /// A License has an SPDX license name and a list of copyright holders. #[derive(serde::Deserialize)] struct License { @@ -212,13 +156,3 @@ fn env_path(var: &str) -> Result { anyhow::bail!("missing environment variable {var}") } } - -/// Escapes any invalid HTML characters -fn escape_html(input: &str) -> String { - static MAPPING: [(char, &str); 3] = [('&', "&"), ('<', "<"), ('>', ">")]; - let mut output = input.to_owned(); - for (ch, s) in &MAPPING { - output = output.replace(*ch, s); - } - output -} diff --git a/src/tools/generate-copyright/templates/COPYRIGHT.html b/src/tools/generate-copyright/templates/COPYRIGHT.html new file mode 100644 index 0000000000000..ccb177a54d419 --- /dev/null +++ b/src/tools/generate-copyright/templates/COPYRIGHT.html @@ -0,0 +1,54 @@ + + + + + Copyright notices for The Rust Toolchain + + + +

Copyright notices for The Rust Toolchain

+ +

This file describes the copyright and licensing information for the source +code within The Rust Project git tree, and the third-party dependencies used +when building the Rust toolchain (including the Rust Standard Library).

+ +

Table of Contents

+ + +

In-tree files

+ +

The following licenses cover the in-tree source files that were used in this +release:

+ +{{ in_tree|safe }} + +

Out-of-tree dependencies

+ +

The following licenses cover the out-of-tree crates that were used in this +release:

+ +{% for (key, value) in dependencies %} +

📦 {{key.name}}-{{key.version}}

+

URL: https://crates.io/crates/{{ key.name }}/{{ key.version }}

+

In libstd: {% if value.is_in_libstd.unwrap() %} Yes {% else %} No {% endif %}

+

Authors: {{ value.authors|join(", ") }}

+

License: {{ value.license }}

+ {% let len = value.notices.len() %} + {% if len > 0 %} +

Notices: + {% for (notice_name, notice_text) in value.notices %} +

+ {{ notice_name }} + 
+{{ notice_text }}
+
+
+ {% endfor %} +

+ {% endif %} +{% endfor %} + + \ No newline at end of file