Skip to content

Commit

Permalink
generate-copyright: use rinja to format the output
Browse files Browse the repository at this point in the history 
I can't find a way to derive rinja::Template for Node - I think because it is a recursive type. So I rendered it manually using html_escape.
  • Loading branch information
@jonathanpallant
jonathanpallant committed Jul 30, 2024
1 parent 21c1a87 commit 93d34de
Show file tree
Hide file tree
Showing 4 changed files with 146 additions and 152 deletions.
6 changes: 4 additions & 2 deletions src/tools/generate-copyright/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ description = "Produces a manifest of all the copyrighted materials in the Rust

[dependencies]
anyhow = "1.0.65"
cargo_metadata = "0.18.1"
html-escape = "0.2.13"
rinja = "0.2.0"
serde = { version = "1.0.147", features = ["derive"] }
serde_json = "1.0.85"
thiserror = "1"
tempfile = "3"
cargo_metadata = "0.18.1"
thiserror = "1"
12 changes: 8 additions & 4 deletions src/tools/generate-copyright/src/cargo_metadata.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Gets metadata about a workspace from Cargo

use std::collections::BTreeMap;
use std::ffi::{OsStr, OsString};
use std::ffi::OsStr;
use std::path::Path;

/// Describes how this module can fail
Expand Down Expand Up @@ -36,7 +36,9 @@ pub struct PackageMetadata {
/// A list of important files from the package, with their contents.
///
/// This includes *COPYRIGHT*, *NOTICE*, *AUTHOR*, *LICENSE*, and *LICENCE* files, case-insensitive.
pub notices: BTreeMap<OsString, String>,
pub notices: BTreeMap<String, String>,
/// If this is true, this dep is in the Rust Standard Library
pub is_in_libstd: Option<bool>,
}

/// Use `cargo metadata` and `cargo vendor` to get a list of dependencies and their license data.
Expand Down Expand Up @@ -101,6 +103,7 @@ pub fn get_metadata(
license: package.license.unwrap_or_else(|| String::from("Unspecified")),
authors: package.authors,
notices: BTreeMap::new(),
is_in_libstd: None,
},
);
}
Expand Down Expand Up @@ -161,8 +164,9 @@ fn load_important_files(
if metadata.is_dir() {
// scoop up whole directory
} else if metadata.is_file() {
println!("Scraping {}", filename.to_string_lossy());
dep.notices.insert(filename.to_owned(), std::fs::read_to_string(path)?);
let filename = filename.to_string_lossy();
println!("Scraping {}", filename);
dep.notices.insert(filename.to_string(), std::fs::read_to_string(path)?);
}
}
}
Expand Down
226 changes: 80 additions & 146 deletions src/tools/generate-copyright/src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,17 @@
use std::collections::BTreeMap;
use std::io::Write;
use std::path::{Path, PathBuf};

use anyhow::Error;
use rinja::Template;

mod cargo_metadata;

static TOP_BOILERPLATE: &str = r##"
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>Copyright notices for The Rust Toolchain</title>
</head>
<body>
<h1>Copyright notices for The Rust Toolchain</h1>
<p>This file describes the copyright and licensing information for the source
code within The Rust Project git tree, and the third-party dependencies used
when building the Rust toolchain (including the Rust Standard Library).</p>
<h2>Table of Contents</h2>
<ul>
<li><a href="#in-tree-files">In-tree files</a></li>
<li><a href="#out-of-tree-dependencies">Out-of-tree dependencies</a></li>
</ul>
"##;

static BOTTOM_BOILERPLATE: &str = r#"
</body>
</html>
"#;
#[derive(Template)]
#[template(path = "COPYRIGHT.html")]
struct CopyrightTemplate {
in_tree: Node,
dependencies: BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
}

/// The entry point to the binary.
///
Expand All @@ -53,150 +33,114 @@ fn main() -> Result<(), Error> {
Path::new("./src/tools/cargo/Cargo.toml"),
Path::new("./library/std/Cargo.toml"),
];
let collected_cargo_metadata =
let mut collected_cargo_metadata =
cargo_metadata::get_metadata_and_notices(&cargo, &out_dir, &root_path, &workspace_paths)?;

let stdlib_set =
cargo_metadata::get_metadata(&cargo, &root_path, &[Path::new("./library/std/Cargo.toml")])?;

let mut buffer = Vec::new();
for (key, value) in collected_cargo_metadata.iter_mut() {
value.is_in_libstd = Some(stdlib_set.contains_key(key));
}

writeln!(buffer, "{}", TOP_BOILERPLATE)?;
let template = CopyrightTemplate {
in_tree: collected_tree_metadata.files,
dependencies: collected_cargo_metadata,
};

writeln!(
buffer,
r#"<h2 id="in-tree-files">In-tree files</h2><p>The following licenses cover the in-tree source files that were used in this release:</p>"#
)?;
render_tree_recursive(&collected_tree_metadata.files, &mut buffer)?;
let output = template.render()?;

writeln!(
buffer,
r#"<h2 id="out-of-tree-dependencies">Out-of-tree dependencies</h2><p>The following licenses cover the out-of-tree crates that were used in this release:</p>"#
)?;
render_deps(&collected_cargo_metadata, &stdlib_set, &mut buffer)?;
std::fs::write(&dest_file, output)?;

writeln!(buffer, "{}", BOTTOM_BOILERPLATE)?;
Ok(())
}

std::fs::write(&dest_file, &buffer)?;
/// Describes a tree of metadata for our filesystem tree
#[derive(serde::Deserialize)]
struct Metadata {
files: Node,
}

/// Describes one node in our metadata tree
#[derive(serde::Deserialize)]
#[serde(rename_all = "kebab-case", tag = "type")]
pub(crate) enum Node {
Root { children: Vec<Node> },
Directory { name: String, children: Vec<Node>, license: Option<License> },
File { name: String, license: License },
Group { files: Vec<String>, directories: Vec<String>, license: License },
}

fn with_box<F>(fmt: &mut std::fmt::Formatter<'_>, inner: F) -> std::fmt::Result
where
F: FnOnce(&mut std::fmt::Formatter<'_>) -> std::fmt::Result,
{
writeln!(fmt, r#"<div style="border:1px solid black; padding: 5px;">"#)?;
inner(fmt)?;
writeln!(fmt, "</div>")?;
Ok(())
}

/// Recursively draw the tree of files/folders we found on disk and their licenses, as
/// markdown, into the given Vec.
fn render_tree_recursive(node: &Node, buffer: &mut Vec<u8>) -> Result<(), Error> {
writeln!(buffer, r#"<div style="border:1px solid black; padding: 5px;">"#)?;
match node {
Node::Root { children } => {
for child in children {
render_tree_recursive(child, buffer)?;
impl std::fmt::Display for Node {
fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Node::Root { children } => {
if children.len() > 1 {
with_box(fmt, |f| {
for child in children {
writeln!(f, "{child}")?;
}
Ok(())
})
} else {
for child in children {
writeln!(fmt, "{child}")?;
}
Ok(())
}
}
}
Node::Directory { name, children, license } => {
render_tree_license(std::iter::once(name), license.as_ref(), buffer)?;
if !children.is_empty() {
writeln!(buffer, "<p><b>Exceptions:</b></p>")?;
for child in children {
render_tree_recursive(child, buffer)?;
Node::Directory { name, children, license } => with_box(fmt, |f| {
render_tree_license(std::iter::once(name), license.as_ref(), f)?;
if !children.is_empty() {
writeln!(f, "<p><b>Exceptions:</b></p>")?;
for child in children {
writeln!(f, "{child}")?;
}
}
Ok(())
}),
Node::Group { files, directories, license } => with_box(fmt, |f| {
render_tree_license(directories.iter().chain(files.iter()), Some(license), f)
}),
Node::File { name, license } => {
with_box(fmt, |f| render_tree_license(std::iter::once(name), Some(license), f))
}
}
Node::Group { files, directories, license } => {
render_tree_license(directories.iter().chain(files.iter()), Some(license), buffer)?;
}
Node::File { name, license } => {
render_tree_license(std::iter::once(name), Some(license), buffer)?;
}
}
writeln!(buffer, "</div>")?;

Ok(())
}

/// Draw a series of sibling files/folders, as markdown, into the given Vec.
/// Draw a series of sibling files/folders, as HTML, into the given formatter.
fn render_tree_license<'a>(
names: impl Iterator<Item = &'a String>,
license: Option<&License>,
buffer: &mut Vec<u8>,
) -> Result<(), Error> {
writeln!(buffer, "<p><b>File/Directory:</b> ")?;
f: &mut std::fmt::Formatter<'_>,
) -> std::fmt::Result {
writeln!(f, "<p><b>File/Directory:</b> ")?;
for name in names {
writeln!(buffer, "<code>{name}</code>")?;
writeln!(f, "<code>{}</code>", html_escape::encode_text(&name))?;
}
writeln!(buffer, "</p>")?;
writeln!(f, "</p>")?;

if let Some(license) = license {
writeln!(buffer, "<p><b>License:</b> {}</p>", license.spdx)?;
writeln!(f, "<p><b>License:</b> {}</p>", html_escape::encode_text(&license.spdx))?;
for copyright in license.copyright.iter() {
writeln!(buffer, "<p><b>Copyright:</b> {copyright}</p>")?;
writeln!(f, "<p><b>Copyright:</b> {}</p>", html_escape::encode_text(&copyright))?;
}
}

Ok(())
}

/// Render a list of out-of-tree dependencies as markdown into the given Vec.
fn render_deps(
all_deps: &BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
stdlib_set: &BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
buffer: &mut Vec<u8>,
) -> Result<(), Error> {
for (package, metadata) in all_deps {
let authors_list = if metadata.authors.is_empty() {
"None Specified".to_owned()
} else {
metadata.authors.join(", ")
};
let url = format!("https://crates.io/crates/{}/{}", package.name, package.version);
writeln!(buffer)?;
writeln!(
buffer,
r#"<h3>📦 {name}-{version}</h3>"#,
name = package.name,
version = package.version,
)?;
writeln!(buffer, r#"<p><b>URL:</b> <a href="{url}">{url}</a></p>"#,)?;
writeln!(
buffer,
"<p><b>In libstd:</b> {}</p>",
if stdlib_set.contains_key(package) { "Yes" } else { "No" }
)?;
writeln!(buffer, "<p><b>Authors:</b> {}</p>", escape_html(&authors_list))?;
writeln!(buffer, "<p><b>License:</b> {}</p>", escape_html(&metadata.license))?;
writeln!(buffer, "<p><b>Notices:</b> ")?;
if metadata.notices.is_empty() {
writeln!(buffer, "None")?;
} else {
for (name, contents) in &metadata.notices {
writeln!(
buffer,
"<details><summary><code>{}</code></summary>",
name.to_string_lossy()
)?;
writeln!(buffer, "<pre>\n{}\n</pre>", contents)?;
writeln!(buffer, "</details>")?;
}
}
writeln!(buffer, "</p>")?;
}
Ok(())
}
/// Describes a tree of metadata for our filesystem tree
#[derive(serde::Deserialize)]
struct Metadata {
files: Node,
}

/// Describes one node in our metadata tree
#[derive(serde::Deserialize)]
#[serde(rename_all = "kebab-case", tag = "type")]
pub(crate) enum Node {
Root { children: Vec<Node> },
Directory { name: String, children: Vec<Node>, license: Option<License> },
File { name: String, license: License },
Group { files: Vec<String>, directories: Vec<String>, license: License },
}

/// A License has an SPDX license name and a list of copyright holders.
#[derive(serde::Deserialize)]
struct License {
Expand All @@ -212,13 +156,3 @@ fn env_path(var: &str) -> Result<PathBuf, Error> {
anyhow::bail!("missing environment variable {var}")
}
}

/// Escapes any invalid HTML characters
fn escape_html(input: &str) -> String {
static MAPPING: [(char, &str); 3] = [('&', "&amp;"), ('<', "&lt;"), ('>', "&gt;")];
let mut output = input.to_owned();
for (ch, s) in &MAPPING {
output = output.replace(*ch, s);
}
output
}
54 changes: 54 additions & 0 deletions src/tools/generate-copyright/templates/COPYRIGHT.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>Copyright notices for The Rust Toolchain</title>
</head>
<body>

<h1>Copyright notices for The Rust Toolchain</h1>

<p>This file describes the copyright and licensing information for the source
code within The Rust Project git tree, and the third-party dependencies used
when building the Rust toolchain (including the Rust Standard Library).</p>

<h2>Table of Contents</h2>
<ul>
<li><a href="#in-tree-files">In-tree files</a></li>
<li><a href="#out-of-tree-dependencies">Out-of-tree dependencies</a></li>
</ul>

<h2 id="in-tree-files">In-tree files</h2>

<p>The following licenses cover the in-tree source files that were used in this
release:</p>

{{ in_tree|safe }}

<h2 id="out-of-tree-dependencies">Out-of-tree dependencies</h2>

<p>The following licenses cover the out-of-tree crates that were used in this
release:</p>

{% for (key, value) in dependencies %}
<h3>📦 {{key.name}}-{{key.version}}</h3>
<p><b>URL:</b> <a href="https://crates.io/crates/{{ key.name }}/{{ key.version }}">https://crates.io/crates/{{ key.name }}/{{ key.version }}</a></p>
<p><b>In libstd:</b> {% if value.is_in_libstd.unwrap() %} Yes {% else %} No {% endif %}</p>
<p><b>Authors:</b> {{ value.authors|join(", ") }}</p>
<p><b>License:</b> {{ value.license }}</p>
{% let len = value.notices.len() %}
{% if len > 0 %}
<p><b>Notices:</b>
{% for (notice_name, notice_text) in value.notices %}
<details>
<summary><code>{{ notice_name }}</code></summary>
<pre>
{{ notice_text }}
</pre>
</details>
{% endfor %}
</p>
{% endif %}
{% endfor %}
</body>
</html>

0 comments on commit 93d34de

Please sign in to comment.