Draft: Allow reuse to scan sub-modules

.reuse/dep5

@@ -1,8 +1,6 @@
 # WARNING: this metadata is currently incomplete, do not rely on it yet.
 
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Files-Excluded:
- src/llvm-project
 
 # Note that we're explicitly listing the individual files at the root of the
 # repository rather than just having `Files: *`. This is explicitly done to
@@ -39,13 +37,8 @@ Files: compiler/*
 Copyright: The Rust Project Developers (see https://thanks.rust-lang.org)
 License: MIT or Apache-2.0
 
-Files: compiler/rustc_codegen_cranelift/src/cranelift_native.rs
-Copyright: The Cranelift Project Developers
-           The Rust Project Developers (see https://thanks.rust-lang.org)
-License: Apache-2.0 WITH LLVM-exception AND (Apache-2.0 OR MIT)
-
 Files: compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
-Copyright: LLVM authors
+Copyright: 2003-2019 University of Illinois at Urbana-Champaign.
            The Rust Project Developers (see https://thanks.rust-lang.org)
 License: Apache-2.0 WITH LLVM-exception AND (Apache-2.0 OR MIT)
 
@@ -85,11 +78,38 @@ Files: src/librustdoc/html/static/css/normalize.css
 Copyright: Nicolas Gallagher and Jonathan Neal
 License: MIT
 
-Files: src/librustdoc/html/static/css/themes/ayu.css
-Copyright: Ike Ku, Jessica Stokes, Leon Guan
+Files: src/librustdoc/html/static/css/rustdoc.css
+Copyright: 2016 Ike Ku
            The Rust Project Developers (see https://thanks.rust-lang.org)
 License: MIT OR Apache-2.0
 
 Files: src/doc/rustc-dev-guide/mermaid.min.js
-Copyright: Knut Sveidqvist
+Copyright: 2014-2021 Knut Sveidqvist
 License: MIT
+
+Files: library/backtrace/*
+Copyright: 2014 Alex Crichton
+           The Rust Project Developers (see https://thanks.rust-lang.org)
+License: MIT OR Apache-2.0
+
+Files: src/doc/embedded-book/*
+Copyright: Rust on Embedded Devices Working Group
+           The Rust Project Developers (see https://thanks.rust-lang.org)
+License: MIT OR Apache-2.0 OR CC-BY-SA-4.0
+
+Files: src/doc/rust-by-example/*
+Copyright: 2014 Jorge Aparicio
+           The Rust Project Developers (see https://thanks.rust-lang.org)
+License: MIT OR Apache-2.0
+
+# Reuse cannot process the LLVM source tree, and so the copyrights for the LLVM
+# submodule are written out here manually. The collect-licence-metadata tool
+# has a specific exception coded within it to ignore ./src/llvm-project so
+# any time LLVM is updated, please revisit this section. The copyrights are
+# taken from the relevant LLVM sub-folders: llvm, lld, lldb, compiler-rt and libunwind.
+
+Files: src/llvm-project/*
+Copyright: 2003-2019 by the contributors listed in [CREDITS.TXT](https://github.com/rust-lang/llvm-project/blob/7738295178045041669876bf32b0543ec8319a5c/llvm/CREDITS.TXT)
+           2010 Apple Inc
+           2003-2019 University of Illinois at Urbana-Champaign.
+License: NCSA OR Apache-2.0 WITH LLVM-exception

LICENSES/CC0-1.0.txt

@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

LICENSES/NCSA.txt

@@ -0,0 +1,29 @@
+Copyright (c) <YEAR> <OWNER ORGANIZATION NAME>.  All rights reserved.
+
+Developed by: <NAME OF DEVELOPMENT GROUP>
+              <NAME OF INSTITUTION>
+              <URL FOR DEVELOPMENT GROUP/INSTITUTION>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to
+do so, subject to the following conditions:
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimers.
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimers in the documentation
+  and/or other materials provided with the distribution.
+* Neither the names of <NAME OF DEVELOPMENT GROUP>, <NAME OF INSTITUTION>,
+  nor the names of its contributors may be used to endorse or promote products
+  derived from this Software without specific prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
+

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -52,7 +52,7 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \
            RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library/test && \
            /scripts/validate-toolstate.sh && \
            /scripts/validate-error-codes.sh && \
-           reuse lint && \
+           reuse --include-submodules lint && \
            # Runs checks to ensure that there are no ES5 issues in our JS code.
            es-check es6 ../src/librustdoc/html/static/js/*.js && \
            eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \

src/tools/collect-license-metadata/src/main.rs

@@ -6,6 +6,16 @@ use crate::licenses::LicensesInterner;
 use anyhow::Error;
 use std::path::PathBuf;
 
+// Some directories have too many slight license differences that'd result in a
+// huge report, and could be considered a standalone project anyway. Those
+// directories are "condensed" into a single licensing block for ease of
+// reading, merging the licensing information.
+//
+// For every `(dir, file)``, every file in `dir` is considered to have the
+// license info of `file`.
+const CONDENSED_DIRECTORIES: &[(&str, &str)] =
+    &[("./src/llvm-project/", "./src/llvm-project/README.md")];
+
 fn main() -> Result<(), Error> {
     let reuse_exe: PathBuf = std::env::var_os("REUSE_EXE").expect("Missing REUSE_EXE").into();
     let dest: PathBuf = std::env::var_os("DEST").expect("Missing DEST").into();

src/tools/collect-license-metadata/src/path_tree.rs

@@ -4,14 +4,15 @@
 //! passes over the tree to remove redundant information.
 
 use crate::licenses::{License, LicenseId, LicensesInterner};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::path::{Path, PathBuf};
 
 #[derive(serde::Serialize)]
 #[serde(rename_all = "kebab-case", tag = "type")]
 pub(crate) enum Node<L> {
     Root { children: Vec<Node<L>> },
     Directory { name: PathBuf, children: Vec<Node<L>>, license: Option<L> },
+    CondensedDirectory { name: PathBuf, licenses: Vec<L> },
     File { name: PathBuf, license: L },
     Group { files: Vec<PathBuf>, directories: Vec<PathBuf>, license: L },
     Empty,
@@ -57,9 +58,9 @@ impl Node<LicenseId> {
                         Node::Directory { name, mut children, license: None } => {
                             directories.entry(name).or_insert_with(Vec::new).append(&mut children);
                         }
-                        file @ Node::File { .. } => {
-                            files.push(file);
-                        }
+                        file @ Node::File { .. } => files.push(file),
+                        // Propagate condensed directories as-is.
+                        condensed @ Node::CondensedDirectory { .. } => files.push(condensed),
                         Node::Empty => {}
                         Node::Root { .. } => {
                             panic!("can't have a root inside another element");
@@ -86,6 +87,7 @@ impl Node<LicenseId> {
             }
             Node::Empty => {}
             Node::File { .. } => {}
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => {
                 panic!("Group should not be present at this stage");
             }
@@ -132,6 +134,7 @@ impl Node<LicenseId> {
                 }
             }
             Node::File { .. } => {}
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => panic!("group should not be present at this stage"),
             Node::Empty => {}
         }
@@ -174,6 +177,9 @@ impl Node<LicenseId> {
                             Node::Directory { name: child_child_name, .. } => {
                                 *child_child_name = child_name.join(&child_child_name);
                             }
+                            Node::CondensedDirectory { name: child_child_name, .. } => {
+                                *child_child_name = child_name.join(&child_child_name);
+                            }
                             Node::File { name: child_child_name, .. } => {
                                 *child_child_name = child_name.join(&child_child_name);
                             }
@@ -188,6 +194,7 @@ impl Node<LicenseId> {
             }
             Node::Empty => {}
             Node::File { .. } => {}
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => panic!("Group should not be present at this stage"),
         }
     }
@@ -255,6 +262,7 @@ impl Node<LicenseId> {
                 }
             }
             Node::File { .. } => {}
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => panic!("FileGroup should not be present at this stage"),
             Node::Empty => {}
         }
@@ -270,6 +278,7 @@ impl Node<LicenseId> {
                 }
                 children.retain(|child| !matches!(child, Node::Empty));
             }
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => {}
             Node::File { .. } => {}
             Node::Empty => {}
@@ -293,7 +302,24 @@ pub(crate) fn build(mut input: Vec<(PathBuf, LicenseId)>) -> Node<LicenseId> {
     // Ensure reproducibility of all future steps.
     input.sort();
 
-    for (path, license) in input {
+    let mut condensed_directories = BTreeMap::new();
+    'outer: for (path, license) in input {
+        // Files in condensed directories are handled separately.
+        for (condensed_directory, allowed_file) in super::CONDENSED_DIRECTORIES {
+            if path.starts_with(condensed_directory) {
+                if path.as_path() == Path::new(allowed_file) {
+                    // The licence on our allowed file is used to represent the entire directory
+                    condensed_directories
+                        .entry(*condensed_directory)
+                        .or_insert_with(BTreeSet::new)
+                        .insert(license);
+                } else {
+                    // don't add the file
+                }
+                continue 'outer;
+            }
+        }
+
         let mut node = Node::File { name: path.file_name().unwrap().into(), license };
         for component in path.parent().unwrap_or_else(|| Path::new(".")).components().rev() {
             node = Node::Directory {
@@ -306,6 +332,22 @@ pub(crate) fn build(mut input: Vec<(PathBuf, LicenseId)>) -> Node<LicenseId> {
         children.push(node);
     }
 
+    for (path, licenses) in condensed_directories {
+        let path = Path::new(path);
+        let mut node = Node::CondensedDirectory {
+            name: path.file_name().unwrap().into(),
+            licenses: licenses.iter().copied().collect(),
+        };
+        for component in path.parent().unwrap_or_else(|| Path::new(".")).components().rev() {
+            node = Node::Directory {
+                name: component.as_os_str().into(),
+                children: vec![node],
+                license: None,
+            };
+        }
+        children.push(node);
+    }
+
     Node::Root { children }
 }
 
@@ -334,6 +376,10 @@ pub(crate) fn expand_interned_licenses(
         Node::Group { files, directories, license } => {
             Node::Group { files, directories, license: interner.resolve(license) }
         }
+        Node::CondensedDirectory { name, licenses } => Node::CondensedDirectory {
+            name,
+            licenses: licenses.into_iter().map(|license| interner.resolve(license)).collect(),
+        },
         Node::Empty => Node::Empty,
     }
 }