Merge branch 'release/v0.3.0'

.github/workflows/build.yml

@@ -21,7 +21,7 @@ jobs:
           cd ui
           ./install_ui
 
-      - uses: actions/upload-artifact@v2.2.4
+      - uses: actions/upload-artifact@v2.3.1
         with:
           name: frontend
           path: |
@@ -42,7 +42,7 @@ jobs:
           distribution: 'zulu'
 
       - name: Setup python
-        uses: actions/setup-python@v2.2.2
+        uses: actions/setup-python@v2.3.1
         with:
           python-version: '3.9'
 
@@ -53,7 +53,7 @@ jobs:
           extractcode --version
 
       - name: Download fronted from the frontend job
-        uses: actions/download-artifact@v2.0.10
+        uses: actions/download-artifact@v2.1.0
         with:
           name: frontend
           path: frontend
@@ -70,7 +70,7 @@ jobs:
       - name: Test with Maven
         run: mvn --no-transfer-progress test
 
-      - uses: actions/upload-artifact@v2.2.4
+      - uses: actions/upload-artifact@v2.3.1
         with:
           name: backend
           path: |
@@ -96,7 +96,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Download backend from the backend job
-        uses: actions/download-artifact@v2.0.10
+        uses: actions/download-artifact@v2.1.0
         with:
           name: backend
           path: target
@@ -106,4 +106,4 @@ jobs:
         with:
           dockerfile: docker/Dockerfile
           image-name: bom-base
-          tags: 0 0.2 0.2.1 v0.2.1 latest ${{ github.sha }}
+          tags: 0 0.3 0.3.0 v0.3.0 latest ${{ github.sha }}

.github/workflows/licenses.yml

@@ -7,18 +7,31 @@ jobs:
   scanLicenses:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-java@v2
-        with:
-          java-version: '11.0.1'
-          distribution: 'zulu'
+    - uses: actions/checkout@v2
+    - uses: actions/setup-java@v2
+      with:
+        java-version: '11.0.13'
+        distribution: 'zulu'
 
-      - name: Create spdx-file
-        id: spdx-builder
-        uses: philips-software/[email protected]
-        with:
-          project: bom-base
-      - uses: actions/[email protected]
-        with:
-          name: licenses
-          path: ${{ steps.spdx-builder.outputs.spdx-file }}
+    - name: Create dependencies list
+      run: |
+        ./mvnw dependency:tree 
+        ./mvnw dependency:tree > dependencies.txt
+
+    - name: Show dependencies.txt
+      run: |
+        cat dependencies.txt
+
+    - name: Create spdx-file
+      id: spdx-builder
+      uses: philips-software/[email protected]
+      with:
+        project: bom-base
+        mode: 'tree'
+        tree: dependencies.txt
+        format: maven
+
+    - uses: actions/[email protected]
+      with:
+        name: licenses
+        path: ${{ steps.spdx-builder.outputs.spdx-file }}

.spdx-builder.yml

@@ -4,13 +4,6 @@ document:
   comment:
   key:
   namespace: "https://research.philips.com/bom-base"
-projects:
-  - id: "Maven:com.philips.research:BOM-base:0.2.1"
-    purl: "pkg:maven/philips/[email protected]"
-    excluded:
-      - "test*"
-      - "development*"
-      - "runtime*"
-      - "annotation*"
-      - "archives"
-  - id: "Pub::ui/pubspec.yaml:"
+internal:
+  - com.philips.research:*
+

CONTRIBUTING.md

@@ -51,9 +51,7 @@ This needs to be improved in the future, but for now:
 Change version into new version in file / linenumber:
 
 - `pom.xml` : line 13.
-- `.spdx-builder.yml` : line 8.
-- `.spdx-builder.yml` : line 9.
-- `.github/workflows/build.yml` : line 95. (docker tags)
+- `.github/workflows/build.yml` : line 109. (docker tags)
 
 Commit changes:
 

pom.xml

@@ -5,12 +5,12 @@
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
-        <version>2.5.5</version>
+        <version>2.6.1</version>
         <relativePath/> <!-- lookup parent from repository -->
     </parent>
     <groupId>com.philips.research</groupId>
     <artifactId>BOM-base</artifactId>
-    <version>0.2.1-SNAPSHOT</version>
+    <version>0.3.0-SNAPSHOT</version>
     <name>BOM-base</name>
     <description>FOSS metadata repository</description>
 
@@ -79,7 +79,7 @@
         <dependency>
             <groupId>nl.jqno.equalsverifier</groupId>
             <artifactId>equalsverifier</artifactId>
-            <version>3.7.1</version>
+            <version>3.8.1</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

scripts/analyze-all.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+###################################################################
+# Script to batch-analyze Black Duck component licenses against 3rd
+# party metadata.
+#
+# Optional $1 parameter is the URL of a BOM-Base server
+#
+# Recipe of this script:
+#
+# The listed project versions are extracted from Black Duck as SPDX
+# files and a tree of Package URLs. The tree of Package URLs is used
+# to create a second SPDX file using independent metadata from ClearlyDefined
+# and package management repositories. The declared licenses from both
+# sources are compared, resulting in a list of all packages and a diff list of
+# packages where the declared licenses don't match. These lists are merged
+# across all projects, and separate lists are extracted for specific package
+# types (e.g. Maven, NuGet, NPM). These lists provide an indications of the
+# license diffs per type of development environment.
+###################################################################
+set -e
+
+
+BOM_BASE_URL="http://localhost:8080"
+if [ "$1" ]; then
+    BOM_BASE_URL=$1
+fi
+
+readonly projects=(
+  'blackduck-project|blackduck-version|analyse-output-filename'
+  'blackduck-project-2|blackduck-version-2|analyse-output-filename-2'
+)
+
+function info () {
+  echo " +================================================================================================ "
+  echo " | ${1}"
+  echo " +------------------------------------------------------------------------------------------------ "
+}
+
+# Start with and empty list of projects
+ALL=()
+
+# Processes a single project
+# $1 is the Black Duck project
+# $2 is the Black Duck project version
+# $3 is the nick name
+# $4 is the BomBase url
+function blackduck () {
+    ./analyze.sh "$1" "$2" "$3" "$4"
+    ALL+=( $3 )
+}
+
+function scan_projects () {
+    local project version filename
+    for fields in "${projects[@]}"
+    do
+        IFS=$'|' read -r project version filename <<< "$fields"
+        # Collect and process per Black Duck project version
+        blackduck "$project" "$version" "$filename" "$BOM_BASE_URL"
+    done
+}
+
+scan_projects
+
+# Truncate totals files
+rm -f packages.csv
+rm -f diffs.csv
+
+# Merge packages of all projects
+for PROJECT in "${ALL[@]}"; do
+    cat "$PROJECT-all.csv" >> packages.csv
+    cat "$PROJECT-diff.csv" >> diffs.csv
+done
+
+sort -u "packages.csv" -o "packages.csv"
+sort -u "diffs.csv" -o "diffs.csv"
+
+# Extracts a package type from the diffs
+# $1 is the group name
+function extractType () {
+    grep "^\"pkg:$1/" packages.csv > "$1-packages.csv"
+    PACKAGES=$(grep -c ^ "$1-packages.csv")
+    grep "^\"pkg:$1/" diffs.csv > "$1-diffs.csv"
+    DIFFS=$(grep -c ^ "$1-diffs.csv")
+    echo "Type $1 totals $PACKAGES with $DIFFS license mismatches"
+}
+
+extractType nuget
+extractType npm
+extractType maven
+extractType pypi

scripts/analyze.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+
+###################################################################
+# Script to batch-analyze Black Duck component licenses against 3rd
+# party metadata.
+#
+# Mandatory $1 Blackduck project
+# Mandatory $2 Blackduck version
+# Mandatory $3 filename
+# Optional $4 parameter is the URL of a BOM-Base server
+#
+# Recipe of this script:
+#
+# The listed project versions are extracted from Black Duck as SPDX
+# files and a tree of Package URLs. The tree of Package URLs is used
+# to create a second SPDX file using independent metadata from ClearlyDefined
+# and package management repositories. The declared licenses from both
+# sources are compared, resulting in a list of all packages and a diff list of
+# packages where the declared licenses don't match.
+###################################################################
+set -e
+
+function info () {
+  echo " +================================================================================================ "
+  echo " | ${1}"
+  echo " +------------------------------------------------------------------------------------------------ "
+}
+
+function checkEnvironmentVariables () {
+  info "Check Environment Variables"
+  if [ -z "$BLACKDUCK_API_TOKEN" ]; then
+    echo " | BLACKDUCK_API_TOKEN is not set"
+    exit 1
+  fi
+
+  if [ -z "$BLACKDUCK_URL" ]; then
+    echo " | BLACKDUCK_URL is not set"
+    exit 1
+  fi
+}
+
+checkEnvironmentVariables
+
+BOM_BASE_URL="http://localhost:8080"
+if [ "$4" ]; then
+    BOM_BASE_URL=$4
+fi
+
+function checkBOMBaseAvailable () {
+  info "Check BOMBase is running"
+  # Aborts if BOM-Base server cannot be reached
+  if [ $(curl --write-out '%{http_code}' --head --silent --output /dev/null $BOM_BASE_URL/packages) -ne 200 ]; then
+      echo " | Could not reach BOM-Base at URL $BOM_BASE_URL (see https://github.com/philips-software/bom-base)"
+      exit 1
+  fi
+}
+
+checkBOMBaseAvailable
+
+# Aborts if the specified tool is not installed
+# $1 is the command name
+# $2 is a link to installation instructions for the tool
+function checkInstalled () {
+    if ! type $1 &> /dev/null; then
+        echo "Requires '$1' (see $2)"
+        exit 1
+    fi
+}
+
+info "Check Installed tools"
+
+# Ensure the required tools are installed
+checkInstalled spdx-builder https://github.com/philips-software/spdx-builder
+checkInstalled bompare https://github.com/philips-labs/bompare
+
+# Extracts an SPDX and tree file from a Black Duck project.
+# $1 is the Black Duck project
+# $2 is the Black Duck project version
+# $3 is the nick name
+function exportFromBD () {
+    if [ -f "$3.spdx" ] && [ -f "$3.tree" ]; then
+        echo " | Skipping Black Duck export for $1 $2; files already exist"
+    else
+        echo " | Exporting $3 could take a while..."
+        spdx-builder blackduck $1 $2 -o "$3.spdx" --tree > "$3.tree" 2>&1
+    fi
+}
+
+# Builds an SPDX file from a tree file.
+# $1 is the nick name
+function buildFromTree () {
+    echo " | Building $1 from tree..."
+    cat $1.tree | spdx-builder tree --format purl --config "shared.yml" --bombase $BOM_BASE_URL --force -o "$1-tree.spdx"
+}
+
+# Lists differences between Black Duck and tree SPDX files.
+# $1 is the nick name
+function licensesDiff () {
+    bompare licenses --spdx-tag-value "$1.spdx" --spdx-tag-value "$1-tree.spdx" --out "$1-diff.csv" --diffOnly
+}
+
+# Lists all packages in the Black Duck SPDX output.
+# $1 is the nick name
+function listPackages() {
+    echo " | list Packages"
+    bompare bom --spdx-tag-value "$1.spdx" --out "$1-all.csv"
+}
+
+# Processes a single project
+# $1 is the Black Duck project
+# $2 is the Black Duck project version
+# $3 is the nick name
+function blackduck () {
+    info "Blackduck function "$1" "$2" "$3""
+    exportFromBD "$1" "$2" "$3"
+    listPackages "$3"
+    buildFromTree "$3"
+    licensesDiff "$3"
+}
+
+# Collect and process per Black Duck project version
+blackduck "$1" "$2" "$3"