Merge pull request #24 from nexB/23-recursion-error

Do not trigger a RecursionError #23
aboutcode-org · Jun 11, 2021 · 452c2b5 · 452c2b5
2 parents 4b627bd + 02ec56d
commit 452c2b5
Show file tree

Hide file tree

Showing 6 changed files with 371 additions and 10 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -5,6 +5,14 @@ vNext
 -----
 
 
+Version 21.6.11
+---------------
+
+- Do not fail if a Codebase does not have a common shared root #23
+- Consider all Resource attributes when building a VirtualCodebase #23
+- Do not ignore by default sccs and rcs dirs https://github.com/nexB/scancode-toolkit/issues/1422
+
+
 Version 21.6.10
 ---------------
 

diff --git a/src/commoncode/ignore.py b/src/commoncode/ignore.py
@@ -184,7 +184,6 @@ def get_ignores(location, include_defaults=True):
 
     'CVS': 'Default ignore: CVS artifact',
     '.cvsignore': 'Default ignore: CVS config artifact',
-    '*/SCCS': 'Default ignore: CVS artifact',
 
     '*/_MTN': 'Default ignore: Monotone artifact',
     '*/_darcs': 'Default ignore: Darcs artifact',

diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py
@@ -12,6 +12,7 @@
 import posixpath
 import traceback
 import sys
+from pathlib import Path
 
 from collections import deque
 from functools import partial
@@ -1530,7 +1531,7 @@ def _get_or_create_parent(self, path, parent_by_path):
         Note: the root path and root Resource must already be in
         `parent_by_path` or else this function does not work.
         """
-        parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip("/")
+        parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip('/')
         existing_parent = parent_by_path.get(parent_path)
         if existing_parent:
             return existing_parent
@@ -1548,6 +1549,13 @@ def _get_or_create_parent(self, path, parent_by_path):
         parent_by_path[parent_path] = parent_resource
         return parent_resource
 
+    def _set_new_root_directory(self, resources_data, new_root_directory_path):
+        for resource_data in resources_data:
+            resource_path = Path(resource_data['path'])
+            new_resource_path = Path(new_root_directory_path)
+            new_resource_path = new_resource_path.joinpath(resource_path)
+            resource_data['path'] = str(new_resource_path)
+
     def _populate(self, scan_data):
         """
         Populate this codebase with Resource objects.
@@ -1645,6 +1653,21 @@ def _populate(self, scan_data):
             sample_resource_path = sample_resource_data['path']
             sample_resource_path = sample_resource_path.strip('/')
             root_path = sample_resource_path.split('/')[0]
+
+            # Check to see if the Resources from the scan we received has a common root directory.
+            for resource_data in resources_data:
+                resource_path = resource_data.get('path')
+                resource_path = resource_path.strip('/')
+                resource_root_path = resource_path.split('/')[0]
+                # If not, set a common root directory for all Resources.
+                if resource_root_path != root_path:
+                    self._set_new_root_directory(
+                        resources_data=resources_data,
+                        new_root_directory_path='virtual_root'
+                    )
+                    root_path = 'virtual_root'
+                    break
+
         root_name = root_path
         root_is_file = False
         root_data = self._create_empty_resource_data()

diff --git a/tests/data/resource/virtual_codebase/docker-hello-world.json b/tests/data/resource/virtual_codebase/docker-hello-world.json
@@ -0,0 +1,100 @@
+{
+"headers": [
+{
+  "tool_name": "scanpipe",
+  "tool_version": "21.5.12",
+  "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n",
+  "uuid": "2f8e9a35-048e-4d9f-ae56-feea8863d6eb",
+  "created_date": "2021-06-10T13:56:44.418Z",
+  "input_files": [
+    "hello_world.tar"
+  ],
+  "runs": [
+    {
+      "pipeline_name": "docker",
+      "description": "A pipeline to analyze a Docker image.",
+      "uuid": "106296b1-6334-4393-9f1a-209459954812",
+      "created_date": "2021-06-10T13:56:44.421302Z",
+      "task_id": "c851f2e7-7cc1-4952-8d93-1cdee4a147bd",
+      "task_start_date": "2021-06-10T13:56:44.434313Z",
+      "task_end_date": "2021-06-10T13:56:44.819773Z",
+      "task_exitcode": 0,
+      "task_output": "",
+      "log": "2021-06-10 13:56:44.43 Pipeline [docker] starting\n2021-06-10 13:56:44.43 Step [extract_images] starting\n2021-06-10 13:56:44.44 Step [extract_images] completed in 0.00 seconds\n2021-06-10 13:56:44.44 Step [extract_layers] starting\n2021-06-10 13:56:44.44 Step [extract_layers] completed in 0.00 seconds\n2021-06-10 13:56:44.45 Step [find_images_linux_distro] starting\n2021-06-10 13:56:44.45 Step [find_images_linux_distro] completed in 0.00 seconds\n2021-06-10 13:56:44.45 Step [collect_images_information] starting\n2021-06-10 13:56:44.45 Step [collect_images_information] completed in 0.00 seconds\n2021-06-10 13:56:44.46 Step [collect_and_create_codebase_resources] starting\n2021-06-10 13:56:44.46 Step [collect_and_create_codebase_resources] completed in 0.01 seconds\n2021-06-10 13:56:44.47 Step [collect_and_create_system_packages] starting\n2021-06-10 13:56:44.47 Step [collect_and_create_system_packages] completed in 0.00 seconds\n2021-06-10 13:56:44.47 Step [tag_uninteresting_codebase_resources] starting\n2021-06-10 13:56:44.48 Step [tag_uninteresting_codebase_resources] completed in 0.00 seconds\n2021-06-10 13:56:44.48 Step [tag_empty_files] starting\n2021-06-10 13:56:44.48 Step [tag_empty_files] completed in 0.00 seconds\n2021-06-10 13:56:44.48 Step [scan_for_application_packages] starting\n2021-06-10 13:56:44.53 Step [scan_for_application_packages] completed in 0.05 seconds\n2021-06-10 13:56:44.54 Step [scan_for_files] starting\n2021-06-10 13:56:44.80 Step [scan_for_files] completed in 0.26 seconds\n2021-06-10 13:56:44.80 Step [analyze_scanned_files] starting\n2021-06-10 13:56:44.81 Step [analyze_scanned_files] completed in 0.00 seconds\n2021-06-10 13:56:44.81 Step [tag_not_analyzed_codebase_resources] starting\n2021-06-10 13:56:44.81 Step [tag_not_analyzed_codebase_resources] completed in 0.00 seconds\n2021-06-10 13:56:44.81 Pipeline completed\n",
+      "execution_time": 0
+    }
+  ],
+  "extra_data": {
+    "images": [
+      {
+        "os": "linux",
+        "tags": [],
+        "author": null,
+        "distro": null,
+        "labels": {},
+        "sha256": null,
+        "comment": null,
+        "created": "2021-03-05T23:25:25.230064203Z",
+        "history": [
+          {
+            "created": "2021-03-05T23:25:25.052495292Z",
+            "created_by": "/bin/sh -c #(nop) COPY file:7bf12aab75c3867a023fe3b8bd6d113d43a4fcc415f3cc27cbcf0fff37b65a02 in / "
+          },
+          {
+            "created": "2021-03-05T23:25:25.230064203Z",
+            "created_by": "/bin/sh -c #(nop)  CMD [\"/hello\"]",
+            "empty_layer": true
+          }
+        ],
+        "variant": null,
+        "image_id": "d1165f2212346b2bab48cb01c1e39ee8ad1be46b87873d9ca7a4e434980a7726",
+        "os_version": null,
+        "architecture": "amd64",
+        "image_format": "docker",
+        "config_digest": "sha256:d1165f2212346b2bab48cb01c1e39ee8ad1be46b87873d9ca7a4e434980a7726",
+        "docker_version": "19.03.12"
+      }
+    ]
+  }
+}],
+"packages": [
+],
+"files": [
+{
+  "for_packages": [],
+  "compliance_alert": "",
+  "path": "/hello_world.tar-extract/f22b99068db93900abe17f7f5e09ec775c2826ecfe9db961fea68293744144bd/hello",
+  "size": 13336,
+  "sha1": "ab2ceef6fe98ef7acd088f5d79034dc859ff0d27",
+  "md5": "773e022bbbd0b4cc88edb6c7d12daf5d",
+  "copyrights": [],
+  "holders": [],
+  "authors": [],
+  "licenses": [],
+  "license_expressions": [],
+  "emails": [],
+  "urls": [
+    {
+      "url": "https://hub.docker.com/",
+      "end_line": 19,
+      "start_line": 19
+    },
+    {
+      "url": "https://docs.docker.com/get-started/",
+      "end_line": 21,
+      "start_line": 21
+    }
+  ],
+  "status": "no-licenses",
+  "type": "file",
+  "extra_data": {},
+  "name": "hello",
+  "extension": "",
+  "programming_language": "",
+  "mime_type": "application/x-executable",
+  "file_type": "ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, stripped",
+  "is_binary": true,
+  "is_text": false,
+  "is_archive": false
+}]
+}
diff --git a/tests/data/resource/virtual_codebase/zephyr-binary.json b/tests/data/resource/virtual_codebase/zephyr-binary.json
@@ -0,0 +1,220 @@
+{
+"headers": [
+{
+  "tool_name": "scanpipe",
+  "tool_version": "21.5.12",
+  "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n",
+  "uuid": "d0d9bf3d-980a-4822-ac5b-e4949470499d",
+  "created_date": "2021-06-10T23:17:30.522Z",
+  "input_files": [
+    "SBOM-Plugfest.zip"
+  ],
+  "runs": [
+    {
+      "pipeline_name": "scan_codebase",
+      "description": "A pipeline to scan a codebase with ScanCode-toolkit.\n\nThe input files are copied to the project codebase/ directory and extracted\nin place before running the scan.\nAlternatively, the code can be manually copied to the project codebase/\ndirectory.",
+      "uuid": "13efbd78-3921-4895-b4bd-e0b393b92ec3",
+      "created_date": "2021-06-10T23:17:30.526728Z",
+      "task_id": "f68d383a-4910-4e05-ad64-82ef1f0b5888",
+      "task_start_date": "2021-06-10T23:17:30.546024Z",
+      "task_end_date": "2021-06-10T23:17:36.396574Z",
+      "task_exitcode": 0,
+      "task_output": "",
+      "log": "2021-06-10 23:17:30.54 Pipeline [scan_codebase] starting\n2021-06-10 23:17:30.54 Step [copy_inputs_to_codebase_directory] starting\n2021-06-10 23:17:30.55 Step [copy_inputs_to_codebase_directory] completed in 0.00 seconds\n2021-06-10 23:17:30.55 Step [run_extractcode] starting\n2021-06-10 23:17:30.83 Step [run_extractcode] completed in 0.28 seconds\n2021-06-10 23:17:30.84 Step [run_scancode] starting\n2021-06-10 23:17:36.33 Step [run_scancode] completed in 5.49 seconds\n2021-06-10 23:17:36.34 Step [build_inventory_from_scan] starting\n2021-06-10 23:17:36.37 Step [build_inventory_from_scan] completed in 0.03 seconds\n2021-06-10 23:17:36.37 Step [csv_output] starting\n2021-06-10 23:17:36.39 Step [csv_output] completed in 0.02 seconds\n2021-06-10 23:17:36.39 Pipeline completed\n",
+      "execution_time": 5
+    }
+  ],
+  "extra_data": {}
+}],
+"packages": [
+],
+"files": [
+{
+  "for_packages": [],
+  "compliance_alert": "",
+  "path": "SBOM-Plugfest.zip",
+  "size": 1762970,
+  "sha1": "db3fa3422912685484860aec79960f3ef3de95a8",
+  "md5": "8a9423ff930d0afd9399600f351b0507",
+  "copyrights": [],
+  "holders": [],
+  "authors": [],
+  "licenses": [],
+  "license_expressions": [],
+  "emails": [],
+  "urls": [],
+  "status": "",
+  "type": "file",
+  "extra_data": {},
+  "name": "SBOM-Plugfest.zip",
+  "extension": ".zip",
+  "programming_language": "",
+  "mime_type": "application/zip",
+  "file_type": "Zip archive data, at least v2.0 to extract",
+  "is_binary": true,
+  "is_text": false,
+  "is_archive": true
+},
+{
+  "for_packages": [],
+  "compliance_alert": "",
+  "path": "SBOM-Plugfest.zip-extract",
+  "size": 0,
+  "sha1": "",
+  "md5": "",
+  "copyrights": [],
+  "holders": [],
+  "authors": [],
+  "licenses": [],
+  "license_expressions": [],
+  "emails": [],
+  "urls": [],
+  "status": "",
+  "type": "directory",
+  "extra_data": {},
+  "name": "SBOM-Plugfest.zip-extract",
+  "extension": "",
+  "programming_language": "",
+  "mime_type": "",
+  "file_type": "",
+  "is_binary": false,
+  "is_text": false,
+  "is_archive": false
+},
+{
+  "for_packages": [],
+  "compliance_alert": "",
+  "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest",
+  "size": 0,
+  "sha1": "",
+  "md5": "",
+  "copyrights": [],
+  "holders": [],
+  "authors": [],
+  "licenses": [],
+  "license_expressions": [],
+  "emails": [],
+  "urls": [],
+  "status": "",
+  "type": "directory",
+  "extra_data": {},
+  "name": "SBOM-Plugfest",
+  "extension": "",
+  "programming_language": "",
+  "mime_type": "",
+  "file_type": "",
+  "is_binary": false,
+  "is_text": false,
+  "is_archive": false
+},
+{
+  "for_packages": [],
+  "compliance_alert": "",
+  "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/Readme.md",
+  "size": 238,
+  "sha1": "b3edec38166cb4747ce67e7e229cbe9279e011c3",
+  "md5": "2c2fdcabb9e3a46bbc60f9e235ec0389",
+  "copyrights": [],
+  "holders": [],
+  "authors": [],
+  "licenses": [],
+  "license_expressions": [],
+  "emails": [],
+  "urls": [
+    {
+      "url": "https://github.com/LibreSolar/charge-controller-firmware/tree/sbom-plugfest",
+      "end_line": 7,
+      "start_line": 7
+    }
+  ],
+  "status": "",
+  "type": "file",
+  "extra_data": {},
+  "name": "Readme.md",
+  "extension": ".md",
+  "programming_language": "",
+  "mime_type": "text/plain",
+  "file_type": "ASCII text",
+  "is_binary": false,
+  "is_text": true,
+  "is_archive": false
+},
+{
+  "for_packages": [],
+  "compliance_alert": "",
+  "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.bin",
+  "size": 95600,
+  "sha1": "3e5510346a723c4ecdd698330436d1d89f2e6428",
+  "md5": "fdc8b20a54f7960b204abce20f8dae98",
+  "copyrights": [],
+  "holders": [],
+  "authors": [],
+  "licenses": [],
+  "license_expressions": [],
+  "emails": [],
+  "urls": [],
+  "status": "",
+  "type": "file",
+  "extra_data": {},
+  "name": "zephyr.bin",
+  "extension": ".bin",
+  "programming_language": "",
+  "mime_type": "application/octet-stream",
+  "file_type": "data",
+  "is_binary": true,
+  "is_text": false,
+  "is_archive": false
+},
+{
+  "for_packages": [],
+  "compliance_alert": "",
+  "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.elf",
+  "size": 1397388,
+  "sha1": "a0c9fca9a551a935026c86fb56c4af563f0a926f",
+  "md5": "b7f937d8a1fbd71a0a48f16c845ed36e",
+  "copyrights": [],
+  "holders": [],
+  "authors": [],
+  "licenses": [],
+  "license_expressions": [],
+  "emails": [],
+  "urls": [],
+  "status": "",
+  "type": "file",
+  "extra_data": {},
+  "name": "zephyr.elf",
+  "extension": ".elf",
+  "programming_language": "",
+  "mime_type": "application/x-executable",
+  "file_type": "ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), statically linked, with debug_info, not stripped",
+  "is_binary": true,
+  "is_text": false,
+  "is_archive": false
+},
+{
+  "for_packages": [],
+  "compliance_alert": "",
+  "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.hex",
+  "size": 269060,
+  "sha1": "c70fc409d4003e437ea2e84d9546de8083333ab2",
+  "md5": "2505d99bb160d8b05e8892026209923d",
+  "copyrights": [],
+  "holders": [],
+  "authors": [],
+  "licenses": [],
+  "license_expressions": [],
+  "emails": [],
+  "urls": [],
+  "status": "",
+  "type": "file",
+  "extra_data": {},
+  "name": "zephyr.hex",
+  "extension": ".hex",
+  "programming_language": "",
+  "mime_type": "text/plain",
+  "file_type": "ASCII text, with CRLF line terminators",
+  "is_binary": false,
+  "is_text": true,
+  "is_archive": false
+}]
+}