From a57d26bbc9d3c5e1ddee5a2adde5363756590efc Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 28 Oct 2024 21:35:01 +0400 Subject: [PATCH 01/12] Bump version for v34.8.2 release Signed-off-by: tdruez Signed-off-by: Alok Kumar --- CHANGELOG.rst | 2 +- scancodeio/__init__.py | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0885a0ef..d279f4132 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Changelog ========= -v34.8.2 (unreleased) +v34.8.2 (2024-10-28) -------------------- - Add ``android_analysis`` to ``extra_requires``. This installs the package diff --git a/scancodeio/__init__.py b/scancodeio/__init__.py index 375b6671b..c1f2739d7 100644 --- a/scancodeio/__init__.py +++ b/scancodeio/__init__.py @@ -28,7 +28,7 @@ import git -VERSION = "34.8.1" +VERSION = "34.8.2" PROJECT_DIR = Path(__file__).resolve().parent ROOT_DIR = PROJECT_DIR.parent diff --git a/setup.cfg b/setup.cfg index d8bb937c8..b76a30bea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scancodeio -version = 34.8.1 +version = 34.8.2 license = Apache-2.0 description = Automate software composition analysis pipelines long_description = file:README.rst From 842ee85042f2a51b6fae2fcd2ddc2f6fb0e4ca5e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 30 Oct 2024 20:16:32 +0530 Subject: [PATCH 02/12] Update sctk version to v32.3.0 (#1418) * Bump scancode-toolkit version to v32.3.0 Signed-off-by: Ayan Sinha Mahapatra * Rename dependency and license match attributes * Rename is_resolved to is_pinned for dependencies * Rename spdx_license_expression to license_expression_spdx for license matches Signed-off-by: Ayan Sinha Mahapatra * Regen scancode scan fixtures Signed-off-by: Ayan Sinha Mahapatra --------- Signed-off-by: Ayan Sinha Mahapatra Signed-off-by: Alok Kumar --- scanpipe/admin.py | 4 +- scanpipe/api/serializers.py | 4 +- scanpipe/filters.py | 8 +- ...ename_discovered_dependencies_attribute.py | 44 +++ scanpipe/models.py | 6 +- scanpipe/pipelines/find_vulnerabilities.py | 2 +- scanpipe/pipes/purldb.py | 4 +- scanpipe/pipes/resolve.py | 2 +- .../templates/scanpipe/dependency_list.html | 4 +- .../templates/scanpipe/project_charts.html | 6 +- scanpipe/tests/__init__.py | 6 +- .../data/asgiref/asgiref-3.3.0.spdx.json | 42 +-- .../data/asgiref/asgiref-3.3.0_fixtures.json | 137 +++---- ...asgiref-3.3.0_load_inventory_expected.json | 56 +-- .../asgiref-3.3.0_scanpipe_output.json | 122 +++--- .../asgiref/asgiref-3.3.0_toolkit_scan.json | 126 +++---- .../asgiref-3.3.0_walk_test_fixtures.json | 137 +++---- .../data/cyclonedx/asgiref-3.3.0.cdx.json | 22 +- .../tests/data/d2d/about_files/expected.json | 2 +- .../tests/data/d2d/flume-ng-node-d2d.json | 158 ++++---- .../resolved_dependencies_cocoapods.json | 122 +++--- ...ved_dependencies_npm_inspect_packages.json | 40 +- .../resolved_dependencies_nuget.json | 336 ++++++++--------- .../resolved_dependencies_pip.json | 58 +-- ..._dependencies_poetry_inspect_packages.json | 54 +-- ...d_dependencies_swift_inspect_packages.json | 40 +- .../docker/alpine_3_15_4_scan_codebase.json | 28 +- .../data/docker/centos_scan_codebase.json | 346 ++++++++++-------- .../data/docker/debian_scan_codebase.json | 14 +- .../gcr_io_distroless_base_scan_codebase.json | 26 +- .../openpdf-parent-1.3.11_scan_package.json | 50 +-- .../data/manifests/package.expected.json | 12 +- .../request_post_response.json | 2 +- .../rootfs/basic-rootfs_root_filesystems.json | 14 +- ...-0.6.0-py3-none-any.whl_scan_codebase.json | 50 +-- .../scancode/is-npm-1.0.0_scan_codebase.json | 10 +- .../scancode/is-npm-1.0.0_scan_package.json | 16 +- .../is-npm-1.0.0_scan_package_summary.json | 4 +- .../data/scancode/is-npm-1.0.0_summary.json | 4 +- .../multiple-is-npm-1.0.0_scan_package.json | 28 +- ...ple-is-npm-1.0.0_scan_package_summary.json | 4 +- .../scancode/package_assembly_codebase.json | 28 +- scanpipe/tests/pipes/test_input.py | 2 +- scanpipe/tests/pipes/test_output.py | 2 +- scanpipe/tests/pipes/test_resolve.py | 2 +- scanpipe/tests/test_api.py | 2 +- scanpipe/tests/test_filters.py | 6 +- scanpipe/tests/test_models.py | 4 +- scanpipe/views.py | 8 +- setup.cfg | 2 +- 50 files changed, 1150 insertions(+), 1056 deletions(-) create mode 100644 scanpipe/migrations/0068_rename_discovered_dependencies_attribute.py diff --git a/scanpipe/admin.py b/scanpipe/admin.py index 628ef564e..f8272c44d 100644 --- a/scanpipe/admin.py +++ b/scanpipe/admin.py @@ -152,7 +152,7 @@ class DiscoveredDependencyAdmin(ScanPipeBaseAdmin): "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "project", ] @@ -171,7 +171,7 @@ class DiscoveredDependencyAdmin(ScanPipeBaseAdmin): "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", ] ordering = ["project", "dependency_uid"] diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index 5da4f1186..ded09bef5 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -268,7 +268,7 @@ def get_discovered_dependencies_summary(self, project): "total": base_qs.count(), "is_runtime": base_qs.filter(is_runtime=True).count(), "is_optional": base_qs.filter(is_optional=True).count(), - "is_resolved": base_qs.filter(is_resolved=True).count(), + "is_pinned": base_qs.filter(is_pinned=True).count(), } def get_codebase_relations_summary(self, project): @@ -448,7 +448,7 @@ class Meta: "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "dependency_uid", "for_package_uid", diff --git a/scanpipe/filters.py b/scanpipe/filters.py index a440bba5e..8c8be0eeb 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -739,7 +739,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "datasource_id", "is_vulnerable", @@ -760,7 +760,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "for_package", "resolved_to_package", @@ -775,7 +775,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): datasource_id = ModelFieldValuesFilter() is_runtime = StrictBooleanFilter() is_optional = StrictBooleanFilter() - is_resolved = StrictBooleanFilter() + is_pinned = StrictBooleanFilter() is_direct = StrictBooleanFilter() is_vulnerable = IsVulnerable(field_name="affected_by_vulnerabilities") @@ -794,7 +794,7 @@ class Meta: "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "datasource_id", "is_vulnerable", diff --git a/scanpipe/migrations/0068_rename_discovered_dependencies_attribute.py b/scanpipe/migrations/0068_rename_discovered_dependencies_attribute.py new file mode 100644 index 000000000..f8a03e3b0 --- /dev/null +++ b/scanpipe/migrations/0068_rename_discovered_dependencies_attribute.py @@ -0,0 +1,44 @@ +# Generated by Django 5.0.7 on 2024-10-21 07:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scanpipe", "0067_discoveredpackage_notes"), + ] + + operations = [ + migrations.AlterModelOptions( + name="discovereddependency", + options={ + "ordering": [ + "-is_runtime", + "-is_pinned", + "is_optional", + "dependency_uid", + "for_package", + "datafile_resource", + "datasource_id", + ], + "verbose_name": "discovered dependency", + "verbose_name_plural": "discovered dependencies", + }, + ), + migrations.RemoveIndex( + model_name="discovereddependency", + name="scanpipe_di_is_reso_10570c_idx", + ), + migrations.RenameField( + model_name="discovereddependency", + old_name="is_resolved", + new_name="is_pinned", + ), + migrations.AddIndex( + model_name="discovereddependency", + index=models.Index( + fields=["is_pinned"], name="scanpipe_di_is_pinn_5667b2_idx" + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 9dc86428f..fd84836f0 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3700,7 +3700,7 @@ class DiscoveredDependency( default=False, help_text=_("True if this dependency is an optional dependency"), ) - is_resolved = models.BooleanField( + is_pinned = models.BooleanField( default=False, help_text=_( "True if this dependency version requirement has been pinned " @@ -3722,7 +3722,7 @@ class Meta: verbose_name_plural = "discovered dependencies" ordering = [ "-is_runtime", - "-is_resolved", + "-is_pinned", "is_optional", "dependency_uid", "for_package", @@ -3733,7 +3733,7 @@ class Meta: models.Index(fields=["scope"]), models.Index(fields=["is_runtime"]), models.Index(fields=["is_optional"]), - models.Index(fields=["is_resolved"]), + models.Index(fields=["is_pinned"]), models.Index(fields=["is_direct"]), ] constraints = [ diff --git a/scanpipe/pipelines/find_vulnerabilities.py b/scanpipe/pipelines/find_vulnerabilities.py index 7a6323b98..6a48812a2 100644 --- a/scanpipe/pipelines/find_vulnerabilities.py +++ b/scanpipe/pipelines/find_vulnerabilities.py @@ -62,7 +62,7 @@ def lookup_packages_vulnerabilities(self): def lookup_dependencies_vulnerabilities(self): """Check for vulnerabilities for each of the project's discovered dependency.""" - dependencies = self.project.discovereddependencies.filter(is_resolved=True) + dependencies = self.project.discovereddependencies.filter(is_pinned=True) vulnerablecode.fetch_vulnerabilities( packages=dependencies, ignore_set=self.project.ignored_vulnerabilities_set, diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index ba1d2181a..801e79325 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -289,7 +289,7 @@ def feed_purldb(packages, chunk_size, logger=logger.info): def get_unique_resolved_purls(project): """Return PURLs from project's resolved DiscoveredDependencies.""" - packages_resolved = project.discovereddependencies.filter(is_resolved=True) + packages_resolved = project.discovereddependencies.filter(is_pinned=True) distinct_results = packages_resolved.values("type", "namespace", "name", "version") @@ -300,7 +300,7 @@ def get_unique_resolved_purls(project): def get_unique_unresolved_purls(project): """Return PURLs from project's unresolved DiscoveredDependencies.""" packages_unresolved = project.discovereddependencies.filter( - is_resolved=False + is_pinned=False ).exclude(extracted_requirement="*") distinct_unresolved_results = packages_unresolved.values( diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index e841176de..55bf93197 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -151,7 +151,7 @@ def create_dependencies_from_packages_extra_data(project): resolved_to_package=resolved_to_package, datafile_resource=datafile_resource, is_runtime=True, - is_resolved=True, + is_pinned=True, is_direct=True, ) created_count += 1 diff --git a/scanpipe/templates/scanpipe/dependency_list.html b/scanpipe/templates/scanpipe/dependency_list.html index 92a3de0b6..b089df1a6 100644 --- a/scanpipe/templates/scanpipe/dependency_list.html +++ b/scanpipe/templates/scanpipe/dependency_list.html @@ -52,10 +52,10 @@ {{ dependency.is_optional }} - {{ dependency.is_resolved }} + {{ dependency.is_pinned }} - {{ dependency.is_direct }} + {{ dependency.is_direct }} {% if dependency.for_package %} diff --git a/scanpipe/templates/scanpipe/project_charts.html b/scanpipe/templates/scanpipe/project_charts.html index 9da3108ac..20db76198 100644 --- a/scanpipe/templates/scanpipe/project_charts.html +++ b/scanpipe/templates/scanpipe/project_charts.html @@ -35,7 +35,7 @@

-
+
{% endif %} @@ -82,7 +82,7 @@

{{ dependency_type|json_script:"dependency_type" }} {{ dependency_is_runtime|json_script:"dependency_is_runtime" }} {{ dependency_is_optional|json_script:"dependency_is_optional" }} - {{ dependency_is_resolved|json_script:"dependency_is_resolved" }} + {{ dependency_is_pinned|json_script:"dependency_is_pinned" }}