From f5a20a6d5eb2bcbf92dce959ffff209addb75a25 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Fri, 1 Nov 2024 18:24:40 +0400 Subject: [PATCH 1/8] Add a `list-pipelines` management command #1397 (#1428) * Add a `list-pipelines` management command #1397 Signed-off-by: tdruez * Refine the output of the list-pipelines command #1397 Signed-off-by: tdruez * Add unit test for the list-pipelines command #1397 Signed-off-by: tdruez --------- Signed-off-by: tdruez --- CHANGELOG.rst | 3 + docs/built-in-pipelines.rst | 2 +- docs/command-line-interface.rst | 38 ++++++++---- .../management/commands/list-pipelines.py | 59 +++++++++++++++++++ scanpipe/tests/test_commands.py | 26 ++++++++ 5 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 scanpipe/management/commands/list-pipelines.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0cc778bbc..601199700 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ v34.9.0 (unreleased) - Add ability to declared pipeline selected groups in create project REST API endpoint. https://github.com/aboutcode-org/scancode.io/issues/1426 +- Add a new ``list-pipelines`` management command. + https://github.com/aboutcode-org/scancode.io/issues/1397 + v34.8.3 (2024-10-30) -------------------- diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index e21920d44..92f1b4c41 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -45,7 +45,7 @@ Analyse Docker Windows Image .. _pipeline_collect_strings_gettext: Collect string with Xgettext (addon) ---------------------------------------------- +------------------------------------ .. autoclass:: scanpipe.pipelines.collect_strings_gettext.CollectStringsGettext() :members: :member-order: bysource diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index 7482abc75..0548b52e5 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -54,16 +54,23 @@ ScanPipe's own commands are listed under the ``[scanpipe]`` section:: $ scanpipe --help ... [scanpipe] - add-input - add-pipeline - archive-project - create-project - delete-project - execute - list-project - output - show-pipeline - status + add-input + add-pipeline + archive-project + check-compliance + create-project + create-user + delete-project + execute + flush-projects + list-pipelines + list-project + output + purldb-scan-worker + reset-project + run + show-pipeline + status `$ scanpipe --help` @@ -127,6 +134,17 @@ Optional arguments: Pipelines are added and are executed in order. +`$ scanpipe list-pipeline [--verbosity {0,1,2,3}]` +-------------------------------------------------- + +Displays a list of available pipelines. +Use ``--verbosity=2`` to include details of each pipeline's steps." + +Optional arguments: + +- ``--verbosity {0,1,2}`` Verbosity level. + + `$ scanpipe list-project [--search SEARCH] [--include-archived]` ---------------------------------------------------------------- diff --git a/scanpipe/management/commands/list-pipelines.py b/scanpipe/management/commands/list-pipelines.py new file mode 100644 index 000000000..e025fdf76 --- /dev/null +++ b/scanpipe/management/commands/list-pipelines.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + + +from django.apps import apps +from django.core.management.base import BaseCommand + +scanpipe_app = apps.get_app_config("scanpipe") + + +class Command(BaseCommand): + help = ( + "Displays a list of available pipelines. " + "Use --verbosity=2 to include details of each pipeline's steps." + ) + + def handle(self, *args, **options): + verbosity = options["verbosity"] + + for module_name, pipeline_class in scanpipe_app.pipelines.items(): + msg = self.style.HTTP_INFO(module_name) + if pipeline_class.is_addon: + msg += " (addon)" + self.stdout.write(msg) + pipeline_info = pipeline_class.get_info() + if verbosity >= 1: + self.stdout.write(pipeline_info["summary"]) + + if verbosity >= 2: + steps = pipeline_info["steps"] + for step_info in steps: + step_name = step_info["name"] + step_doc = step_info["doc"].replace("\n", "\n ") + step_groups = step_info["groups"] + self.stdout.write(f" > [{step_name}]: {step_doc}") + if step_groups: + self.stdout.write(f" {{Group}}: {','.join(step_groups)}") + + if verbosity >= 1: + self.stdout.write("\n") diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 88650afeb..65689ddb7 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -451,6 +451,32 @@ def test_scanpipe_management_command_status(self): ) self.assertIn(expected, output) + def test_scanpipe_management_command_list_pipelines(self): + options = [] + out = StringIO() + call_command("list-pipelines", *options, stdout=out) + output = out.getvalue() + self.assertIn("analyze_docker_image", output) + self.assertIn("Analyze Docker images.", output) + self.assertIn("(addon)", output) + self.assertNotIn("[extract_images]", output) + self.assertNotIn("Extract images from input tarballs.", output) + + options = ["--verbosity=2"] + out = StringIO() + call_command("list-pipelines", *options, stdout=out) + output = out.getvalue() + self.assertIn("[extract_images]", output) + self.assertIn("Extract images from input tarballs.", output) + + options = ["--verbosity=0"] + out = StringIO() + call_command("list-pipelines", *options, stdout=out) + output = out.getvalue() + self.assertIn("analyze_docker_image", output) + self.assertNotIn("Analyze Docker images.", output) + self.assertIn("(addon)", output) + def test_scanpipe_management_command_list_project(self): project1 = Project.objects.create(name="project1") project2 = Project.objects.create(name="project2") From a7c1bf5ee8a9b5d31f2b869e5285e993073db29c Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:39:53 +0400 Subject: [PATCH 2/8] Refactor the policies related code to its own module #386 (#1430) Signed-off-by: tdruez --- CHANGELOG.rst | 3 + scanpipe/apps.py | 38 ++++----- scanpipe/policies.py | 65 ++++++++++++++ .../includes/project_settings_menu.html | 5 ++ .../templates/scanpipe/project_settings.html | 14 +++ scanpipe/tests/__init__.py | 4 + scanpipe/tests/test_apps.py | 10 --- scanpipe/tests/test_policies.py | 85 +++++++++++++++++++ 8 files changed, 191 insertions(+), 33 deletions(-) create mode 100644 scanpipe/policies.py create mode 100644 scanpipe/tests/test_policies.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 601199700..e556524e3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ v34.9.0 (unreleased) - Add a new ``list-pipelines`` management command. https://github.com/aboutcode-org/scancode.io/issues/1397 +- Refactor the policies related code to its own module. + https://github.com/aboutcode-org/scancode.io/issues/386 + v34.8.3 (2024-10-30) -------------------- diff --git a/scanpipe/apps.py b/scanpipe/apps.py index 78855edcf..8aa0cf132 100644 --- a/scanpipe/apps.py +++ b/scanpipe/apps.py @@ -35,9 +35,11 @@ from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ -import saneyaml from licensedcode.models import load_licenses +from scanpipe.policies import load_policies_file +from scanpipe.policies import make_license_policy_index + try: from importlib import metadata as importlib_metadata except ImportError: @@ -218,31 +220,21 @@ def set_policies(self): """ Compute and sets the `license_policies` on the app instance. - If the policies file is available but formatted properly or doesn't + If the policies file is available but not formatted properly or doesn't include the proper content, we want to raise an exception while the app is loading to warn system admins about the issue. """ - policies_file_location = getattr(settings, "SCANCODEIO_POLICIES_FILE", None) - - if policies_file_location: - policies_file = Path(policies_file_location).expanduser() - - if policies_file.exists(): - logger.debug(style.SUCCESS(f"Load policies from {policies_file}")) - policies = saneyaml.load(policies_file.read_text()) - license_policies = policies.get("license_policies", []) - self.license_policies_index = self.get_policies_index( - policies_list=license_policies, - key="license_key", - ) - - else: - logger.debug(style.WARNING("Policies file not found.")) - - @staticmethod - def get_policies_index(policies_list, key): - """Return an inverted index by `key` of the `policies_list`.""" - return {policy.get(key): policy for policy in policies_list} + policies_file_setting = getattr(settings, "SCANCODEIO_POLICIES_FILE", None) + if not policies_file_setting: + return + + policies_file = Path(policies_file_setting).expanduser() + if policies_file.exists(): + policies = load_policies_file(policies_file) + logger.debug(style.SUCCESS(f"Loaded policies from {policies_file}")) + self.license_policies_index = make_license_policy_index(policies) + else: + logger.debug(style.WARNING("Policies file not found.")) @property def policies_enabled(self): diff --git a/scanpipe/policies.py b/scanpipe/policies.py new file mode 100644 index 000000000..beebf9bd8 --- /dev/null +++ b/scanpipe/policies.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from django.core.exceptions import ValidationError + +import saneyaml + + +def load_policies_yaml(policies_yaml): + """Load provided ``policies_yaml``.""" + try: + return saneyaml.load(policies_yaml) + except saneyaml.YAMLError as e: + raise ValidationError(f"Policies format error: {e}") + + +def load_policies_file(policies_file, validate=True): + """ + Load provided ``policies_file`` into a Python dictionary. + The policies format is validated by default. + """ + policies_dict = load_policies_yaml(policies_yaml=policies_file.read_text()) + if validate: + validate_policies(policies_dict) + return policies_dict + + +def validate_policies(policies_dict): + """Return True if the provided ``policies_dict`` is valid.""" + if not isinstance(policies_dict, dict): + raise ValidationError("The `policies_dict` argument must be a dictionary.") + + if "license_policies" not in policies_dict: + raise ValidationError( + "The `license_policies` key is missing from provided policies data." + ) + + return True + + +def make_license_policy_index(policies_dict): + """Return an inverted index by ``key`` of the ``policies_list``.""" + validate_policies(policies_dict) + + license_policies = policies_dict.get("license_policies", []) + return {policy.get("license_key"): policy for policy in license_policies} diff --git a/scanpipe/templates/scanpipe/includes/project_settings_menu.html b/scanpipe/templates/scanpipe/includes/project_settings_menu.html index ac07326b3..ad944b373 100644 --- a/scanpipe/templates/scanpipe/includes/project_settings_menu.html +++ b/scanpipe/templates/scanpipe/includes/project_settings_menu.html @@ -15,6 +15,11 @@ Ignored +
  • + + Policies + +
  • DejaCode diff --git a/scanpipe/templates/scanpipe/project_settings.html b/scanpipe/templates/scanpipe/project_settings.html index d2ec52f58..f9b8b3771 100644 --- a/scanpipe/templates/scanpipe/project_settings.html +++ b/scanpipe/templates/scanpipe/project_settings.html @@ -131,6 +131,20 @@ +
    +

    Policies

    +
    + {% if project.policies_enabled %} + Policies are enabled for this project. + {% else %} + Policies are not enabled for this project. + {% endif %} +

    + See License Policies and Compliance Alerts documentation for details. +

    +
    +
    +

    DejaCode

    diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index ecb919194..ed63c7e79 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -251,6 +251,10 @@ def make_dependency(project, **extra): }, ] +global_policies = { + "license_policies": license_policies, +} + license_policies_index = { "gpl-3.0": { "color_code": "#c83025", diff --git a/scanpipe/tests/test_apps.py b/scanpipe/tests/test_apps.py index fbf56ecbd..38050e3e8 100644 --- a/scanpipe/tests/test_apps.py +++ b/scanpipe/tests/test_apps.py @@ -30,10 +30,8 @@ from django.test import override_settings from django.utils import timezone -from scanpipe.apps import ScanPipeConfig from scanpipe.models import Project from scanpipe.models import Run -from scanpipe.tests import license_policies from scanpipe.tests import license_policies_index from scanpipe.tests.pipelines.register_from_file import RegisterFromFile @@ -44,14 +42,6 @@ class ScanPipeAppsTest(TestCase): data = Path(__file__).parent / "data" pipelines_location = Path(__file__).parent / "pipelines" - def test_scanpipe_apps_get_policies_index(self): - self.assertEqual({}, ScanPipeConfig.get_policies_index([], "license_key")) - policies_index = ScanPipeConfig.get_policies_index( - policies_list=license_policies, - key="license_key", - ) - self.assertEqual(license_policies_index, policies_index) - def test_scanpipe_apps_set_policies(self): scanpipe_app.license_policies_index = {} policies_files = None diff --git a/scanpipe/tests/test_policies.py b/scanpipe/tests/test_policies.py new file mode 100644 index 000000000..bee8d5aec --- /dev/null +++ b/scanpipe/tests/test_policies.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from pathlib import Path + +from django.apps import apps +from django.core.exceptions import ValidationError +from django.test import TestCase + +from scanpipe.policies import load_policies_file +from scanpipe.policies import load_policies_yaml +from scanpipe.policies import make_license_policy_index +from scanpipe.policies import validate_policies +from scanpipe.tests import global_policies +from scanpipe.tests import license_policies_index + +scanpipe_app = apps.get_app_config("scanpipe") + + +class ScanPipePoliciesTest(TestCase): + data = Path(__file__).parent / "data" + + def test_scanpipe_policies_load_policies_yaml(self): + policies_yaml = "{wrong format" + with self.assertRaises(ValidationError): + load_policies_yaml(policies_yaml) + + policies_files = self.data / "policy" / "policies.yml" + policies_dict = load_policies_yaml(policies_files.read_text()) + self.assertIn("license_policies", policies_dict) + + def test_scanpipe_policies_load_policies_file(self): + policies_files = self.data / "policy" / "policies.yml" + policies_dict = load_policies_file(policies_files) + self.assertIn("license_policies", policies_dict) + + def test_scanpipe_policies_validate_policies(self): + error_msg = "The `policies_dict` argument must be a dictionary." + policies_dict = None + with self.assertRaisesMessage(ValidationError, error_msg): + validate_policies(policies_dict) + + policies_dict = [] + with self.assertRaisesMessage(ValidationError, error_msg): + validate_policies(policies_dict) + + error_msg = "The `license_policies` key is missing from provided policies data." + policies_dict = {} + with self.assertRaisesMessage(ValidationError, error_msg): + validate_policies(policies_dict) + + policies_dict = {"missing": "data"} + with self.assertRaisesMessage(ValidationError, error_msg): + validate_policies(policies_dict) + + policies_dict = global_policies + self.assertTrue(validate_policies(policies_dict)) + + def test_scanpipe_policies_make_license_policy_index(self): + policies_dict = {"missing": "data"} + with self.assertRaises(ValidationError): + make_license_policy_index(policies_dict) + + self.assertEqual( + license_policies_index, make_license_policy_index(global_policies) + ) From 8fa21a7e3df1acd837e5d5f8b6659fde1c393453 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:22:34 +0400 Subject: [PATCH 3/8] Project policies #386 (#1440) Signed-off-by: tdruez --- CHANGELOG.rst | 5 + docs/command-line-interface.rst | 3 + docs/faq.rst | 5 + docs/index.rst | 1 + docs/policies.rst | 108 +++++++++++++++ docs/project-configuration.rst | 12 ++ docs/tutorial_license_policies.rst | 126 ++++++------------ scanpipe/apps.py | 5 - scanpipe/forms.py | 28 ++++ scanpipe/models.py | 104 +++++++++++---- scanpipe/pipelines/__init__.py | 9 +- scanpipe/pipes/flag.py | 10 ++ scanpipe/pipes/output.py | 2 +- scanpipe/templates/scanpipe/package_list.html | 8 +- .../templates/scanpipe/project_settings.html | 26 ++-- .../templates/scanpipe/resource_list.html | 8 +- scanpipe/tests/__init__.py | 6 - .../data/policies/include_policies_file.zip | Bin 0 -> 11560 bytes .../data/{policy => policies}/policies.yml | 3 - .../tests/data/policies/scancode-config.yml | 6 + scanpipe/tests/pipes/test_flag.py | 10 ++ scanpipe/tests/test_apps.py | 8 +- scanpipe/tests/test_forms.py | 27 ++++ scanpipe/tests/test_models.py | 40 ++++++ scanpipe/tests/test_pipelines.py | 4 +- scanpipe/tests/test_policies.py | 78 ++++++++++- scanpipe/views.py | 12 -- 27 files changed, 484 insertions(+), 170 deletions(-) create mode 100644 docs/policies.rst create mode 100644 scanpipe/tests/data/policies/include_policies_file.zip rename scanpipe/tests/data/{policy => policies}/policies.yml (77%) create mode 100644 scanpipe/tests/data/policies/scancode-config.yml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e556524e3..245b8eb74 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,11 @@ v34.9.0 (unreleased) - Refactor the policies related code to its own module. https://github.com/aboutcode-org/scancode.io/issues/386 +- Add support for project-specific license policies and compliance alerts. + Enhance Project model to handle policies from local settings, project input + "policies.yml" files, or global app settings. + https://github.com/aboutcode-org/scancode.io/issues/386 + v34.8.3 (2024-10-30) -------------------- diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index 0548b52e5..08914b6aa 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -285,6 +285,9 @@ your outputs on the host machine when running with Docker. .. tip:: To specify a CycloneDX spec version (default to latest), use the syntax ``cyclonedx:VERSION`` as format value. For example: ``--format cyclonedx:1.5``. + +.. _cli_check_compliance: + `$ scanpipe check-compliance --project PROJECT` ----------------------------------------------- diff --git a/docs/faq.rst b/docs/faq.rst index 29e895270..7485a4726 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -278,3 +278,8 @@ data older than 7 days:: See :ref:`command_line_interface` chapter for more information about the scanpipe command. + +How can I provide my license policies ? +--------------------------------------- + +For detailed information about the policies system, refer to :ref:`policies`. diff --git a/docs/index.rst b/docs/index.rst index 0c318fbc9..283b3854b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,7 @@ you’ll find information on: custom-pipelines scanpipe-pipes project-configuration + policies data-models output-files command-line-interface diff --git a/docs/policies.rst b/docs/policies.rst new file mode 100644 index 000000000..52060cc94 --- /dev/null +++ b/docs/policies.rst @@ -0,0 +1,108 @@ +.. _policies: + +License Policies and Compliance Alerts +====================================== + +ScanCode.io enables you to define **license policies** that check your projects +against a **compliance system**. + +Creating Policies Files +----------------------- + +A valid policies file is required to **enable compliance-related features**. + +The policies file, by default named ``policies.yml``, is a **YAML file** with a +structure similar to the following: + +.. code-block:: yaml + + license_policies: + - license_key: mit + label: Approved License + compliance_alert: '' + - license_key: mpl-2.0 + label: Restricted License + compliance_alert: warning + - license_key: gpl-3.0 + label: Prohibited License + compliance_alert: error + +- In the example above, licenses are referenced by the ``license_key``, + such as `mit` and `gpl-3.0`, which represent the ScanCode license keys used to + match against licenses detected in scan results. +- Each policy is defined with a ``label`` and a ``compliance_alert``. + You can customize the labels as desired. +- The ``compliance_alert`` field accepts three values: + + - ``''`` (empty string) + - ``warning`` + - ``error`` + +App Policies +------------ + +Policies can be enabled for the entire ScanCode.io app instance or on a per-project +basis. + +By default, ScanCode.io will look for a ``policies.yml`` file in the root of its +application codebase. + +Alternatively, you can specify the location of your policies file in your ``.env`` file +using the :ref:`scancodeio_settings_policies_file` setting. + +If a policies file is found at this location, those policies will be applied to +all projects in the ScanCode.io instance. + +.. tip:: + Refer to the :ref:`scancodeio_settings` section for a full list of settings, + including the policies file setting. + +Per-Project Policies +-------------------- + +Project-specific policies can be provided via a ``policies.yml`` file as one of the +project inputs or by defining the ``policies`` value in the +:ref:`project_configuration`. + +Compliance Alerts Ranking +------------------------- + +The compliance system uses a ``Precedence of Policies`` principle, which ensures the +highest-priority policy is applied in cases where resources or packages have complex +license expressions: + +- **error > warning > missing > '' (empty string)** + +This principle means that if a resource has an ``error``, ``warning``, and ``''`` +in its license expression, the overall compliance alert for that resource would be +``error``. + +.. warning:: + The ``missing`` compliance alert value is applied for licenses not included in the + policies file. + +Web UI +------ + +Compliance alerts are shown directly in the Web user interface in the following +locations: + +* A summary panel in the project detail view: + + .. image:: images/tutorial-policies-compliance-alerts-panel.png + +* A dedicated column in the Packages and Resources list tables: + + .. image:: images/tutorial-policies-compliance-alerts-column.png + +REST API +-------- + +For more details on retrieving compliance data through the REST API, see the +:ref:`rest_api_compliance` section. + +Command Line Interface +---------------------- + +A dedicated ``check-compliance`` management command is available. See the +:ref:`cli_check_compliance` section for more information. diff --git a/docs/project-configuration.rst b/docs/project-configuration.rst index a133e67fc..b41a55644 100644 --- a/docs/project-configuration.rst +++ b/docs/project-configuration.rst @@ -163,3 +163,15 @@ You can provide ``VCID`` from VulnerableCode or any aliases such as ``CVE`` or - OSV-2020-871 - BIT-django-2024-24680 - PYSEC-2024-28 + +.. _project_configuration_settings_policies: + +policies +^^^^^^^^ + +For detailed information about the policies system, refer to :ref:`policies`. + +Instead of providing a separate ``policies.yml`` file, policies can be directly +defined within the project configuration. +This can be done through the web UI, see :ref:`user_interface_project_settings`, +or by using a ``scancode-config.yml`` file. diff --git a/docs/tutorial_license_policies.rst b/docs/tutorial_license_policies.rst index 36cd997b1..9e0a6aea9 100644 --- a/docs/tutorial_license_policies.rst +++ b/docs/tutorial_license_policies.rst @@ -3,24 +3,23 @@ License Policies and Compliance Alerts ====================================== -In this tutorial, we'll introduce ScanCode.io's **license policies** and **compliance -alerts** system and use the **results of a pipeline run** to demonstrate an example -of the license policies and compliance alerts output. +In this tutorial, we'll introduce ScanCode.io's **license policies** and +**compliance alerts** system and use the **results of a pipeline run** to demonstrate +an example of the license policies and compliance alerts output. -As already mentioned, ScanCode.io automates the process of **Software Composition -Analysis "SCA"** to identify existing open source components and their license -compliance data in an application's codebase. +As already mentioned, ScanCode.io automates the process of +**Software Composition Analysis "SCA"** to identify existing open source components +and their license compliance data in an application's codebase. ScanCode.io also gives users the ability to define a set of **license policies** to have their projects checked against with a **compliance system**. -Creating Policies Files ------------------------ +Refer to :ref:`policies` for details about the policies system. -A valid policies file is required to **enable compliance-related features**. +Instructions +------------ -The policies file, by default ``policies.yml``, is a **YAML file** with a structure -similar to the following: +Create a ``policies.yml`` file with the following content: .. code-block:: yaml @@ -28,75 +27,18 @@ similar to the following: - license_key: mit label: Approved License compliance_alert: '' - - license_key: mpl-2.0 - label: Restricted License - compliance_alert: warning - license_key: gpl-3.0 label: Prohibited License compliance_alert: error -- In the above policies file, licenses are referenced by the ``license_key``, - such as mit and gpl-3.0, which represents the ScanCode license key to match - against detected licenses in the scan results. -- A policy is defined with a ``label`` and a ``compliance_alert``. - The labels can be customized to your preferred wording. -- The ``compliance_alert`` accepts 3 values: - - - ``''`` (empty string) - - ``warning`` - - ``error`` - -Policies File Location ----------------------- - -By default, ScanCode.io will look for a ``policies.yml`` file at the root of its -app codebase. - -Alternatively, you can configure the location of policies files using the -dedicated :ref:`scancodeio_settings_policies_file` setting in your ``.env`` file. - -.. tip:: - Check out our :ref:`scancodeio_settings` section for a comprehensive list of - settings including policies file setting. - -How Does The Compliance Alert Work? ------------------------------------ - -The compliance system works by following a ``Precedence of Policies`` principal -allowing the highest precedence policy to be applied in case of resources or -packages with complex license expressions: - -- **error > warning > missing > '' (empty string)** - -This principal means a given resource with ``error AND warning AND ''`` -license expression would have an overall compliance alert of ``error``. - -.. warning:: - The ``missing`` compliance alert value is applied for licenses not present in the - policies file. - -Example Output --------------- - -Create a ``policies.yml`` file in the root directory of your ScanCode.io codebase, with -the following content: - -.. code-block:: yaml - - license_policies: - - license_key: mit - label: Approved License - compliance_alert: '' - - license_key: gpl-3.0 - label: Prohibited License - compliance_alert: error - -Run the following command to create a project and run the ``scan_codebase`` pipeline: +Run the following command to create a project and run the ``scan_codebase`` pipeline +(make sure to use the proper path for the policies.yml file): .. code-block:: bash $ scanpipe create-project cuckoo-filter-with-policies \ --input-url https://files.pythonhosted.org/packages/75/fc/f5b2e466d763dcc381d5127b73ffc265e8cdaf39ddafa422b7896e625432/cuckoo_filter-1.0.6.tar.gz \ + --input-file policies.yml \ --pipeline scan_codebase \ --execute @@ -104,10 +46,10 @@ Generate results: .. code-block:: bash - $ scanpipe output --project cuckoo-filter-with-policies + $ scanpipe output --print --project cuckoo-filter-with-policies The computed compliance alerts are now included in the results, available for each -detected licenses, and computed at the codebase resource level, for example: +detected license, and computed at the codebase resource level, for example: .. code-block:: json @@ -123,38 +65,46 @@ detected licenses, and computed at the codebase resource level, for example: "label": "Recommended License", "compliance_alert": "" }, - } + }, { "key": "gpl-3.0", "name": "GNU General Public License 3.0", "policy": { "label": "Prohibited License", "compliance_alert": "error" - }, + } + } ], "license_expressions": [ - "mit OR gpl-3.0", + "mit OR gpl-3.0" ], "status": "scanned", "name": "README", "[...]": "[...]" } -Web UI ------- +Run the ``check-compliance`` command +------------------------------------ -Compliance alerts are visible directly in the Web user interface through the following: +Run the ``check-compliance`` command to get a listing of the compliance alerts detected +in the project: -* A summary panel in the project detail view: - -.. image:: images/tutorial-policies-compliance-alerts-panel.png +.. code-block:: bash -* A dedicated column within the Packages and Resources list tables: + $ scanpipe check-compliance --project cuckoo-filter-with-policies --verbosity 2 -.. image:: images/tutorial-policies-compliance-alerts-column.png +.. code-block:: bash -REST API --------- + 4 compliance issues detected on this project. + [packages] + > ERROR: 3 + pkg:pypi/cuckoo-filter@. + pkg:pypi/cuckoo-filter@1.0.6 + pkg:pypi/cuckoo-filter@1.0.6 + [resources] + > ERROR: 1 + cuckoo_filter-1.0.6.tar.gz-extract/cuckoo_filter-1.0.6/README.md -For more details on retrieving compliance data via the REST API, refer to the -:ref:`rest_api_compliance` section. +.. tip:: + In case of compliance alerts, the command returns a non-zero exit code which + may be useful to trigger a failure in an automated process. diff --git a/scanpipe/apps.py b/scanpipe/apps.py index 8aa0cf132..a404f95e6 100644 --- a/scanpipe/apps.py +++ b/scanpipe/apps.py @@ -236,11 +236,6 @@ def set_policies(self): else: logger.debug(style.WARNING("Policies file not found.")) - @property - def policies_enabled(self): - """Return True if the policies were provided and loaded properly.""" - return bool(self.license_policies_index) - def sync_runs_and_jobs(self): """Synchronize ``QUEUED`` and ``RUNNING`` Run with their related Jobs.""" logger.info("Synchronizing QUEUED and RUNNING Run with their related Jobs...") diff --git a/scanpipe/forms.py b/scanpipe/forms.py index f854235aa..8eb1663a5 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -32,6 +32,8 @@ from scanpipe.models import Run from scanpipe.pipelines import convert_markdown_to_html from scanpipe.pipes import fetch +from scanpipe.policies import load_policies_yaml +from scanpipe.policies import validate_policies scanpipe_app = apps.get_app_config("scanpipe") @@ -378,6 +380,7 @@ class ProjectSettingsForm(forms.ModelForm): "ignored_patterns", "ignored_dependency_scopes", "ignored_vulnerabilities", + "policies", "attribution_template", "product_name", "product_version", @@ -420,6 +423,25 @@ class ProjectSettingsForm(forms.ModelForm): }, ), ) + policies = forms.CharField( + label="License policies", + required=False, + help_text=( + "Refer to the documentation for syntax details: " + "https://scancodeio.readthedocs.io/en/latest/tutorial_license_policies.html" + ), + widget=forms.Textarea( + attrs={ + "class": "textarea is-dynamic", + "rows": 3, + "placeholder": ( + "license_policies:\n" + "- license_key: gpl-2.0\n" + " compliance_alert: error" + ), + } + ), + ) attribution_template = forms.CharField( label="Attribution template", required=False, @@ -487,6 +509,12 @@ def update_project_settings(self, project): project.settings.update(config) project.save(update_fields=["settings"]) + def clean_policies(self): + if policies := self.cleaned_data.get("policies"): + policies_dict = load_policies_yaml(policies) + validate_policies(policies_dict) + return policies + class ProjectCloneForm(forms.Form): clone_name = forms.CharField(widget=forms.TextInput(attrs={"class": "input"})) diff --git a/scanpipe/models.py b/scanpipe/models.py index fd84836f0..fb2f5caf1 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -92,6 +92,7 @@ import scancodeio from scanpipe import humanize_time +from scanpipe import policies from scanpipe import tasks logger = logging.getLogger(__name__) @@ -780,45 +781,57 @@ def get_codebase_config_directory(self): if config_directory.exists(): return config_directory - def get_input_config_file(self): + def get_file_from_work_directory(self, filename): """ - Return the ``scancode-config.yml`` file from the input/ directory + Return the ``filename`` file from the input/ directory or from the codebase/ immediate subdirectories. Priority order: - 1. If a config file exists directly in the input/ directory, return it. - 2. If exactly one config file exists in a codebase/ immediate subdirectory, - return it. - 3. If multiple config files are found in subdirectories, report an error. + 1. If a ``filename`` file exists directly in the input/ directory, return it. + 2. If exactly one ``filename`` file exists in a codebase/ immediate + subdirectory, return it. + 3. If multiple ``filename`` files are found in subdirectories, report an error. """ - config_filename = settings.SCANCODEIO_CONFIG_FILE - - # Check for the config file in the root of the input/ directory. - root_config_file = self.input_path / config_filename - if root_config_file.exists(): - return root_config_file + # Check for the ``filename`` file in the root of the input/ directory. + root_file = self.input_path / filename + if root_file.exists(): + return root_file - # Search for config files in immediate codebase/ subdirectories. - subdir_config_files = list(self.codebase_path.glob(f"*/{config_filename}")) + # Search for files in immediate codebase/ subdirectories. + subdir_files = list(self.codebase_path.glob(f"*/{filename}")) - # If exactly one config file is found in codebase/ subdirectories, return it. - if len(subdir_config_files) == 1: - return subdir_config_files[0] + # If exactly one file is found in codebase/ subdirectories, return it. + if len(subdir_files) == 1: + return subdir_files[0] - # If multiple config files are found, report an error. - if len(subdir_config_files) > 1: + # If multiple files are found, report an error. + if len(subdir_files) > 1: self.add_warning( - f"More than one {config_filename} found. " + f"More than one {filename} found. " f"Could not determine which one to use.", model="Project", details={ "resources": [ - str(path.relative_to(self.work_path)) - for path in subdir_config_files + str(path.relative_to(self.work_path)) for path in subdir_files ] }, ) + def get_input_config_file(self): + """ + Return the ``scancode-config.yml`` file from the input/ directory + or from the codebase/ immediate subdirectories. + """ + config_filename = settings.SCANCODEIO_CONFIG_FILE + return self.get_file_from_work_directory(config_filename) + + def get_input_policies_file(self): + """ + Return the "policies.yml" file from the input/ directory + or from the codebase/ immediate subdirectories. + """ + return self.get_file_from_work_directory("policies.yml") + def get_settings_as_yml(self): """Return the ``settings`` file content as yml, suitable for a config file.""" return saneyaml.dump(self.settings) @@ -1400,10 +1413,37 @@ def has_single_resource(self): """ return self.resource_count == 1 + def get_policy_index(self): + """ + Return the policy index for this project instance. + + The policies are loaded from the following locations in that order: + 1. the project local settings + 2. the "policies.yml" file in the project input/ directory + 3. the global app settings license policies + """ + if policies_from_settings := self.get_env("policies"): + policies_dict = policies_from_settings + if isinstance(policies_from_settings, str): + policies_dict = policies.load_policies_yaml(policies_from_settings) + return policies.make_license_policy_index(policies_dict) + + elif policies_file := self.get_input_policies_file(): + policies_dict = policies.load_policies_file(policies_file) + return policies.make_license_policy_index(policies_dict) + + else: + return scanpipe_app.license_policies_index + + @cached_property + def policy_index(self): + """Return the cached policy index for this project instance.""" + return self.get_policy_index() + @property def policies_enabled(self): - """Return True if the policies are enabled.""" - return scanpipe_app.policies_enabled + """Return True if the policies are enabled for this project.""" + return bool(self.policy_index) class GroupingQuerySetMixin: @@ -2422,7 +2462,7 @@ def save(self, codebase=None, *args, **kwargs): `codebase` is not used in this context but required for compatibility with the commoncode.resource.Codebase class API. """ - if scanpipe_app.policies_enabled: + if self.policies_enabled: loaded_license_expression = getattr(self, "_loaded_license_expression", "") license_expression = getattr(self, self.license_expression_field, "") if license_expression != loaded_license_expression: @@ -2432,19 +2472,29 @@ def save(self, codebase=None, *args, **kwargs): super().save(*args, **kwargs) + @property + def policy_index(self): + return self.project.policy_index + + @cached_property + def policies_enabled(self): + return self.project.policies_enabled + def compute_compliance_alert(self): """Compute and return the compliance_alert value from the licenses policies.""" license_expression = getattr(self, self.license_expression_field, "") if not license_expression: return "" - alerts = [] - policy_index = scanpipe_app.license_policies_index + policy_index = self.policy_index + if not policy_index: + return licensing = get_licensing() parsed = licensing.parse(license_expression, simple=True) license_keys = licensing.license_keys(parsed) + alerts = [] for license_key in license_keys: if policy := policy_index.get(license_key): alerts.append(policy.get("compliance_alert") or self.Compliance.OK) diff --git a/scanpipe/pipelines/__init__.py b/scanpipe/pipelines/__init__.py index 8c8444d71..b2a3f61cc 100644 --- a/scanpipe/pipelines/__init__.py +++ b/scanpipe/pipelines/__init__.py @@ -72,8 +72,13 @@ def flag_ignored_resources(self): """Flag ignored resources based on Project ``ignored_patterns`` setting.""" from scanpipe.pipes import flag - if ignored_patterns := self.env.get("ignored_patterns"): - flag.flag_ignored_patterns(self.project, patterns=ignored_patterns) + ignored_patterns = self.env.get("ignored_patterns", []) + + if isinstance(ignored_patterns, str): + ignored_patterns = ignored_patterns.splitlines() + ignored_patterns.extend(flag.DEFAULT_IGNORED_PATTERNS) + + flag.flag_ignored_patterns(self.project, patterns=ignored_patterns) def extract_archive(self, location, target): """Extract archive at `location` to `target`. Save errors as messages.""" diff --git a/scanpipe/pipes/flag.py b/scanpipe/pipes/flag.py index 162d9212c..555d18e9d 100644 --- a/scanpipe/pipes/flag.py +++ b/scanpipe/pipes/flag.py @@ -62,6 +62,16 @@ NOT_DEPLOYED = "not-deployed" +# Target files that should be ignored during processing as those are related to the app +# configuration. +DEFAULT_IGNORED_PATTERNS = [ + "scancode-config.yml", # when located in the root dir + "*/scancode-config.yml", + "policies.yml", # when located in the root dir + "*/policies.yml", +] + + def flag_empty_files(project): """Flag empty files as ignored.""" qs = ( diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 885ed91f9..58a7cae06 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -457,7 +457,7 @@ def to_xlsx(project): "license_clues", ] - if not scanpipe_app.policies_enabled: + if not project.policies_enabled: exclude_fields.append("compliance_alert") model_names = [ diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index 8fc0654a1..ee415673f 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -36,13 +36,13 @@ {{ package.declared_license_expression }} - {% if display_compliance_alert %} - + + {% if package.compliance_alert %} {{ package.compliance_alert }} - - {% endif %} + {% endif %} + {{ package.copyright|truncatechars:150|linebreaksbr }} diff --git a/scanpipe/templates/scanpipe/project_settings.html b/scanpipe/templates/scanpipe/project_settings.html index f9b8b3771..fd6c9eedb 100644 --- a/scanpipe/templates/scanpipe/project_settings.html +++ b/scanpipe/templates/scanpipe/project_settings.html @@ -134,14 +134,24 @@

    Policies

    - {% if project.policies_enabled %} - Policies are enabled for this project. - {% else %} - Policies are not enabled for this project. - {% endif %} -

    - See License Policies and Compliance Alerts documentation for details. -

    +
    + {% if project.policies_enabled %} + + Policies are enabled for this project. + {% else %} + + Policies are not enabled for this project. + {% endif %} +
    +
    + +
    + {{ form.policies }} +
    +

    {{ form.policies.help_text|urlize }}

    +
    diff --git a/scanpipe/templates/scanpipe/resource_list.html b/scanpipe/templates/scanpipe/resource_list.html index 6c7bcc8ea..79ec374b8 100644 --- a/scanpipe/templates/scanpipe/resource_list.html +++ b/scanpipe/templates/scanpipe/resource_list.html @@ -59,11 +59,11 @@ {{ resource.detected_license_expression }} - {% if display_compliance_alert %} - + + {% if resource.compliance_alert %} {{ resource.compliance_alert }} - - {% endif %} + {% endif %} +