Add support for the XLSX report in REST API #1524

Signed-off-by: tdruez <[email protected]>

CHANGELOG.rst

@@ -4,6 +4,9 @@ Changelog
 v34.9.5 (unreleased)
 --------------------
 
+- Add support for the XLSX report in REST API.
+  https://github.com/aboutcode-org/scancode.io/issues/1524
+
 v34.9.4 (2025-01-21)
 --------------------
 

docs/rest-api.rst

@@ -587,3 +587,69 @@ This action deletes a "not started" or "queued" pipeline run.
     {
         "status": "Pipeline pipeline_name deleted."
     }
+
+XLSX Report
+-----------
+
+Generates an XLSX report of selected projects based on the provided criteria.
+The model needs to be provided using the ``model`` query parameter.
+
+``GET /api/projects/?model=MODEL``
+
+Data:
+  - ``model``: ``package``, ``dependency``, ``resource``, ``relation``, ``message``,
+    ``todo``.
+
+**Any available filters can be applied** to **select the set of projects** you want to
+include in the report, such as a string contained in the name, or filter by labels:
+
+Example usage:
+
+1. Generate a report for all projects tagged with "d2d" and include the **TODOS**
+worksheet::
+
+    GET /api/projects/?model=todo&label=d2d
+
+
+2. Generate a report for projects whose names contain the word "audit" and include the
+**PACKAGES** worksheet::
+
+   GET /api/projects/?model=package&name__contains=audit
+
+
+XLSX Report
+-----------
+
+Generates an XLSX report for selected projects based on specified criteria. The
+``model`` query parameter is required to determine the type of data to include in the
+report.
+
+Endpoint:
+``GET /api/projects/report/?model=MODEL``
+
+Parameters:
+
+- ``model``: Defines the type of data to include in the report.
+  Accepted values: ``package``, ``dependency``, ``resource``, ``relation``, ``message``,
+  ``todo``.
+
+.. note::
+
+   You can apply any available filters to select the projects to include in the
+   report. Filters can be based on project attributes, such as a substring in the
+   name or specific labels.
+
+Example Usage:
+
+1. Generate a report for projects tagged with "d2d" and include the ``TODOS`` worksheet:
+
+   .. code-block::
+
+      GET /api/projects/report/?model=todo&label=d2d
+
+2. Generate a report for projects whose names contain "audit" and include the
+   ``PACKAGES`` worksheet:
+
+   .. code-block::
+
+      GET /api/projects/report/?model=package&name__contains=audit

scanpipe/api/views.py

@@ -52,6 +52,7 @@
 from scanpipe.models import Project
 from scanpipe.models import Run
 from scanpipe.models import RunInProgressError
+from scanpipe.pipes import filename_now
 from scanpipe.pipes import output
 from scanpipe.pipes.compliance import get_project_compliance_alerts
 from scanpipe.views import project_results_json_response
@@ -79,6 +80,11 @@ class ProjectFilterSet(django_filters.rest_framework.FilterSet):
         method="filter_names",
     )
     uuid = django_filters.CharFilter()
+    label = django_filters.CharFilter(
+        label="Label",
+        field_name="labels__slug",
+        distinct=True,
+    )
 
     class Meta:
         model = Project
@@ -90,6 +96,7 @@ class Meta:
             "names",
             "uuid",
             "is_archived",
+            "label",
         ]
 
     def filter_names(self, qs, name, value):
@@ -195,6 +202,40 @@ def pipelines(self, request, *args, **kwargs):
         ]
         return Response(pipeline_data)
 
+    @action(detail=False)
+    def report(self, request, *args, **kwargs):
+        project_qs = self.filter_queryset(self.get_queryset())
+
+        model_choices = list(output.object_type_to_model_name.keys())
+        model = request.GET.get("model")
+        if not model:
+            message = {
+                "error": (
+                    "Specifies the model to include in the XLSX report. "
+                    "Using: ?model=MODEL"
+                ),
+                "choices": ", ".join(model_choices),
+            }
+            return Response(message, status=status.HTTP_400_BAD_REQUEST)
+
+        if model not in model_choices:
+            message = {
+                "error": f"{model} is not on of the valid choices",
+                "choices": ", ".join(model_choices),
+            }
+            return Response(message, status=status.HTTP_400_BAD_REQUEST)
+
+        output_file = output.get_xlsx_report(
+            project_qs=project_qs,
+            model_short_name=model,
+        )
+        output_file.seek(0)
+        return FileResponse(
+            output_file,
+            filename=f"scancodeio-report-{filename_now()}.xlsx",
+            as_attachment=True,
+        )
+
     def get_filtered_response(
         self, request, queryset, filterset_class, serializer_class
     ):

scanpipe/tests/test_api.py

@@ -32,6 +32,7 @@
 from django.urls import reverse
 from django.utils import timezone
 
+import openpyxl
 from rest_framework import status
 from rest_framework.exceptions import ErrorDetail
 from rest_framework.test import APIClient
@@ -68,7 +69,7 @@ class ScanPipeAPITest(TransactionTestCase):
     data = Path(__file__).parent / "data"
 
     def setUp(self):
-        self.project1 = Project.objects.create(name="Analysis")
+        self.project1 = make_project(name="Analysis")
         self.resource1 = CodebaseResource.objects.create(
             project=self.project1,
             path="daglib-0.3.2.tar.gz-extract/daglib-0.3.2/PKG-INFO",
@@ -102,8 +103,8 @@ def test_scanpipe_api_browsable_formats_available(self):
         self.assertContains(response, self.project1_detail_url)
 
     def test_scanpipe_api_project_list(self):
-        Project.objects.create(name="2")
-        Project.objects.create(name="3")
+        make_project(name="2")
+        make_project(name="3")
 
         with self.assertNumQueries(8):
             response = self.csrf_client.get(self.project_list_url)
@@ -120,8 +121,8 @@ def test_scanpipe_api_project_list(self):
         self.assertNotContains(response, "dependency_count")
 
     def test_scanpipe_api_project_list_filters(self):
-        project2 = Project.objects.create(name="pro2ject", is_archived=True)
-        project3 = Project.objects.create(name="3project", is_archived=True)
+        project2 = make_project(name="pro2ject", is_archived=True)
+        project3 = make_project(name="3project", is_archived=True)
 
         response = self.csrf_client.get(self.project_list_url)
         self.assertEqual(3, response.data["count"])
@@ -178,6 +179,15 @@ def test_scanpipe_api_project_list_filters(self):
         self.assertContains(response, project2.uuid)
         self.assertContains(response, project3.uuid)
 
+        project2.labels.add("label1")
+        project3.labels.add("label1")
+        data = {"label": "label1"}
+        response = self.csrf_client.get(self.project_list_url, data=data)
+        self.assertEqual(2, response.data["count"])
+        self.assertNotContains(response, self.project1.uuid)
+        self.assertContains(response, project2.uuid)
+        self.assertContains(response, project3.uuid)
+
     def test_scanpipe_api_project_detail(self):
         response = self.csrf_client.get(self.project1_detail_url)
         self.assertIn(self.project1_detail_url, response.data["url"])
@@ -659,6 +669,47 @@ def test_scanpipe_api_project_action_pipelines(self):
         expected = ["name", "summary", "description", "steps", "available_groups"]
         self.assertEqual(expected, list(response.data[0].keys()))
 
+    def test_scanpipe_api_project_action_report(self):
+        url = reverse("project-report")
+
+        response = self.csrf_client.get(url)
+        self.assertEqual(400, response.status_code)
+        expected = (
+            "Specifies the model to include in the XLSX report. Using: ?model=MODEL"
+        )
+        self.assertEqual(expected, response.data["error"])
+
+        data = {"model": "bad value"}
+        response = self.csrf_client.get(url, data=data)
+        self.assertEqual(400, response.status_code)
+        expected = "bad value is not on of the valid choices"
+        self.assertEqual(expected, response.data["error"])
+
+        make_package(self.project1, package_url="pkg:generic/p1")
+        project2 = make_project()
+        project2.labels.add("label1")
+        package2 = make_package(project2, package_url="pkg:generic/p2")
+
+        data = {
+            "model": "package",
+            "label": "label1",
+        }
+        response = self.csrf_client.get(url, data=data)
+        self.assertEqual(200, response.status_code)
+        self.assertTrue(response.filename.startswith("scancodeio-report-"))
+        self.assertTrue(response.filename.endswith(".xlsx"))
+
+        output_file = io.BytesIO(b"".join(response.streaming_content))
+        workbook = openpyxl.load_workbook(output_file, read_only=True, data_only=True)
+        self.assertEqual(["PACKAGES"], workbook.get_sheet_names())
+
+        todos_sheet = workbook.get_sheet_by_name("PACKAGES")
+        rows = list(todos_sheet.values)
+        self.assertEqual(2, len(rows))
+        self.assertEqual("project", rows[0][0])  # header row
+        self.assertEqual(project2.name, rows[1][0])
+        self.assertEqual(package2.package_url, rows[1][1])
+
     def test_scanpipe_api_project_action_resources(self):
         url = reverse("project-resources", args=[self.project1.uuid])
         response = self.csrf_client.get(url)
@@ -927,7 +978,7 @@ def test_scanpipe_api_project_action_add_pipeline(self, mock_execute_pipeline_ta
         run = self.project1.runs.get()
         self.assertEqual(data["pipeline"], run.pipeline_name)
 
-        project2 = Project.objects.create(name="Analysis 2")
+        project2 = make_project(name="Analysis 2")
         url = reverse("project-add-pipeline", args=[project2.uuid])
         data["execute_now"] = True
         response = self.csrf_client.post(url, data=data)