Load multiple inventories at once #451 (#452)

* Add support for multiple inputs in the LoadInventory pipeline #451

Signed-off-by: Thomas Druez <[email protected]>

CHANGELOG.rst

@@ -27,8 +27,11 @@ v31.0.0 (next)
   Reference: https://tracker.debian.org/pkg/wait-for-it
   https://github.com/nexB/scancode.io/issues/387
 
+- Add support for multiple inputs in the LoadInventory pipeline.
+  https://github.com/nexB/scancode.io/issues/451
+
 - Add new SCANCODEIO_REDIS_PASSWORD environment variable and setting
-  to optionally set Redis instance password
+  to optionally set Redis instance password.
 
 - Ensure a project cannot be deleted through the API while a pipeline is running.
   https://github.com/nexB/scancode.io/issues/402

scanpipe/pipelines/load_inventory.py

@@ -26,30 +26,30 @@
 
 class LoadInventory(Pipeline):
     """
-    A pipeline to load an inventory of files and packages from a ScanCode JSON scan.
-    (Presumably containing file information and package scan data).
+    A pipeline to load one or more inventory of files and packages from a ScanCode JSON
+    scan results. (Presumably containing resource information and package scan data).
     """
 
     @classmethod
     def steps(cls):
         return (
-            cls.get_scan_json_input,
-            cls.build_inventory_from_scan,
+            cls.get_scan_json_inputs,
+            cls.build_inventory_from_scans,
         )
 
-    def get_scan_json_input(self):
+    def get_scan_json_inputs(self):
         """
-        Locates a JSON scan input from a project's input/ directory.
+        Locates all the ScanCode JSON scan results from the project's input/ directory.
+        This includes all files with a .json extension.
         """
-        inputs = list(self.project.inputs(pattern="*.json"))
+        self.input_locations = [
+            str(scan_input.absolute())
+            for scan_input in self.project.inputs(pattern="*.json")
+        ]
 
-        if len(inputs) != 1:
-            raise Exception("Only 1 JSON input file supported")
-
-        self.input_location = str(inputs[0].absolute())
-
-    def build_inventory_from_scan(self):
+    def build_inventory_from_scans(self):
         """
-        Processes a JSON Scan results file to populate codebase resources and packages.
+        Processes JSON scan results files to populate codebase resources and packages.
         """
-        scancode.create_inventory_from_scan(self.project, self.input_location)
+        for input_location in self.input_locations:
+            scancode.create_inventory_from_scan(self.project, input_location)

scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json

@@ -8,7 +8,7 @@
         {
           "pipeline_name": "load_inventory",
           "status": "not_started",
-          "description": "A pipeline to load an inventory of files and packages from a ScanCode JSON scan.\n(Presumably containing file information and package scan data).",
+          "description": "A pipeline to load one or more inventory of files and packages from a ScanCode JSON\nscan results. (Presumably containing resource information and package scan data).",
           "scancodeio_version": "",
           "task_id": null,
           "task_start_date": null,

scanpipe/tests/test_pipelines.py

@@ -77,9 +77,9 @@ def test_scanpipe_pipeline_class_execute(self):
         run = project1.add_pipeline("do_nothing")
         pipeline = run.make_pipeline_instance()
 
-        exitcode, output = pipeline.execute()
+        exitcode, out = pipeline.execute()
         self.assertEqual(0, exitcode)
-        self.assertEqual("", output)
+        self.assertEqual("", out)
 
         run.refresh_from_db()
         self.assertIn("Pipeline [do_nothing] starting", run.log)
@@ -94,14 +94,14 @@ def test_scanpipe_pipeline_class_execute_with_exception(self):
         run = project1.add_pipeline("raise_exception")
         pipeline = run.make_pipeline_instance()
 
-        exitcode, output = pipeline.execute()
+        exitcode, out = pipeline.execute()
         self.assertEqual(1, exitcode)
-        self.assertTrue(output.startswith("Error message"))
-        self.assertIn("Traceback:", output)
-        self.assertIn("in execute", output)
-        self.assertIn("step(self)", output)
-        self.assertIn("in raise_exception", output)
-        self.assertIn("raise ValueError", output)
+        self.assertTrue(out.startswith("Error message"))
+        self.assertIn("Traceback:", out)
+        self.assertIn("in execute", out)
+        self.assertIn("step(self)", out)
+        self.assertIn("in raise_exception", out)
+        self.assertIn("raise ValueError", out)
 
         run.refresh_from_db()
         self.assertIn("Pipeline [raise_exception] starting", run.log)
@@ -145,7 +145,7 @@ def test_scanpipe_pipelines_profile_decorator(self):
         run = project1.add_pipeline("profile_step")
         pipeline_instance = run.make_pipeline_instance()
 
-        exitcode, output = pipeline_instance.execute()
+        exitcode, out = pipeline_instance.execute()
         self.assertEqual(0, exitcode)
 
         run.refresh_from_db()
@@ -296,8 +296,8 @@ def test_scanpipe_scan_package_pipeline_integration_test(self):
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        exitcode, output = pipeline.execute()
-        self.assertEqual(0, exitcode, msg=output)
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(4, project1.codebaseresources.count())
         self.assertEqual(1, project1.discoveredpackages.count())
@@ -329,8 +329,8 @@ def test_scanpipe_scan_package_pipeline_integration_test_multiple_packages(self)
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        exitcode, output = pipeline.execute()
-        self.assertEqual(0, exitcode, msg=output)
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(9, project1.codebaseresources.count())
         self.assertEqual(2, project1.discoveredpackages.count())
@@ -357,8 +357,8 @@ def test_scanpipe_scan_codebase_pipeline_integration_test(self):
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        exitcode, _ = pipeline.execute()
-        self.assertEqual(0, exitcode)
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(6, project1.codebaseresources.count())
         self.assertEqual(1, project1.discoveredpackages.count())
@@ -379,8 +379,8 @@ def test_scanpipe_docker_pipeline_alpine_integration_test(self):
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        exitcode, _ = pipeline.execute()
-        self.assertEqual(0, exitcode)
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(83, project1.codebaseresources.count())
         self.assertEqual(14, project1.discoveredpackages.count())
@@ -401,8 +401,8 @@ def test_scanpipe_docker_pipeline_rpm_integration_test(self):
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        exitcode, _ = pipeline.execute()
-        self.assertEqual(0, exitcode)
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(25, project1.codebaseresources.count())
         self.assertEqual(101, project1.discoveredpackages.count())
@@ -423,8 +423,8 @@ def test_scanpipe_docker_pipeline_debian_integration_test(self):
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        exitcode, _ = pipeline.execute()
-        self.assertEqual(0, exitcode)
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(6, project1.codebaseresources.count())
         self.assertEqual(2, project1.discoveredpackages.count())
@@ -443,8 +443,8 @@ def test_scanpipe_rootfs_pipeline_integration_test(self):
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        exitcode, _ = pipeline.execute()
-        self.assertEqual(0, exitcode)
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(6, project1.codebaseresources.count())
         self.assertEqual(4, project1.discoveredpackages.count())
@@ -463,8 +463,8 @@ def test_scanpipe_load_inventory_pipeline_integration_test(self):
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        exitcode, _ = pipeline.execute()
-        self.assertEqual(0, exitcode)
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(18, project1.codebaseresources.count())
         self.assertEqual(2, project1.discoveredpackages.count())