diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8f3b04058..697447812 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,8 +27,11 @@ v31.0.0 (next) Reference: https://tracker.debian.org/pkg/wait-for-it https://github.com/nexB/scancode.io/issues/387 +- Add support for multiple inputs in the LoadInventory pipeline. + https://github.com/nexB/scancode.io/issues/451 + - Add new SCANCODEIO_REDIS_PASSWORD environment variable and setting - to optionally set Redis instance password + to optionally set Redis instance password. - Ensure a project cannot be deleted through the API while a pipeline is running. https://github.com/nexB/scancode.io/issues/402 diff --git a/scanpipe/pipelines/load_inventory.py b/scanpipe/pipelines/load_inventory.py index 06fb739bd..4dc356be7 100644 --- a/scanpipe/pipelines/load_inventory.py +++ b/scanpipe/pipelines/load_inventory.py @@ -26,30 +26,30 @@ class LoadInventory(Pipeline): """ - A pipeline to load an inventory of files and packages from a ScanCode JSON scan. - (Presumably containing file information and package scan data). + A pipeline to load one or more inventory of files and packages from a ScanCode JSON + scan results. (Presumably containing resource information and package scan data). """ @classmethod def steps(cls): return ( - cls.get_scan_json_input, - cls.build_inventory_from_scan, + cls.get_scan_json_inputs, + cls.build_inventory_from_scans, ) - def get_scan_json_input(self): + def get_scan_json_inputs(self): """ - Locates a JSON scan input from a project's input/ directory. + Locates all the ScanCode JSON scan results from the project's input/ directory. + This includes all files with a .json extension. """ - inputs = list(self.project.inputs(pattern="*.json")) + self.input_locations = [ + str(scan_input.absolute()) + for scan_input in self.project.inputs(pattern="*.json") + ] - if len(inputs) != 1: - raise Exception("Only 1 JSON input file supported") - - self.input_location = str(inputs[0].absolute()) - - def build_inventory_from_scan(self): + def build_inventory_from_scans(self): """ - Processes a JSON Scan results file to populate codebase resources and packages. + Processes JSON scan results files to populate codebase resources and packages. """ - scancode.create_inventory_from_scan(self.project, self.input_location) + for input_location in self.input_locations: + scancode.create_inventory_from_scan(self.project, input_location) diff --git a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json index 34362cc78..4d76c2b74 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json +++ b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json @@ -8,7 +8,7 @@ { "pipeline_name": "load_inventory", "status": "not_started", - "description": "A pipeline to load an inventory of files and packages from a ScanCode JSON scan.\n(Presumably containing file information and package scan data).", + "description": "A pipeline to load one or more inventory of files and packages from a ScanCode JSON\nscan results. (Presumably containing resource information and package scan data).", "scancodeio_version": "", "task_id": null, "task_start_date": null, diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index c2aa96af2..730495c24 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -77,9 +77,9 @@ def test_scanpipe_pipeline_class_execute(self): run = project1.add_pipeline("do_nothing") pipeline = run.make_pipeline_instance() - exitcode, output = pipeline.execute() + exitcode, out = pipeline.execute() self.assertEqual(0, exitcode) - self.assertEqual("", output) + self.assertEqual("", out) run.refresh_from_db() self.assertIn("Pipeline [do_nothing] starting", run.log) @@ -94,14 +94,14 @@ def test_scanpipe_pipeline_class_execute_with_exception(self): run = project1.add_pipeline("raise_exception") pipeline = run.make_pipeline_instance() - exitcode, output = pipeline.execute() + exitcode, out = pipeline.execute() self.assertEqual(1, exitcode) - self.assertTrue(output.startswith("Error message")) - self.assertIn("Traceback:", output) - self.assertIn("in execute", output) - self.assertIn("step(self)", output) - self.assertIn("in raise_exception", output) - self.assertIn("raise ValueError", output) + self.assertTrue(out.startswith("Error message")) + self.assertIn("Traceback:", out) + self.assertIn("in execute", out) + self.assertIn("step(self)", out) + self.assertIn("in raise_exception", out) + self.assertIn("raise ValueError", out) run.refresh_from_db() self.assertIn("Pipeline [raise_exception] starting", run.log) @@ -145,7 +145,7 @@ def test_scanpipe_pipelines_profile_decorator(self): run = project1.add_pipeline("profile_step") pipeline_instance = run.make_pipeline_instance() - exitcode, output = pipeline_instance.execute() + exitcode, out = pipeline_instance.execute() self.assertEqual(0, exitcode) run.refresh_from_db() @@ -296,8 +296,8 @@ def test_scanpipe_scan_package_pipeline_integration_test(self): run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - exitcode, output = pipeline.execute() - self.assertEqual(0, exitcode, msg=output) + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(4, project1.codebaseresources.count()) self.assertEqual(1, project1.discoveredpackages.count()) @@ -329,8 +329,8 @@ def test_scanpipe_scan_package_pipeline_integration_test_multiple_packages(self) run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - exitcode, output = pipeline.execute() - self.assertEqual(0, exitcode, msg=output) + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(9, project1.codebaseresources.count()) self.assertEqual(2, project1.discoveredpackages.count()) @@ -357,8 +357,8 @@ def test_scanpipe_scan_codebase_pipeline_integration_test(self): run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - exitcode, _ = pipeline.execute() - self.assertEqual(0, exitcode) + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(6, project1.codebaseresources.count()) self.assertEqual(1, project1.discoveredpackages.count()) @@ -379,8 +379,8 @@ def test_scanpipe_docker_pipeline_alpine_integration_test(self): run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - exitcode, _ = pipeline.execute() - self.assertEqual(0, exitcode) + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(83, project1.codebaseresources.count()) self.assertEqual(14, project1.discoveredpackages.count()) @@ -401,8 +401,8 @@ def test_scanpipe_docker_pipeline_rpm_integration_test(self): run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - exitcode, _ = pipeline.execute() - self.assertEqual(0, exitcode) + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(25, project1.codebaseresources.count()) self.assertEqual(101, project1.discoveredpackages.count()) @@ -423,8 +423,8 @@ def test_scanpipe_docker_pipeline_debian_integration_test(self): run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - exitcode, _ = pipeline.execute() - self.assertEqual(0, exitcode) + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(6, project1.codebaseresources.count()) self.assertEqual(2, project1.discoveredpackages.count()) @@ -443,8 +443,8 @@ def test_scanpipe_rootfs_pipeline_integration_test(self): run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - exitcode, _ = pipeline.execute() - self.assertEqual(0, exitcode) + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(6, project1.codebaseresources.count()) self.assertEqual(4, project1.discoveredpackages.count()) @@ -463,8 +463,8 @@ def test_scanpipe_load_inventory_pipeline_integration_test(self): run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - exitcode, _ = pipeline.execute() - self.assertEqual(0, exitcode) + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(18, project1.codebaseresources.count()) self.assertEqual(2, project1.discoveredpackages.count())