From 04d3495a42a5d18adfe4c38970b2414a6b0908eb Mon Sep 17 00:00:00 2001 From: Rhys Short Date: Thu, 9 Feb 2023 13:05:53 +0000 Subject: [PATCH] Human readable dates and multiple file path support in the collate command (#27) * Use human friendly date format in collate Use human friendly date format in collate, so when collating a large number of files becomes easy to translate requested dates from auditors into cli arguments. There is compatbility between the new human friendly format and the previous format negate the need for a breaking change release and avoid users changing their scripts. * Collate multiple files for the same date range Using comma seperated paths, enable collate to pull multiole files at once. This makes it easier for humans to pull multiple bits of evidence at once * Update version number and Changes Update version to 1.3.0 and add changes entry. --- CHANGES.md | 5 ++ harvest/__init__.py | 2 +- harvest/cli.py | 29 ++++++----- test/test_cli_collate.py | 101 +++++++++++++++++++++++++++++++++++---- 4 files changed, 116 insertions(+), 21 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 53d3861..1b87ab3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +# [1.3.0](https://github.com/ComplianceAsCode/auditree-harvest/releases/tag/v1.3.0) + +- [ADDED] Added option to use the date format YYYY-MM-DD in addition to YYYYMMDD when collating files. +- [ADDED] Made it possible to collate multiple files during a single run. + # [1.2.1](https://github.com/ComplianceAsCode/auditree-harvest/releases/tag/v1.2.1) - [CHANGED] Removed yapf in favour of black as code formatter. diff --git a/harvest/__init__.py b/harvest/__init__.py index 010b262..de74e65 100644 --- a/harvest/__init__.py +++ b/harvest/__init__.py @@ -13,4 +13,4 @@ # limitations under the License. """The Auditree file collating and reporting tool.""" -__version__ = "1.2.0" +__version__ = "1.3.0" diff --git a/harvest/cli.py b/harvest/cli.py index 7bd715c..b94a9aa 100644 --- a/harvest/cli.py +++ b/harvest/cli.py @@ -88,6 +88,7 @@ def _init_arguments(self): "the relative path to a file in a git repository " "that you wish to retrieve" ), + nargs="+", ) self.add_argument( "--end", @@ -95,7 +96,7 @@ def _init_arguments(self): "the end of date range for the file you wish to retrieve - " "defaults to the current date" ), - metavar="YYYYMMDD", + metavar="YYYY-MM-DD or YYYYMMDD", default=False, ) self.add_argument( @@ -104,17 +105,23 @@ def _init_arguments(self): "the start of date range for the file you wish to retrieve - " "defaults to same value as the end of date range" ), - metavar="YYYYMMDD", + metavar="YYYY-MM-DD or YYYYMMDD", default=False, ) def _validate_arguments(self, args): if not args.end: - args.end = datetime.today().strftime("%Y%m%d") + args.end = datetime.today().strftime("%Y-%m-%d") if not args.start: args.start = args.end - args.start = datetime.strptime(args.start, "%Y%m%d") - args.end = datetime.strptime(args.end, "%Y%m%d") + if "-" in args.start: + args.start = datetime.strptime(args.start, "%Y-%m-%d") + else: + args.start = datetime.strptime(args.start, "%Y%m%d") + if "-" in args.end: + args.end = datetime.strptime(args.end, "%Y-%m-%d") + else: + args.end = datetime.strptime(args.end, "%Y%m%d") if args.start > datetime.today(): return "ERROR: start date cannot be in the future" if args.start > args.end: @@ -131,12 +138,12 @@ def _run(self, args): args.repo_path, args.no_validate, ) - try: - collator.write( - args.filepath, collator.read(args.filepath, args.start, args.end) - ) - except ValueError as e: - self.err(f"ERROR: {str(e)}") + + for file in args.filepath: + try: + collator.write(file, collator.read(file, args.start, args.end)) + except ValueError as e: + self.err(f"ERROR: {str(e)}") class Report(_CoreHarvestCommand): diff --git a/test/test_cli_collate.py b/test/test_cli_collate.py index 72ee2a3..60b8208 100644 --- a/test/test_cli_collate.py +++ b/test/test_cli_collate.py @@ -15,7 +15,7 @@ import unittest from datetime import datetime, timedelta -from unittest.mock import patch +from unittest.mock import call, patch from harvest.cli import Harvest @@ -42,6 +42,42 @@ def test_collate_no_dates(self, mock_read, mock_write): ) mock_write.assert_called_once_with("my/path/baz.json", ["commit-foo"]) + @patch("harvest.collator.Collator.write") + @patch("harvest.collator.Collator.read") + def test_collate_multiple_paths(self, mock_read, mock_write): + """Ensures collate sub-command works when no dates provided.""" + mock_read.return_value = ["commit-foo"] + self.harvest.run( + [ + "collate", + "https://github.com/foo/bar", + "my/path/baz.json", + "my/path/bar.json", + ] + ) + today = datetime.today() + + mock_read.assert_has_calls( + [ + call( + "my/path/baz.json", + datetime(today.year, today.month, today.day), + datetime(today.year, today.month, today.day), + ), + call( + "my/path/bar.json", + datetime(today.year, today.month, today.day), + datetime(today.year, today.month, today.day), + ), + ] + ) + mock_write.assert_has_calls( + [ + call("my/path/baz.json", ["commit-foo"]), + call("my/path/bar.json", ["commit-foo"]), + ] + ) + @patch("harvest.collator.Collator.write") @patch("harvest.collator.Collator.read") def test_collate_using_repo_path(self, mock_read, mock_write): @@ -68,6 +104,32 @@ def test_collate_using_repo_path(self, mock_read, mock_write): @patch("harvest.collator.Collator.write") @patch("harvest.collator.Collator.read") def test_collate_start_date_only(self, mock_read, mock_write): + """Ensures collate sub-command works when only start date provided.""" + mock_read.return_value = ["commit-foo", "commit-bar", "commit-baz"] + self.harvest.run( + [ + "collate", + "https://github.com/foo/bar", + "my/path/baz.json", + "--start", + "2019-10-20", + ] + ) + today = datetime.today() + mock_read.assert_called_once_with( + "my/path/baz.json", + datetime(2019, 10, 20), + datetime(today.year, today.month, today.day), + ) + mock_write.assert_called_once_with( + "my/path/baz.json", ["commit-foo", "commit-bar", "commit-baz"] + ) + + @patch("harvest.collator.Collator.write") + @patch("harvest.collator.Collator.read") + def test_collate_start_date_only_without_date_seperator( + self, mock_read, mock_write + ): """Ensures collate sub-command works when only start date provided.""" mock_read.return_value = ["commit-foo", "commit-bar", "commit-baz"] self.harvest.run( @@ -92,6 +154,27 @@ def test_collate_start_date_only(self, mock_read, mock_write): @patch("harvest.collator.Collator.write") @patch("harvest.collator.Collator.read") def test_collate_end_date_only(self, mock_read, mock_write): + """Ensures collate sub-command works when only end date provided.""" + mock_read.return_value = ["commit-foo", "commit-bar", "commit-baz"] + self.harvest.run( + [ + "collate", + "https://github.com/foo/bar", + "my/path/baz.json", + "--end", + "2019-10-20", + ] + ) + mock_read.assert_called_once_with( + "my/path/baz.json", datetime(2019, 10, 20), datetime(2019, 10, 20) + ) + mock_write.assert_called_once_with( + "my/path/baz.json", ["commit-foo", "commit-bar", "commit-baz"] + ) + + @patch("harvest.collator.Collator.write") + @patch("harvest.collator.Collator.read") + def test_collate_end_date_only_no_date_seperators(self, mock_read, mock_write): """Ensures collate sub-command works when only end date provided.""" mock_read.return_value = ["commit-foo", "commit-bar", "commit-baz"] self.harvest.run( @@ -121,9 +204,9 @@ def test_collate_both_dates(self, mock_read, mock_write): "https://github.com/foo/bar", "my/path/baz.json", "--start", - "20191020", + "2019-10-20", "--end", - "20191120", + "2019-11-20", ] ) mock_read.assert_called_once_with( @@ -144,9 +227,9 @@ def test_collate_start_eq_end(self, mock_read, mock_write): "https://github.com/foo/bar", "my/path/baz.json", "--start", - "20191120", + "2019-11-20", "--end", - "20191120", + "2019-11-20", ] ) mock_read.assert_called_once_with( @@ -166,9 +249,9 @@ def test_collate_start_gt_end(self, mock_read, mock_write): "https://github.com/foo/bar", "my/path/baz.json", "--start", - "20191120", + "2019-11-20", "--end", - "20191020", + "2019-10-20", ] ) mock_read.assert_not_called() @@ -184,7 +267,7 @@ def test_collate_future_start(self, mock_read, mock_write): "https://github.com/foo/bar", "my/path/baz.json", "--start", - (datetime.today() + timedelta(days=1)).strftime("%Y%m%d"), + (datetime.today() + timedelta(days=1)).strftime("%Y-%m-%d"), ] ) mock_read.assert_not_called() @@ -202,7 +285,7 @@ def test_collate_future_end(self, mock_read, mock_write): "--start", "20191120", "--end", - (datetime.today() + timedelta(days=1)).strftime("%Y%m%d"), + (datetime.today() + timedelta(days=1)).strftime("%Y-%m-%d"), ] ) mock_read.assert_not_called()