Human readable dates and multiple file path support in the collate co…

…mmand (#27)

* Use human friendly date format in collate

Use human friendly date format in collate, so when collating a large
number of files becomes easy to translate requested dates from auditors
into cli arguments. There is compatbility between the new human friendly
format and the previous format negate the need for a breaking change
release and avoid users changing their scripts.

* Collate multiple files for the same date range

Using comma seperated paths, enable collate to pull multiole files at
once. This makes it easier for humans to pull multiple bits of evidence
at once

* Update version number and Changes

Update version to 1.3.0 and add changes entry.

CHANGES.md

@@ -1,3 +1,8 @@
+# [1.3.0](https://github.com/ComplianceAsCode/auditree-harvest/releases/tag/v1.3.0)
+
+- [ADDED] Added option to use the date format YYYY-MM-DD in addition to YYYYMMDD when collating files.
+- [ADDED] Made it possible to collate multiple files during a single run.
+
 # [1.2.1](https://github.com/ComplianceAsCode/auditree-harvest/releases/tag/v1.2.1)
 
 - [CHANGED] Removed yapf in favour of black as code formatter.

harvest/__init__.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """The Auditree file collating and reporting tool."""
 
-__version__ = "1.2.0"
+__version__ = "1.3.0"

harvest/cli.py

@@ -88,14 +88,15 @@ def _init_arguments(self):
                 "the relative path to a file in a git repository "
                 "that you wish to retrieve"
             ),
+            nargs="+",
         )
         self.add_argument(
             "--end",
             help=(
                 "the end of date range for the file you wish to retrieve - "
                 "defaults to the current date"
             ),
-            metavar="YYYYMMDD",
+            metavar="YYYY-MM-DD or YYYYMMDD",
             default=False,
         )
         self.add_argument(
@@ -104,17 +105,23 @@ def _init_arguments(self):
                 "the start of date range for the file you wish to retrieve - "
                 "defaults to same value as the end of date range"
             ),
-            metavar="YYYYMMDD",
+            metavar="YYYY-MM-DD or YYYYMMDD",
             default=False,
         )
 
     def _validate_arguments(self, args):
         if not args.end:
-            args.end = datetime.today().strftime("%Y%m%d")
+            args.end = datetime.today().strftime("%Y-%m-%d")
         if not args.start:
             args.start = args.end
-        args.start = datetime.strptime(args.start, "%Y%m%d")
-        args.end = datetime.strptime(args.end, "%Y%m%d")
+        if "-" in args.start:
+            args.start = datetime.strptime(args.start, "%Y-%m-%d")
+        else:
+            args.start = datetime.strptime(args.start, "%Y%m%d")
+        if "-" in args.end:
+            args.end = datetime.strptime(args.end, "%Y-%m-%d")
+        else:
+            args.end = datetime.strptime(args.end, "%Y%m%d")
         if args.start > datetime.today():
             return "ERROR: start date cannot be in the future"
         if args.start > args.end:
@@ -131,12 +138,12 @@ def _run(self, args):
             args.repo_path,
             args.no_validate,
         )
-        try:
-            collator.write(
-                args.filepath, collator.read(args.filepath, args.start, args.end)
-            )
-        except ValueError as e:
-            self.err(f"ERROR: {str(e)}")
+
+        for file in args.filepath:
+            try:
+                collator.write(file, collator.read(file, args.start, args.end))
+            except ValueError as e:
+                self.err(f"ERROR: {str(e)}")
 
 
 class Report(_CoreHarvestCommand):

test/test_cli_collate.py

@@ -15,7 +15,7 @@
 
 import unittest
 from datetime import datetime, timedelta
-from unittest.mock import patch
+from unittest.mock import call, patch
 
 from harvest.cli import Harvest
 
@@ -42,6 +42,42 @@ def test_collate_no_dates(self, mock_read, mock_write):
         )
         mock_write.assert_called_once_with("my/path/baz.json", ["commit-foo"])
 
+    @patch("harvest.collator.Collator.write")
+    @patch("harvest.collator.Collator.read")
+    def test_collate_multiple_paths(self, mock_read, mock_write):
+        """Ensures collate sub-command works when no dates provided."""
+        mock_read.return_value = ["commit-foo"]
+        self.harvest.run(
+            [
+                "collate",
+                "https://github.com/foo/bar",
+                "my/path/baz.json",
+                "my/path/bar.json",
+            ]
+        )
+        today = datetime.today()
+
+        mock_read.assert_has_calls(
+            [
+                call(
+                    "my/path/baz.json",
+                    datetime(today.year, today.month, today.day),
+                    datetime(today.year, today.month, today.day),
+                ),
+                call(
+                    "my/path/bar.json",
+                    datetime(today.year, today.month, today.day),
+                    datetime(today.year, today.month, today.day),
+                ),
+            ]
+        )
+        mock_write.assert_has_calls(
+            [
+                call("my/path/baz.json", ["commit-foo"]),
+                call("my/path/bar.json", ["commit-foo"]),
+            ]
+        )
+
     @patch("harvest.collator.Collator.write")
     @patch("harvest.collator.Collator.read")
     def test_collate_using_repo_path(self, mock_read, mock_write):
@@ -68,6 +104,32 @@ def test_collate_using_repo_path(self, mock_read, mock_write):
     @patch("harvest.collator.Collator.write")
     @patch("harvest.collator.Collator.read")
     def test_collate_start_date_only(self, mock_read, mock_write):
+        """Ensures collate sub-command works when only start date provided."""
+        mock_read.return_value = ["commit-foo", "commit-bar", "commit-baz"]
+        self.harvest.run(
+            [
+                "collate",
+                "https://github.com/foo/bar",
+                "my/path/baz.json",
+                "--start",
+                "2019-10-20",
+            ]
+        )
+        today = datetime.today()
+        mock_read.assert_called_once_with(
+            "my/path/baz.json",
+            datetime(2019, 10, 20),
+            datetime(today.year, today.month, today.day),
+        )
+        mock_write.assert_called_once_with(
+            "my/path/baz.json", ["commit-foo", "commit-bar", "commit-baz"]
+        )
+
+    @patch("harvest.collator.Collator.write")
+    @patch("harvest.collator.Collator.read")
+    def test_collate_start_date_only_without_date_seperator(
+        self, mock_read, mock_write
+    ):
         """Ensures collate sub-command works when only start date provided."""
         mock_read.return_value = ["commit-foo", "commit-bar", "commit-baz"]
         self.harvest.run(
@@ -92,6 +154,27 @@ def test_collate_start_date_only(self, mock_read, mock_write):
     @patch("harvest.collator.Collator.write")
     @patch("harvest.collator.Collator.read")
     def test_collate_end_date_only(self, mock_read, mock_write):
+        """Ensures collate sub-command works when only end date provided."""
+        mock_read.return_value = ["commit-foo", "commit-bar", "commit-baz"]
+        self.harvest.run(
+            [
+                "collate",
+                "https://github.com/foo/bar",
+                "my/path/baz.json",
+                "--end",
+                "2019-10-20",
+            ]
+        )
+        mock_read.assert_called_once_with(
+            "my/path/baz.json", datetime(2019, 10, 20), datetime(2019, 10, 20)
+        )
+        mock_write.assert_called_once_with(
+            "my/path/baz.json", ["commit-foo", "commit-bar", "commit-baz"]
+        )
+
+    @patch("harvest.collator.Collator.write")
+    @patch("harvest.collator.Collator.read")
+    def test_collate_end_date_only_no_date_seperators(self, mock_read, mock_write):
         """Ensures collate sub-command works when only end date provided."""
         mock_read.return_value = ["commit-foo", "commit-bar", "commit-baz"]
         self.harvest.run(
@@ -121,9 +204,9 @@ def test_collate_both_dates(self, mock_read, mock_write):
                 "https://github.com/foo/bar",
                 "my/path/baz.json",
                 "--start",
-                "20191020",
+                "2019-10-20",
                 "--end",
-                "20191120",
+                "2019-11-20",
             ]
         )
         mock_read.assert_called_once_with(
@@ -144,9 +227,9 @@ def test_collate_start_eq_end(self, mock_read, mock_write):
                 "https://github.com/foo/bar",
                 "my/path/baz.json",
                 "--start",
-                "20191120",
+                "2019-11-20",
                 "--end",
-                "20191120",
+                "2019-11-20",
             ]
         )
         mock_read.assert_called_once_with(
@@ -166,9 +249,9 @@ def test_collate_start_gt_end(self, mock_read, mock_write):
                 "https://github.com/foo/bar",
                 "my/path/baz.json",
                 "--start",
-                "20191120",
+                "2019-11-20",
                 "--end",
-                "20191020",
+                "2019-10-20",
             ]
         )
         mock_read.assert_not_called()
@@ -184,7 +267,7 @@ def test_collate_future_start(self, mock_read, mock_write):
                 "https://github.com/foo/bar",
                 "my/path/baz.json",
                 "--start",
-                (datetime.today() + timedelta(days=1)).strftime("%Y%m%d"),
+                (datetime.today() + timedelta(days=1)).strftime("%Y-%m-%d"),
             ]
         )
         mock_read.assert_not_called()
@@ -202,7 +285,7 @@ def test_collate_future_end(self, mock_read, mock_write):
                 "--start",
                 "20191120",
                 "--end",
-                (datetime.today() + timedelta(days=1)).strftime("%Y%m%d"),
+                (datetime.today() + timedelta(days=1)).strftime("%Y-%m-%d"),
             ]
         )
         mock_read.assert_not_called()