Add support for including file path for collation (#30)

Add support for including file path for collated files, this solves the
problem where if you are collecting files from multiple agents, file
path clashes occur. Adding the --include-file-path option will ensure
that the file names are unique

harvest/cli.py

@@ -108,6 +108,12 @@ def _init_arguments(self):
             metavar="YYYY-MM-DD or YYYYMMDD",
             default=False,
         )
+        self.add_argument(
+            "--include-file-path",
+            help="Should the file path be included in the saved file names",
+            action="store_true",
+            dest="include_file_path",
+        )
 
     def _validate_arguments(self, args):
         if not args.end:
@@ -137,6 +143,7 @@ def _run(self, args):
             "master",
             args.repo_path,
             args.no_validate,
+            include_file_path=args.include_file_path,
         )
 
         for file in args.filepath:

harvest/collator.py

@@ -27,7 +27,15 @@
 class Collator(object):
     """Harvest collator to retrieve Git repository content."""
 
-    def __init__(self, repo_url, creds, branch, repo_path=None, validate=True):
+    def __init__(
+        self,
+        repo_url,
+        creds,
+        branch,
+        repo_path=None,
+        validate=True,
+        include_file_path=False,
+    ):
         """Construct the Collator object."""
         parsed = urlparse(repo_url)
         self.scheme = parsed.scheme
@@ -38,6 +46,7 @@ def __init__(self, repo_url, creds, branch, repo_path=None, validate=True):
         self.repo_path = repo_path
         self.git_repo = None
         self.validate = validate
+        self.include_file_path = include_file_path
 
     @property
     def local_path(self):
@@ -84,16 +93,21 @@ def read(self, filepath, from_dt, until_dt):
             raise FileMissingError(f"{filepath} not found between {since} and {until}")
         return commits
 
-    def write(self, filepath, commits):
+    def write(self, filepath: str, commits):
         """
         Create file artifacts.
 
         :param str filepath: The relative path to the file within the repo
         :param list commits: A list of commits for a given file and date range
         """
+        file_path_include = ""
+        if self.include_file_path:
+            file_path_include = "_".join(filepath.rsplit("/")[:-1]) + "_"
+
         for commit in commits:
             file_name = (
                 f"./{self._ts_to_str(commit.committed_date)}_"
+                f"{file_path_include}"
                 f'{filepath.rsplit("/", 1).pop()}'
             )
             with open(file_name, "w+") as f:

test/test_cli_collate.py

@@ -307,3 +307,27 @@ def test_collate_local(self, mock_read, mock_write):
             datetime(today.year, today.month, today.day),
         )
         mock_write.assert_called_once_with("my/path/baz.json", ["commit-foo"])
+
+    @patch("harvest.collator.Collator.write")
+    @patch("harvest.collator.Collator.read")
+    def test_collate_include_file_path(self, mock_read, mock_write):
+        """Ensures collate sub-command works when '--include-file-path' is provided."""
+        mock_read.return_value = ["commit-foo"]
+        self.harvest.run(
+            [
+                "collate",
+                "local",
+                "my/path/baz.json",
+                "--include-file-path",
+                "--repo-path",
+                "os/repo/path",
+            ]
+        )
+        today = datetime.today()
+
+        mock_read.assert_called_once_with(
+            "my/path/baz.json",
+            datetime(today.year, today.month, today.day),
+            datetime(today.year, today.month, today.day),
+        )
+        mock_write.assert_called_once_with("my/path/baz.json", ["commit-foo"])

test/test_collator.py

@@ -153,6 +153,18 @@ def test_write_functionality(self):
         self.assertIn(call("./20191105_foo.json", "w+"), m.mock_calls)
         self.assertIn(call("./20191101_foo.json", "w+"), m.mock_calls)
 
+    def test_write_includes_file_path(self):
+        m = mock_open()
+        with patch("builtins.open", m):
+            collator = Collator(*self.args, include_file_path=True)
+            collator.write("raw/foo/foo.json", self.commits)
+        handle = m()
+
+        self.assertEqual(handle.write.call_count, 3)
+        self.assertIn(call("./20191106_raw_foo_foo.json", "w+"), m.mock_calls)
+        self.assertIn(call("./20191105_raw_foo_foo.json", "w+"), m.mock_calls)
+        self.assertIn(call("./20191101_raw_foo_foo.json", "w+"), m.mock_calls)
+
     @patch("harvest.collator.git.Repo.clone_from")
     @patch("harvest.collator.os.path.isdir")
     def test_checkout_clone(self, is_dir_mock, clone_from_mock):