Allow Google Sheets and Github Configuration to be passed as a list of files/directories

plugins/default.py

@@ -1,15 +1,20 @@
 """"Default plugin for SheetShuttle."""
 
+from pathlib import Path
+from typing import List
+
 from sheetshuttle import github_objects
 from sheetshuttle import sheet_collector
 
-from github import Github
-
-# import os
-
 
-def run(sheets_keys_file, sheets_config_directory, gh_config_directory, **kwargs):
+def run(
+    sheets_keys_file: str, sheets_config: List[Path], gh_config: List[Path], **kwargs
+):
     """Standard run function."""
     print("hello from the default plugin")
-    print("Additional arguments")
-    print(kwargs["args"])
+    print(f"sheets_keys_file: {sheets_keys_file}")
+    print(f"sheets_config: {sheets_config}")
+    print(f"gh_config: {gh_config}")
+    print(f"Additional arguments {kwargs}")
+    my_collector = sheet_collector.SheetCollector(sheets_config, sheets_keys_file)
+    my_collector.collect_files()

sheetshuttle/main.py

@@ -5,6 +5,7 @@
 
 import json
 from pathlib import Path
+from typing import List
 import typer
 
 from pluginbase import PluginBase  # type: ignore[import]
@@ -14,15 +15,21 @@
 
 app = typer.Typer(name="sheetshuttle")
 
-STANDARD_PLUGIN = '''""""Standard empty plugin for SheetShuttle."""
+STANDARD_PLUGIN = '''""""SheetShuttle Plugin Template."""
+
+from pathlib import Path
+from typing import List
 
 from sheetshuttle import github_objects
 from sheetshuttle import sheet_collector
 
-# This function is required
-def run(sheets_keys_file, sheets_config_directory, gh_config_directory, **kwargs):
+def run(sheets_keys_file: str, sheets_config: List[Path], gh_config: List[Path], **kwargs):
     """Standard run function."""
-    pass
+    print("hello from the default plugin")
+    print(f"sheets_keys_file: {sheets_keys_file}")
+    print(f"sheets_config: {sheets_config}")
+    print(f"gh_config: {gh_config}")
+    print(f"Additional arguments {kwargs}")
 '''
 
 
@@ -51,17 +58,19 @@ def sheetshuttle_run(
         "-kf",
         help="Path to the Sheets api keys, either .json or .env file",
     ),
-    sheets_config_directory: str = typer.Option(
-        "config/sheet_sources/",
-        "--sheets-config-directory",
-        "-sd",
-        help="Directory to get the sheets configuration .yaml files from",
+    sheets_config: List[Path] = typer.Option(
+        [],
+        "--sheets-config",
+        "-sc",
+        help="List of YAML files or directories containing YAML"
+        " files for Google Sheets configuration",
     ),
-    gh_config_directory: str = typer.Option(
-        "config/gh_sources/",
-        "--gh-config-directory",
-        "-gd",
-        help="Directory to get the Github configuration .yaml files from",
+    gh_config: List[Path] = typer.Option(
+        [],
+        "--github-config",
+        "-gh",
+        help="List of YAML files or directories containing YAML"
+        " files for GitHub configuration",
     ),
     plugins_directory: str = typer.Option(
         "plugins/",
@@ -93,8 +102,8 @@ def sheetshuttle_run(
         raise Exception(f"ERROR: function run was not found in {plugin_name} plugin.")
     my_plugin.run(
         sheets_keys_file,
-        sheets_config_directory,
-        gh_config_directory,
+        sheets_config,
+        gh_config,
         args=load_json_file(json_args),
     )
 

sheetshuttle/sheet_collector.py

@@ -2,9 +2,9 @@
 
 import json
 import os
-import pathlib
 import pickle
 from typing import Dict, List
+from pathlib import Path, PosixPath
 
 import pandas as pd  # type: ignore[import]
 import yaml
@@ -112,26 +112,27 @@ class MissingAuthenticationVariable(Exception):
 class SheetCollector:
     """Authenticate Sheets api and store retrieved data."""
 
-    def __init__(self, key_file=".env", sources_dir="config/sheet_sources") -> None:
+    def __init__(self, yaml_sources: List[Path], key_file=".env") -> None:
         """
         Create a SheetCollector object that stores a dictionary of sheets.
 
         Uses the yaml files in the config/sheet_sources directory.
 
         Args:
+            yaml_sources (List[Path]): List of paths where the sheet collector
+            should retrive yamls from. Must be yaml files.
+
             key_file (str, optional): path to Google Sheets API user keys and
             tokens. Defaults to ".env".
-
-            sources_dir (str, optional): path to where the configuration
-            is stored. Defaults to "config/sheet_sources"
         """
         self.key_file: str = key_file
         (
             self.credentials,
             self.service,
             self.sheets,
         ) = SheetCollector.authenticate_api(self.key_file)
-        self.config_dir = pathlib.Path(sources_dir)
+        assert yaml_sources, "Sheets config sources cannot be empty"
+        self.yaml_sources = yaml_sources
         self.sheets_data: Dict[str, Sheet] = {}
 
     def print_contents(self) -> None:
@@ -150,11 +151,22 @@ def collect_files(self) -> None:
         """
         if not self.sheets:
             raise Exception("ERROR: Collector was not authenticated")
-        # get a list of all yaml and yml path objects in the config_dir
-        config_files: List[pathlib.Path] = util.get_yaml_files(self.config_dir)
-        if not config_files:
-            raise Exception(f"ERROR: No configuration files found in {self.config_dir}")
-        for yaml_file in config_files:
+        # Check all yaml sources before trying to open them
+        issues: List[str] = []
+        for source in self.yaml_sources:
+            if source.suffix not in [".yaml", ".yml"]:
+                issues.append(f"\t-{str(source)} is not a YAML file\n")
+            if not source.is_file():
+                issues.append(f"\t-{str(source)} is not a file\n")
+            if not source.exists():
+                issues.append(f"\t-{str(source)} does not exist\n")
+        # if issues has some items then exit and print out the issues
+        if issues:
+            raise Exception(
+                "Error while collecting yaml configurations."
+                f' Here are the found issues:\n{"".join(issues)}'
+            )
+        for yaml_file in self.yaml_sources:
             # Open yaml file as read
             with open(yaml_file, "r", encoding="utf-8") as config_file:
                 config_data = yaml.safe_load(config_file)
@@ -415,23 +427,21 @@ def print_region(self):
         print(f"end range: {self.end_range}")
         print(self.data.to_markdown())
 
-    def region_to_pickle(self, directory: pathlib.PosixPath):
+    def region_to_pickle(self, directory: PosixPath):
         """Write the region object to a Pickle file.
 
         Args:
-            directory (pathlib.PosixPath): path to the directory where the file
+            directory (PosixPath): path to the directory where the file
                 be stored
         """
-        with open(
-            pathlib.Path(".") / directory / f"{self.full_name}.pkl", "wb"
-        ) as outfile:
+        with open(Path(".") / directory / f"{self.full_name}.pkl", "wb") as outfile:
             pickle.dump(self, outfile)
 
-    def region_to_json(self, directory: pathlib.PosixPath):
+    def region_to_json(self, directory: PosixPath):
         """Write the region object to a JSON file.
 
         Args:
-            directory (pathlib.PosixPath): path to the directory where the file
+            directory (PosixPath): path to the directory where the file
                 be stored
         """
         self_data = {
@@ -443,7 +453,7 @@ def region_to_json(self, directory: pathlib.PosixPath):
             "data": self.data.to_dict("index"),
         }
         with open(
-            pathlib.Path(".") / directory / f"{self.full_name}.json",
+            Path(".") / directory / f"{self.full_name}.json",
             "w+",
             encoding="utf-8",
         ) as outfile:

tests/test_sheet_collector.py

@@ -236,7 +236,9 @@ def test_sheet_execute_sheet_call_no_error():
     Will be skipped if authentication environment variables don't exist."""
     # initialize with environment variables authentication
     try:
-        my_sheet_collector = sheet_collector.SheetCollector()
+        my_sheet_collector = sheet_collector.SheetCollector(
+            pathlib.Path("example.yaml")
+        )
     except sheet_collector.MissingAuthenticationVariable:
         pytest.skip("Sheets authentication environment variables not found")
     expected_data = [
@@ -265,7 +267,9 @@ def test_sheet_execute_sheet_call_empty_return():
     Will be skipped if authentication environment variables don't exist."""
     # initialize with environment variables authentication
     try:
-        my_sheet_collector = sheet_collector.SheetCollector()
+        my_sheet_collector = sheet_collector.SheetCollector(
+            pathlib.Path("example.yaml")
+        )
     except sheet_collector.MissingAuthenticationVariable:
         pytest.skip("Sheets authentication environment variables not found")
     api = my_sheet_collector.sheets
@@ -284,7 +288,9 @@ def test_sheet_execute_sheet_call_throws_error():
     Will be skipped if authentication environment variables don't exist."""
     # initialize with environment variables authentication
     try:
-        my_sheet_collector = sheet_collector.SheetCollector()
+        my_sheet_collector = sheet_collector.SheetCollector(
+            pathlib.Path("example.yaml")
+        )
     except sheet_collector.MissingAuthenticationVariable:
         pytest.skip("Sheets authentication environment variables not found")
     api = my_sheet_collector.sheets
@@ -305,7 +311,9 @@ def test_sheet_collect_regions(test_data):
 
     Can be skipped if authentication variables aren't available"""
     try:
-        my_sheet_collector = sheet_collector.SheetCollector()
+        my_sheet_collector = sheet_collector.SheetCollector(
+            pathlib.Path("example.yaml")
+        )
     except sheet_collector.MissingAuthenticationVariable:
         pytest.skip("Sheets authentication environment variables not found")
     api = my_sheet_collector.sheets
@@ -391,7 +399,7 @@ def test_sheet_collector_collect_files_throws_error():
 
     Can be skipped if environment variables isn't set."""
     try:
-        my_collector = sheet_collector.SheetCollector()
+        my_collector = sheet_collector.SheetCollector(pathlib.Path("example.yaml"))
     except sheet_collector.MissingAuthenticationVariable:
         pytest.skip("Sheets authentication environment variables not found")
     my_collector.sheets = None
@@ -402,20 +410,21 @@ def test_sheet_collector_collect_files_throws_error():
 def test_sheet_collector_collect_files_prints_output(tmpdir, test_data):
     """Check that a dictionary of Sheet objects is created correctly in collect_files()"""
     try:
-        my_collector = sheet_collector.SheetCollector()
+        my_collector = sheet_collector.SheetCollector(pathlib.Path("example.yaml"))
     except sheet_collector.MissingAuthenticationVariable:
         pytest.skip("Sheets authentication environment variables not found")
 
     # setting up temporary config files using test_data
     temporary_directory = tmpdir.mkdir("temp")
     temp_path = str(temporary_directory)
+    config_list = []
     for file_name, config_val in test_data["collect_files_test"]["temp_files"].items():
-        with open(
-            pathlib.Path(".") / temp_path / file_name, "w+", encoding="utf-8"
-        ) as outfile:
+        current_file = pathlib.Path(".") / temp_path / file_name
+        config_list.append(current_file)
+        with open(current_file, "w+", encoding="utf-8") as outfile:
             yaml.dump(config_val, outfile)
     # Initialize the sheet collector and collect the files from the temporary directory
-    my_collector = sheet_collector.SheetCollector(sources_dir=temp_path)
+    my_collector = sheet_collector.SheetCollector(config_list)
     my_collector.collect_files()
     for sheet_key in test_data["collect_files_test"]["expected_keys"]:
         assert sheet_key in my_collector.sheets_data