Build latest changes.

build/lib/dataEng_container_tools/cla.py

@@ -2,7 +2,7 @@
 import json
 import sys
 from enum import Enum
-from .safe_stdout import setup_stdout, default_gcs_secret_locations
+from .safe_stdout import setup_stdout, default_gcs_secret_locations, secrets_files
 import os
 
 class custom_command_line_argument:
@@ -175,9 +175,27 @@ def get_output_uris(self):
         return output
 
     def get_secret_locations(self):
-        if not self.__secret_locations:
+        if self.__secret_locations:
+            return self.__args.gcs_secret_locations
+        if len(secrets_files) > 0:
+            return secrets_files
+        return None
+
+    def get_secrets(self):
+        return_list = {}
+        secret_list = None
+        if self.__secret_locations:
+            secret_list = self.__args.gcs_secret_locations
+        elif len(secrets_files) > 0:
+            secret_list = secrets_files
+        else:
             return None
-        return self.__args.gcs_secret_locations
+        for item in secret_list:
+            try:
+                return_list[item.split('/')[-1]] = json.load(open(item,'r'))
+            except ValueError:
+                print(item, "is not a properly formatted json file.")
+        return return_list
 
     def check_args(self):
         #TODO: Implement this

build/lib/dataEng_container_tools/safe_stdout.py

@@ -4,6 +4,7 @@
 
 default_secret_folder = '/vault/secrets/'
 default_gcs_secret_locations = [default_secret_folder + 'gcp-sa-storage.json']
+secrets_files = []
 
 class safe_stdout:
     def __init__(self, bad_words):
@@ -56,6 +57,7 @@ def setup_default_stdout(folder = default_secret_folder):
             secret = json.load(open(file,'r'))
         except ValueError:
             print(file, "is not a properly formatted json file.")
+        secrets_files.append(file)
         these_bad_words = set(secret.values())
         bad_words.update(these_bad_words)
         for word in these_bad_words:

dataEng_container_tools.egg-info/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: dataEng-container-tools
-Version: 0.1.8
+Version: 0.1.9
 Summary: A package containing tools for data engineering containers.
 Home-page: https://github.com/colpal/dataEng-container-tools
 Author: Alexander Saff
 Author-email: [email protected]
 License: UNKNOWN
 Description: # Data Engineering Container Tools
 
-        This packages is split into three parts: **CLA**, **GCS**, and **safe_stdout**.
+        This packages is split into four parts: **CLA**, **GCS**, **safe_stdout**, and **simple_setup**.
 
         ## CLA:
         Deals with receiving input from the command line. Has three classes: `custom_command_line_argument`, `command_line_argument_type`, and `command_line_arguments`.
@@ -106,7 +106,7 @@ Description: # Data Engineering Container Tools
         * `default_secret_folder`: Variable containing the folder in which secrets are injected by default. Currently set to `'/vault/secrets/'`.
 
         ## Example:
-        An example workflow might look something like this:
+        An example workflow using the classes above might look something like this:
         ```
         from dataEng_container_tools.cla import command_line_arguments, command_line_argument_type
         from dataEng_container_tools.gcs import gcs_file_io
@@ -126,6 +126,19 @@ Description: # Data Engineering Container Tools
         result = file_io.upload_file_from_object(gcs_uri=output_uris[0], object_to_upload=pqt_obj)
         ```
 
+        ## simple_setup:
+        A simple way to get input from the command line, and download and upload documents to/from GCS. Fewer options than the classes above but also fewer lines of code to write. A brief example (documentation to come):
+        ```
+        from dataEng_container_tools.simple_setup import simple_setup
+        simple = simple_setup(['input_left', 'input_right', 'output_inner', 'output_outer', 'secret_location', 'example_flag'])
+        objects = simple.get_input_objects()
+        #
+        # Edit the objects in some way here.
+        #
+        return_objs = {'output_outer': objects['input_left'], 'output_inner': objects['input_right']}
+        upload = simp.upload_objects(return_objs)
+        ```
+
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License

setup.py

@@ -6,7 +6,7 @@
 
 setuptools.setup(
     name="dataEng-container-tools",
-    version="0.1.8",
+    version="0.1.9",
     author="Alexander Saff",
     author_email="[email protected]",
     description="A package containing tools for data engineering containers.",