Integrated Amazon S3 for fallback when Google Drive fails

- Integrated Amazon S3
- Updated requirements
- Updated manifest(s)
- Added env variable support for Google API/Amazon API

.gitignore

@@ -160,3 +160,4 @@ tester.py
 /proxyshop/datas/
 /proxyshop/version_tracker.json
 /release
+/proxyshop/env.json

proxyshop/constants.py

@@ -316,6 +316,15 @@
     ' WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'
 }
 
+# API Keys
+with open(os.path.join(cwd, "proxyshop/env.json"), "r", encoding="utf-8") as api_keys:
+    keys = json.load(api_keys)
+    g_api = keys['g_api']
+    a_api = keys['a_api']
+# Manual definitions
+# g_api = ""
+# a_api = ""
+
 
 # For object permanence
 class Singleton(type):
@@ -421,6 +430,10 @@ def load_values(self):
         # Version tracker
         self.versions = versions
 
+        # API keys
+        self.google_api = g_api
+        self.amazon_api = a_api
+
     def reload(self):
         self.load_values()
 

proxyshop/core.py

@@ -5,16 +5,11 @@
 import re
 import sys
 import json
-import pydrive2.auth
+import requests
 from glob import glob
 from pathlib import Path
 from typing import Optional, Callable
 from importlib import util, import_module
-
-import requests
-from pydrive2.auth import GoogleAuth
-from pydrive2.drive import GoogleDrive
-
 from proxyshop import gdown
 from proxyshop.constants import con
 cwd = os.getcwd()
@@ -200,7 +195,13 @@ def check_for_updates():
 
     # Base app manifest
     with open("proxyshop/manifest.json", encoding="utf-8") as f:
-        for cat, temps in json.load(f).items():
+        # Get config info
+        data = json.load(f)
+        s3_bucket = data['__CONFIG__']['S3']
+        data.pop("__CONFIG__")
+
+        # Build update dict
+        for cat, temps in data.items():
             for name, temp in temps.items():
 
                 # Is the ID valid?
@@ -211,6 +212,7 @@ def check_for_updates():
                 temp['name'] = name
                 temp['plugin'] = None
                 temp['manifest'] = os.path.join(cwd, 'proxyshop/manifest.json')
+                temp['s3'] = s3_bucket
 
                 # Does this template need an update?
                 file = version_check(temp)
@@ -232,7 +234,16 @@ def check_for_updates():
     # Check the manifest of each plugin
     for plug in plugins:
         with open(plug['path'], encoding="utf-8") as f:
-            for cat, temps in json.load(f).items():
+            # Check for S3 compatibility
+            data = json.load(f)
+            if "__CONFIG__" in data:
+                if "S3" in data['__CONFIG__']:
+                    s3_bucket = data['__CONFIG__']['S3']
+                else: s3_bucket = None
+                data.pop("__CONFIG__")
+
+            # Append to the updates dict
+            for cat, temps in data.items():
                 for name, temp in temps.items():
 
                     # Is the ID valid?
@@ -243,6 +254,7 @@ def check_for_updates():
                     temp['name'] = name
                     temp['plugin'] = plug['name']
                     temp['manifest'] = plug['path']
+                    temp['s3'] = s3_bucket
 
                     # Does this template need an update?
                     file = version_check(temp, plug['name'])
@@ -282,7 +294,8 @@ def version_check(temp: dict, plugin: Optional[str] = None):
         'plugin': temp['plugin'],
         'version_old': current,
         'version_new': data['description'],
-        'size': data['size']
+        'size': data['size'],
+        's3': temp['s3']
     }
 
     # Yes update if file has never been downloaded
@@ -326,14 +339,19 @@ def get_current_version(file_id: str, path: str):
     return version
 
 
-def update_template(temp: dict, callback: Callable):
+def update_template(temp: dict, callback: Callable, s3_callback: Callable):
     """
     Update a given template to the latest version.
     @param temp: Dict containing template information.
     @param callback: Callback method to update progress bar.
+    @param s3_callback: Callback method to update progress bar for S3 downloading.
     """
     # Download using authorization
     result = gdown.download(temp['id'], temp['path'], callback)
+    if not result:
+        if temp['s3']:
+            # Try grabbing from Amazon S3 instead
+            result = gdown.download_s3(temp, s3_callback)
 
     # Change the version to match the new version
     if result:
@@ -349,8 +367,7 @@ def gdrive_metadata(file_id: str):
     @return: Dict of metadata
     """
     source = "https://www.googleapis.com/drive/" \
-             f"v3/files/{file_id}?alt=json&fields=description,name,size&" \
-             "key=AIzaSyD2WloDIEefGe2LY48K5wMQTQcyChxZqiw"
+             f"v3/files/{file_id}?alt=json&fields=description,name,size&key={con.google_api}"
     return requests.get(source, headers=con.http_header).json()
 
 

proxyshop/gdown.py

@@ -4,6 +4,7 @@
 License: https://github.com/wkentaro/gdown/blob/main/LICENSE
 """
 from __future__ import print_function
+import json
 from pathlib import Path
 import os
 import os.path as osp
@@ -12,10 +13,17 @@
 import sys
 import tempfile
 import textwrap
+from typing import Callable
+
+import boto3
 import requests
+from boto3.s3.transfer import TransferConfig
+from s3transfer.constants import ALLOWED_DOWNLOAD_ARGS
+
+from proxyshop.constants import con
 
 CHUNK_SIZE = 1024 * 1024  # 512KB
-home = osp.expanduser("~")
+cwd = os.getcwd()
 
 
 def get_url_from_gdrive_confirmation(contents):
@@ -53,7 +61,8 @@ def get_url_from_gdrive_confirmation(contents):
 def download(
     file_id: str,
     path: str,
-    callback: any
+    callback: any,
+    use_cookies: bool = True
 ):
     """
     Download file from Gdrive ID.
@@ -66,6 +75,8 @@ def download(
         Output path.
     callback: any
         Function to call on each chunk downloaded.
+    use_cookies: bool
+        Use cookies with request.
 
     Returns
     -------
@@ -80,12 +91,32 @@ def download(
         "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"  # NOQA
     }
 
+    # Cookies
+    cache_dir = osp.join(cwd, "tmp")
+    if not osp.exists(cache_dir):
+        os.makedirs(cache_dir)
+    cookies_file = osp.join(cache_dir, "cookies.json")
+    if osp.exists(cookies_file) and use_cookies:
+        with open(cookies_file) as f:
+            cookies = json.load(f)
+        for k, v in cookies:
+            sess.cookies[k] = v
+
     # Get file resource
     while True:
         res = sess.get(url, headers=headers, stream=True, verify=True)
 
+        # Save cookies
+        with open(cookies_file, "w") as f:
+            cookies = [
+                (k, v)
+                for k, v in sess.cookies.items()
+                if not k.startswith("download_warning_")
+            ]
+            json.dump(cookies, f, indent=2)
+
+        # Is this the right file?
         if "Content-Disposition" in res.headers:
-            # This is the file
             break
 
         # Need to redirect with confirmation
@@ -139,7 +170,6 @@ def download(
 
     # Try to download
     try:
-
         for chunk in res.iter_content(chunk_size=CHUNK_SIZE):
             f.write(chunk)
             current += int(CHUNK_SIZE)
@@ -153,5 +183,32 @@ def download(
         return False
     finally:
         sess.close()
-
     return True
+
+
+def download_s3(temp: dict, callback: Callable) -> bool:
+    """
+    Download template from Amazon S3 bucket.
+    @param temp: Dict containing template data.
+    @param callback: Callback function to update progress.
+    @return: True if success, False if failed.
+    """
+    # Establish this object's key
+    if temp['plugin']:
+        key = f"{temp['plugin']}/{temp['filename']}"
+    else: key = temp['filename']
+
+    # Establish S3 client
+    client_s3 = boto3.client(
+        "s3",
+        aws_access_key_id=con.amazon_api['access'],
+        aws_secret_access_key=con.amazon_api['secret']
+    )
+
+    # Attempt the download
+    try:
+        client_s3.download_file(temp['s3'], key, temp['path'], Callback=callback)
+        return True
+    except Exception as e:
+        print(e)
+        return False

proxyshop/gui.py

@@ -320,10 +320,12 @@ def __init__(self, parent: Popup, temp: dict, bg_color: str, **kwargs):
         super().__init__(**kwargs)
 
     async def download_update(self, download: BoxLayout) -> None:
-        self.progress = ProgressBar()
+        self.progress = UpdateProgress(self.data['size'])
         download.clear_widgets()
         download.add_widget(self.progress)
-        result = await ak.run_in_thread(lambda: update_template(self.data, self.update_progress), daemon=True)
+        result = await ak.run_in_thread(
+            lambda: update_template(self.data, self.progress.update_progress, self.progress.s3_update_progress
+        ), daemon=True)
         await ak.sleep(.5)
         if result:
             self.root.ids.container.remove_widget(self.root.entries[self.data['id']])
@@ -336,6 +338,20 @@ def update_progress(self, tran: int, total: int) -> None:
         self.progress.value = progress
 
 
+class UpdateProgress(ProgressBar):
+    def __init__(self, size, **kwargs):
+        super().__init__(**kwargs)
+        self.download_size = int(size)
+        self.current = 0
+
+    def update_progress(self, tran: int, total: int) -> None:
+        self.value = int((tran / total) * 100)
+
+    def s3_update_progress(self, tran: int) -> None:
+        self.current += tran
+        self.value = int((self.current / self.download_size) * 100)
+
+
 """
 UTILITY FUNCTIONS
 """

proxyshop/manifest.json

@@ -1,4 +1,7 @@
 {
+    "__CONFIG__": {
+        "S3": "proxyshop-gh-app"
+    },
     "Normal": {
         "Normal": {
             "file": "normal.psd",

proxyshop/plugins/MrTeferi/manifest.json

@@ -1,4 +1,7 @@
 {
+	"__CONFIG__": {
+        "S3": "proxyshop-gh-app"
+    },
 	"Normal": {
 		"Sketch": {
 			"file": "sketch.psd",

proxyshop/plugins/SilvanMTG/manifest.json

@@ -1,4 +1,7 @@
 {
+  "__CONFIG__": {
+        "S3": "proxyshop-gh-app"
+    },
   "Normal": {
     "Silvan Extended": {
       "file": "extended.psd",