From de4056efbe61b8cd9b07374619a888b35c0b9492 Mon Sep 17 00:00:00 2001 From: MrTeferi Date: Tue, 16 Aug 2022 04:20:31 -0500 Subject: [PATCH] Integrated Amazon S3 for fallback when Google Drive fails - Integrated Amazon S3 - Updated requirements - Updated manifest(s) - Added env variable support for Google API/Amazon API --- .gitignore | 1 + proxyshop/constants.py | 13 +++++ proxyshop/core.py | 41 +++++++++---- proxyshop/gdown.py | 67 ++++++++++++++++++++-- proxyshop/gui.py | 20 ++++++- proxyshop/manifest.json | 3 + proxyshop/plugins/MrTeferi/manifest.json | 3 + proxyshop/plugins/SilvanMTG/manifest.json | 3 + requirements.txt | Bin 304 -> 282 bytes 9 files changed, 132 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index aa153806..8f66006f 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,4 @@ tester.py /proxyshop/datas/ /proxyshop/version_tracker.json /release +/proxyshop/env.json diff --git a/proxyshop/constants.py b/proxyshop/constants.py index 3295f5d8..4543b3a8 100644 --- a/proxyshop/constants.py +++ b/proxyshop/constants.py @@ -316,6 +316,15 @@ ' WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36' } +# API Keys +with open(os.path.join(cwd, "proxyshop/env.json"), "r", encoding="utf-8") as api_keys: + keys = json.load(api_keys) + g_api = keys['g_api'] + a_api = keys['a_api'] +# Manual definitions +# g_api = "" +# a_api = "" + # For object permanence class Singleton(type): @@ -421,6 +430,10 @@ def load_values(self): # Version tracker self.versions = versions + # API keys + self.google_api = g_api + self.amazon_api = a_api + def reload(self): self.load_values() diff --git a/proxyshop/core.py b/proxyshop/core.py index 843689f3..794f4281 100644 --- a/proxyshop/core.py +++ b/proxyshop/core.py @@ -5,16 +5,11 @@ import re import sys import json -import pydrive2.auth +import requests from glob import glob from pathlib import Path from typing import Optional, Callable from importlib import util, import_module - -import requests -from pydrive2.auth import GoogleAuth -from pydrive2.drive import GoogleDrive - from proxyshop import gdown from proxyshop.constants import con cwd = os.getcwd() @@ -200,7 +195,13 @@ def check_for_updates(): # Base app manifest with open("proxyshop/manifest.json", encoding="utf-8") as f: - for cat, temps in json.load(f).items(): + # Get config info + data = json.load(f) + s3_bucket = data['__CONFIG__']['S3'] + data.pop("__CONFIG__") + + # Build update dict + for cat, temps in data.items(): for name, temp in temps.items(): # Is the ID valid? @@ -211,6 +212,7 @@ def check_for_updates(): temp['name'] = name temp['plugin'] = None temp['manifest'] = os.path.join(cwd, 'proxyshop/manifest.json') + temp['s3'] = s3_bucket # Does this template need an update? file = version_check(temp) @@ -232,7 +234,16 @@ def check_for_updates(): # Check the manifest of each plugin for plug in plugins: with open(plug['path'], encoding="utf-8") as f: - for cat, temps in json.load(f).items(): + # Check for S3 compatibility + data = json.load(f) + if "__CONFIG__" in data: + if "S3" in data['__CONFIG__']: + s3_bucket = data['__CONFIG__']['S3'] + else: s3_bucket = None + data.pop("__CONFIG__") + + # Append to the updates dict + for cat, temps in data.items(): for name, temp in temps.items(): # Is the ID valid? @@ -243,6 +254,7 @@ def check_for_updates(): temp['name'] = name temp['plugin'] = plug['name'] temp['manifest'] = plug['path'] + temp['s3'] = s3_bucket # Does this template need an update? file = version_check(temp, plug['name']) @@ -282,7 +294,8 @@ def version_check(temp: dict, plugin: Optional[str] = None): 'plugin': temp['plugin'], 'version_old': current, 'version_new': data['description'], - 'size': data['size'] + 'size': data['size'], + 's3': temp['s3'] } # Yes update if file has never been downloaded @@ -326,14 +339,19 @@ def get_current_version(file_id: str, path: str): return version -def update_template(temp: dict, callback: Callable): +def update_template(temp: dict, callback: Callable, s3_callback: Callable): """ Update a given template to the latest version. @param temp: Dict containing template information. @param callback: Callback method to update progress bar. + @param s3_callback: Callback method to update progress bar for S3 downloading. """ # Download using authorization result = gdown.download(temp['id'], temp['path'], callback) + if not result: + if temp['s3']: + # Try grabbing from Amazon S3 instead + result = gdown.download_s3(temp, s3_callback) # Change the version to match the new version if result: @@ -349,8 +367,7 @@ def gdrive_metadata(file_id: str): @return: Dict of metadata """ source = "https://www.googleapis.com/drive/" \ - f"v3/files/{file_id}?alt=json&fields=description,name,size&" \ - "key=AIzaSyD2WloDIEefGe2LY48K5wMQTQcyChxZqiw" + f"v3/files/{file_id}?alt=json&fields=description,name,size&key={con.google_api}" return requests.get(source, headers=con.http_header).json() diff --git a/proxyshop/gdown.py b/proxyshop/gdown.py index f0f88f4d..b332efa3 100644 --- a/proxyshop/gdown.py +++ b/proxyshop/gdown.py @@ -4,6 +4,7 @@ License: https://github.com/wkentaro/gdown/blob/main/LICENSE """ from __future__ import print_function +import json from pathlib import Path import os import os.path as osp @@ -12,10 +13,17 @@ import sys import tempfile import textwrap +from typing import Callable + +import boto3 import requests +from boto3.s3.transfer import TransferConfig +from s3transfer.constants import ALLOWED_DOWNLOAD_ARGS + +from proxyshop.constants import con CHUNK_SIZE = 1024 * 1024 # 512KB -home = osp.expanduser("~") +cwd = os.getcwd() def get_url_from_gdrive_confirmation(contents): @@ -53,7 +61,8 @@ def get_url_from_gdrive_confirmation(contents): def download( file_id: str, path: str, - callback: any + callback: any, + use_cookies: bool = True ): """ Download file from Gdrive ID. @@ -66,6 +75,8 @@ def download( Output path. callback: any Function to call on each chunk downloaded. + use_cookies: bool + Use cookies with request. Returns ------- @@ -80,12 +91,32 @@ def download( "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36" # NOQA } + # Cookies + cache_dir = osp.join(cwd, "tmp") + if not osp.exists(cache_dir): + os.makedirs(cache_dir) + cookies_file = osp.join(cache_dir, "cookies.json") + if osp.exists(cookies_file) and use_cookies: + with open(cookies_file) as f: + cookies = json.load(f) + for k, v in cookies: + sess.cookies[k] = v + # Get file resource while True: res = sess.get(url, headers=headers, stream=True, verify=True) + # Save cookies + with open(cookies_file, "w") as f: + cookies = [ + (k, v) + for k, v in sess.cookies.items() + if not k.startswith("download_warning_") + ] + json.dump(cookies, f, indent=2) + + # Is this the right file? if "Content-Disposition" in res.headers: - # This is the file break # Need to redirect with confirmation @@ -139,7 +170,6 @@ def download( # Try to download try: - for chunk in res.iter_content(chunk_size=CHUNK_SIZE): f.write(chunk) current += int(CHUNK_SIZE) @@ -153,5 +183,32 @@ def download( return False finally: sess.close() - return True + + +def download_s3(temp: dict, callback: Callable) -> bool: + """ + Download template from Amazon S3 bucket. + @param temp: Dict containing template data. + @param callback: Callback function to update progress. + @return: True if success, False if failed. + """ + # Establish this object's key + if temp['plugin']: + key = f"{temp['plugin']}/{temp['filename']}" + else: key = temp['filename'] + + # Establish S3 client + client_s3 = boto3.client( + "s3", + aws_access_key_id=con.amazon_api['access'], + aws_secret_access_key=con.amazon_api['secret'] + ) + + # Attempt the download + try: + client_s3.download_file(temp['s3'], key, temp['path'], Callback=callback) + return True + except Exception as e: + print(e) + return False diff --git a/proxyshop/gui.py b/proxyshop/gui.py index 1b909842..258770fd 100644 --- a/proxyshop/gui.py +++ b/proxyshop/gui.py @@ -320,10 +320,12 @@ def __init__(self, parent: Popup, temp: dict, bg_color: str, **kwargs): super().__init__(**kwargs) async def download_update(self, download: BoxLayout) -> None: - self.progress = ProgressBar() + self.progress = UpdateProgress(self.data['size']) download.clear_widgets() download.add_widget(self.progress) - result = await ak.run_in_thread(lambda: update_template(self.data, self.update_progress), daemon=True) + result = await ak.run_in_thread( + lambda: update_template(self.data, self.progress.update_progress, self.progress.s3_update_progress + ), daemon=True) await ak.sleep(.5) if result: self.root.ids.container.remove_widget(self.root.entries[self.data['id']]) @@ -336,6 +338,20 @@ def update_progress(self, tran: int, total: int) -> None: self.progress.value = progress +class UpdateProgress(ProgressBar): + def __init__(self, size, **kwargs): + super().__init__(**kwargs) + self.download_size = int(size) + self.current = 0 + + def update_progress(self, tran: int, total: int) -> None: + self.value = int((tran / total) * 100) + + def s3_update_progress(self, tran: int) -> None: + self.current += tran + self.value = int((self.current / self.download_size) * 100) + + """ UTILITY FUNCTIONS """ diff --git a/proxyshop/manifest.json b/proxyshop/manifest.json index 010f861d..639ae145 100644 --- a/proxyshop/manifest.json +++ b/proxyshop/manifest.json @@ -1,4 +1,7 @@ { + "__CONFIG__": { + "S3": "proxyshop-gh-app" + }, "Normal": { "Normal": { "file": "normal.psd", diff --git a/proxyshop/plugins/MrTeferi/manifest.json b/proxyshop/plugins/MrTeferi/manifest.json index 54013dc5..c7d97dfb 100644 --- a/proxyshop/plugins/MrTeferi/manifest.json +++ b/proxyshop/plugins/MrTeferi/manifest.json @@ -1,4 +1,7 @@ { + "__CONFIG__": { + "S3": "proxyshop-gh-app" + }, "Normal": { "Sketch": { "file": "sketch.psd", diff --git a/proxyshop/plugins/SilvanMTG/manifest.json b/proxyshop/plugins/SilvanMTG/manifest.json index 47a8c4bd..4ce8bdb9 100644 --- a/proxyshop/plugins/SilvanMTG/manifest.json +++ b/proxyshop/plugins/SilvanMTG/manifest.json @@ -1,4 +1,7 @@ { + "__CONFIG__": { + "S3": "proxyshop-gh-app" + }, "Normal": { "Silvan Extended": { "file": "extended.psd", diff --git a/requirements.txt b/requirements.txt index 8703de77dc69a95361eaeab3da018af04ca66dcb..fdbce904d6b138fa8b7460d4fead4f7cda98ea14 100644 GIT binary patch delta 24 ecmdnMG>d6M&%_N4e7p=?3`q?63?)Ep%m4si*akQN delta 37 rcmbQmw1H_tk8}V-C4&n?5kn?J8AB?A5knn=ErTJ09uON$Z0-aAtwsow