Skip to content

Commit

Permalink
gdrive fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
Michael Chase committed Nov 17, 2014
1 parent 679ebd9 commit 2e06c47
Show file tree
Hide file tree
Showing 8 changed files with 130 additions and 94 deletions.
2 changes: 2 additions & 0 deletions BitcasaFileFetcher/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ def parse(self):
def run_download(self, upload=False):
"""Run the main program checks"""
self.run_level = Args.RUN_LEVEL_MAIN
self.args.src = self.args.src.decode("utf-8")
self.args.dst = self.args.dst.decode("utf-8")
self.args.upload = upload
if not upload:
self.args.local = False
Expand Down
91 changes: 56 additions & 35 deletions BitcasaFileFetcher/threads/folder_traverse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from lib.bitcasa import BitcasaClient, BitcasaFolder, BitcasaFile
from lib.bitcasa.exception import BitcasaException
from lib.gdrive import GoogleDrive
from googleapiclient.errors import HttpError
from lib.googleapiclient.errors import HttpError
from helpers import utils
log = logging.getLogger("BitcasaFileFetcher")

Expand Down Expand Up @@ -86,15 +86,15 @@ def get_local_items(fold, should_exit, results):
fullpath = os.path.join(fold.path, item)
if should_exit.is_set():
break
filesize = 0
filesize = None
try:
if not os.path.isdir(fullpath):
filesize = os.path.getsize(fullpath)
except OSError:
log.exception("Error getting file info")
results.writeError(item, fullpath, "", "Error listing file %s" % item)
continue
if filesize:
if filesize is not None:
bitem = BitcasaFile(None, fullpath, item, None, filesize)
else:
bitem = BitcasaFolder(None, item, fullpath)
Expand Down Expand Up @@ -149,7 +149,7 @@ def folder_list_gdrive(folder, status, results, args, should_exit, g):
if isinstance(item, BitcasaFile):
filesize = item.size
retriesleft = 10
apiratecount = 0
apiratecount = 1
while not should_exit.is_set() and retriesleft > 0:
try:
needtoupload = g.need_to_upload(nm, folder_id, filesize)
Expand All @@ -160,29 +160,29 @@ def folder_list_gdrive(folder, status, results, args, should_exit, g):
retriesleft += 1
log.warn("Google API rate limit reached. Will retry")
else:
log.exception("Error checking is file exists will retry %s more times", retriesleft)
log.exception("Error checking if %s exists will retry %s more times", nm, retriesleft)

if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(nm, tfd, base64_path, "Error queuing file %s" % filename)
results.writeError(nm, tfd, base64_path, "Error queuing file %s" % nm)
except:
retriesleft -= 1
log.exception("Error checking is file exists will retry %s more times", retriesleft)
log.exception("Error checking if %s exists will retry %s more times", nm, retriesleft)
if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(nm, tfd, base64_path, "Error queuing file %s" % filename)
results.writeError(nm, tfd, base64_path, "Error queuing file %s" % nm)
else:
retriesleft = 0
if should_exit.is_set():
log.debug("Stopping folder list")
return
elif needtoupload:
if needtoupload:
if args.dryrun:
if not args.silentqueuer:
log.debug("%s %s", nm, filesize)
results.writeSuccess(tfd, base64_path)
results.writeSuccess(tfd, base64_path)
else:
if not args.silentqueuer:
log.debug("Queuing file download for %s", nm)
Expand All @@ -206,30 +206,51 @@ def folder_list_gdrive(folder, status, results, args, should_exit, g):
if should_exit.is_set():
log.debug("Stopping folder list")
return
elif args.rec and (not args.depth or args.depth > depth):
g_fold = g.get_folder_byname(nm, parent=folder_id, createnotfound=cnf)
remainingtries = 5
while not should_exit.is_set() and g_fold is None and remainingtries > 0:
remainingtries -= 1
log.error("Will retry to get/create %s %s more times", nm, remainingtries)
time.sleep(5)
g_fold = g.get_folder_byname(nm, parent=folder_id, createnotfound=cnf)
if should_exit.is_set():
log.debug("Stopping folder list")
return
elif g_fold is None:
log.error("Failed to get/create folder")
return
if not args.silentqueuer:
log.debug("Queuing folder listing for %s", nm)
folder = {
"folder": item,
"depth": (depth+1),
"path": tfd,
"folder_id": g_fold["id"]
}
status.queue(folder)
except: #Hopefully this won't get called
if not args.rec or ( args.depth and args.depth <= depth ):
continue
retriesleft = 10
apiratecount = 1
while not should_exit.is_set() and retriesleft > 0:
try:
g_fold = g.get_folder_byname(nm, parent=folder_id, createnotfound=True)
except HttpError as e:
retriesleft -= 1
if e.resp.status == 403:
apiratecount += 1
retriesleft += 1
log.warn("Google API rate limit reached. Will retry")
else:
log.exception("Will retry to get/create %s %s more times", nm, retriesleft)

if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(nm, tfd, base64_path, "Failed to get/create folder %s" % nm)
continue
except:
retriesleft -= 1
log.error("Will retry to get/create %s %s more times", nm, retriesleft)
if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(nm, tfd, base64_path, "Failed to get/create folder %s" % nm)
continue
else:
retriesleft = 0

if should_exit.is_set():
log.debug("Stopping folder list")
return
if not args.silentqueuer:
log.debug("Queuing folder listing for %s", nm)
folder = {
"folder": item,
"depth": (depth+1),
"path": tfd,
"folder_id": g_fold["id"]
}
status.queue(folder)
except:
results.writeError(nm, tfd, base64_path, traceback.format_exc())

def folder_list(folder, status, results, args, should_exit):
Expand Down Expand Up @@ -310,7 +331,7 @@ def folder_list(folder, status, results, args, should_exit):
if args.dryrun:
if not args.silentqueuer:
log.debug("%s %s", nm, filesize)
results.writeSuccess(tfd, base64_path)
results.writeSuccess(tfd, base64_path)
else:
if not args.silentqueuer:
log.debug("Queuing file download for %s", nm)
Expand Down
3 changes: 1 addition & 2 deletions BitcasaFileFetcher/threads/upload.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import time, os, logging
from helpers import utils
from lib.gdrive import GoogleDrive
from googleapiclient.errors import HttpError
import requests
from lib.googleapiclient.errors import HttpError
from Queue import Empty as EmptyException

log = logging.getLogger("BitcasaFileFetcher")
Expand Down
7 changes: 5 additions & 2 deletions BitcasaFileLister/server.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import sys, logging
import sys, logging, json
from lib import bottle, cherrypy, BitcasaUtils
from lib.bottle import route, run, request, get, post, template, response, view, static_file, redirect
from lib.bitcasa import BitcasaException, BitcasaFile
Expand Down Expand Up @@ -93,8 +93,11 @@ def do_bitcasa_auth():
error_msg = "Storing permanent token %s" % client.access_token
log.info(error_msg)
try:
with open(utils.BITCASA_TOKEN, "r") as tokenfile:
json_token = json.loads(tokenfile.read())
with open(utils.BITCASA_TOKEN, "w") as tokenfile:
tokenfile.write(client.access_token)
json_token["bitcasa"]["TOKEN"] = client.access_token
tokenfile.write(json.dumps(json_token, indent=4))
except Exception as e:
auth_name="Login"
auth_url="/bitcasafilelister/auth"
Expand Down
7 changes: 7 additions & 0 deletions bitcasa_sample.ini
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"bitcasa": {
"CLIENTID": "758ab3de",
"CLIENTSECRET": "5669c999ac340185a7c80c28d12a4319",
"TOKEN": ""
}
}
3 changes: 1 addition & 2 deletions includes/helpers/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
CLIENTID = "758ab3de"
CLIENTSECRET = "5669c999ac340185a7c80c28d12a4319"
SERVER_HOST = "localhost"
SERVER_PORT = 1115
SERVER_URL = "http://%s:%s/bitcasafilelister" % (SERVER_HOST, SERVER_PORT)
REDIRECT_URI = "%s/auth" % SERVER_URL

import math, os, hashlib, logging, tempfile
BITCASA_TOKEN = os.path.abspath("bitcasa.ini")
BITCASA_SAMPLE_TOKEN = os.path.abspath("bitcasa_sample.ini")
GDRIVE_CREDS = os.path.abspath("gdrive.ini")
GDRIVE_SECRETS = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../lib", "gdrive_secrets.ini"))

Expand Down
28 changes: 24 additions & 4 deletions includes/lib/bitcasa_utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import os, logging, webbrowser, time
import os, logging, webbrowser, time, json
from helpers import utils
from bitcasa import BitcasaClient
from bitcasa import BitcasaException
Expand All @@ -9,14 +9,34 @@ class BitcasaUtils(object):
def __init__(self):
self.client = None
self.token = None
self.client_id = None
self.client_secret = None
self.get_bitcasa_token()

def get_bitcasa_token(self):
if os.path.isfile(utils.BITCASA_TOKEN):
try:
with open(utils.BITCASA_TOKEN, "r") as tokenfile:
self.token = tokenfile.read()
log.debug("Got token")
self.token = tokenfile.read().rstrip()
try:
tokens_json = json.loads(self.token)
except ValueError:
log.info("Converting bitcasa.ini")
log.info("If you are using a custom CLIENTID and CLIENTSECRET please put them in bitcasa.ini")
with open(utils.BITCASA_SAMPLE_TOKEN, "r") as sample, open(utils.BITCASA_TOKEN, "w+") as tokenfile:
json_sample = json.loads(sample.read())
self.client_id = json_sample["bitcasa"]["CLIENTID"]
self.client_secret = json_sample["bitcasa"]["CLIENTSECRET"]
json_sample["bitcasa"]["TOKEN"] = self.token
tokenfile.write(json.dumps(json_sample, indent=4))
else:
self.client_id = tokens_json["bitcasa"]["CLIENTID"]
self.client_secret = tokens_json["bitcasa"]["CLIENTSECRET"]
self.token = tokens_json["bitcasa"]["TOKEN"]
if self.token:
log.debug("Got token")
else:
log.error("No token stored")
except:
log.exception("Failed to read Bitcasa token file")
else:
Expand All @@ -25,7 +45,7 @@ def get_bitcasa_token(self):

def create_client(self, force=False, redirect_uri=utils.REDIRECT_URI):
if (self.token or force) and not self.client:
self.client = BitcasaClient(utils.CLIENTID, utils.CLIENTSECRET, redirect_uri, self.token)
self.client = BitcasaClient(self.client_id, self.client_secret, redirect_uri, self.token)
return self.client

def test_auth(self):
Expand Down
83 changes: 34 additions & 49 deletions includes/lib/gdrive.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
import httplib2, webbrowser, sys, time, logging, os
sys.path.insert(1, "BitcasaFileFetcher/lib/")
from datetime import datetime
from googleapiclient import errors
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
from oauth2client.client import OAuth2WebServerFlow, AccessTokenRefreshError, flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import run_flow as RunFlow
import webbrowser, sys, time, logging
from lib import httplib2
from lib.googleapiclient import errors
from lib.googleapiclient.discovery import build
from lib.googleapiclient.http import MediaFileUpload
from lib.oauth2client.client import AccessTokenRefreshError, flow_from_clientsecrets
from lib.oauth2client.file import Storage
from helpers import utils

log = logging.getLogger("BitcasaFileFetcher")
Expand Down Expand Up @@ -80,55 +78,42 @@ def need_to_upload(self, filename, folder_id, size_bytes):
return False
elif myfile:
log.debug("Filesize incorrect deleting %s", filename)
self.delete_file(myfile["id"])
self.get_service().files().delete(fileId=myfile["id"])
return True
else:
return True

def get_folder_byname(self, foldername, parent="root", createnotfound=False):
try:
children = self.get_service().children().list(folderId=parent,
q="title = '%s' and mimeType = 'application/vnd.google-apps.folder'" % foldername.replace("'", "\\'")).execute()
items = children.get('items', [])

original = None
for child in items:
child_file = self.get_service().files().get(fileId=child["id"]).execute()

if original is None:
original = child_file
else:
o = time.strptime(original["createdDate"], "%Y-%m-%dT%H:%M:%S.%fZ")
t = time.strptime(child_file["createdDate"], "%Y-%m-%dT%H:%M:%S.%fZ")
if t < o:
original = child_file
children = self.get_service().children().list(folderId=parent,
q="title = '%s' and mimeType = 'application/vnd.google-apps.folder'" % foldername.replace("'", "\\'")).execute()
items = children.get('items', [])
original = None
for child in items:
child_file = self.get_service().files().get(fileId=child["id"]).execute()

if original is None:
if createnotfound:
log.info("No items by the name of %s found. Creating", foldername)
body = {
'title': foldername,
'mimeType':'application/vnd.google-apps.folder'
}
if parent != "root":
body['parents'] = [{'id':parent}]
return self.get_service().files().insert(body=body).execute()
else:
log.info("No items by the name of %s found", foldername)
return False
original = child_file
else:
return original
except errors.HttpError:
log.exception("Error getting or creating folder")
return None

o = time.strptime(original["createdDate"], "%Y-%m-%dT%H:%M:%S.%fZ")
t = time.strptime(child_file["createdDate"], "%Y-%m-%dT%H:%M:%S.%fZ")
if t < o:
original = child_file

def delete_file(self, fileid):
try:
self.get_service().files().delete(fileId=fileid)
except errors.HttpError:
log.exception("Error deleting file")
return False
if original is None:
if createnotfound:
log.debug("No items by the name of %s found. Creating", foldername)
body = {
'title': foldername,
'mimeType':'application/vnd.google-apps.folder'
}
if parent != "root":
body['parents'] = [{'id':parent}]
return self.get_service().files().insert(body=body).execute()
else:
log.debug("No items by the name of %s found", foldername)
return False
else:
return original

@property
def token_expired(self):
Expand Down

0 comments on commit 2e06c47

Please sign in to comment.