From f91b06f3ee7092016087508fbbcec76efa5c1694 Mon Sep 17 00:00:00 2001 From: jaideep-seth Date: Fri, 2 Aug 2019 20:38:23 +0530 Subject: [PATCH] ok --- PyOpenWorm/bittorrent.py | 94 ++++++++++++++++++++++++++++++++++++++-- setup.py | 3 ++ tests/BitTorrentTest.py | 35 +++++++++++++++ 3 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 tests/BitTorrentTest.py diff --git a/PyOpenWorm/bittorrent.py b/PyOpenWorm/bittorrent.py index 5e18687af..78d16b7f8 100644 --- a/PyOpenWorm/bittorrent.py +++ b/PyOpenWorm/bittorrent.py @@ -1,7 +1,95 @@ -from .datasource_loader import DataSourceDirLoader +from __future__ import print_function +from rdflib.term import URIRef +from PyOpenWorm.data_trans.csv_ds import CSVDataSource +from PyOpenWorm.data_trans.connections import ConnectomeCSVDataSource +from PyOpenWorm.context import Context +from PyOpenWorm.datasource import DataTranslator, DataSource +from PyOpenWorm import connect +from PyOpenWorm.datasource_loader import DataSourceDirLoader +import pickle +import os.path +from googleapiclient.discovery import build +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request +import io +from apiclient.http import MediaIoBaseDownload, MediaFileUpload +# If modifying these scopes, delete the file token.pickle. +SCOPES = ['https://www.googleapis.com/auth/drive'] +def download_torrent(torrent_name): + """Shows basic usage of the Drive v3 API. + Prints the names and ids of the first 10 files the user has access to. + """ + creds = None + # The file token.pickle stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first + # time. + if os.path.exists('.pow/temp/token.pickle'): + with open('.pow/temp/token.pickle', 'rb') as token: + creds = pickle.load(token) + # If there are no (valid) credentials available, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + '.pow/temp/credentials.json', SCOPES) + creds = flow.run_local_server() + # Save the credentials for the next run + with open('.pow/temp/token.pickle', 'wb') as token: + pickle.dump(creds, token) + + service = build('drive', 'v3', credentials=creds) + + # Call the Drive v3 API + results = service.files().list( + pageSize=10, fields="nextPageToken, files(id, name, description)").execute() + items = results.get('files', []) + + selected_file_name = None + selected_file_id = None + if not items: + print('No files found.') + else: + print('Files:') + for item in items: + name = item['name'] + ids = item['id'] + dex = item['description'] + #Added this .torrent to Google Drives - It contains Merged_Nuclei_Stained_Worm(300MB) + if name == torrent_name: + selected_file_id = ids + selected_file_name = name + + + request = service.files().get_media(fileId=selected_file_id) + fh = io.BytesIO() + downloader = MediaIoBaseDownload(fh, request) + done = False + while done is False: + status, done = downloader.next_chunk() + print("Download %d%%." % int(status.progress() * 100)) + stri = "./" + selected_file_name + with io.open(stri, 'wb') as f: + fh.seek(0) + f.write(fh.read()) + + return selected_file_name class BitTorrentDataSourceDirLoader(DataSourceDirLoader): + def load(self, *data_source): + with connect('.pow/pow.conf') as conn: + ctx = Context(ident='http://openworm.org/data', conf=conn.conf).stored + for d in data_source: + datasource = ctx(DataSource)(ident=URIRef(d)).load() + for g in datasource: + x = list(g.torrent_file_name()) + downloaded_torrent_name = download_torrent(x[0]) + print('downloaded torrent', downloaded_torrent_name) + + + os.system("torrent_cli.py start &") + os.system("torrent_cli.py add "+ downloaded_torrent_name) + + - def load(self, data_source): - pass diff --git a/setup.py b/setup.py index 7197c351d..70ab7c347 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,9 @@ def excludes(base): 'ZConfig==3.0.4', 'zdaemon==4.0.0', 'zodb==4.1.0', + 'torrent-client', + 'google-api-python-client', + 'oauth2client==3.0.0', ] + (['zodbpickle==1.0'] if PY2 else []) + (['Sphinx<1.8.4'] if PY2 else []) + (['backports.tempfile==1.0'] if PY2 else []) diff --git a/tests/BitTorrentTest.py b/tests/BitTorrentTest.py new file mode 100644 index 000000000..03ae6b6ab --- /dev/null +++ b/tests/BitTorrentTest.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import print_function +import unittest +import itertools +from .TestUtilities import xfail_without_db +import PyOpenWorm +from .DataTestTemplate import _DataTest +from PyOpenWorm.bittorrent import BitTorrentDataSourceDirLoader +import os +import six +import sys +import tempfile +from textwrap import dedent +import traceback +import transaction +from pytest import mark, fixture +import unittest + +from PyOpenWorm.data_trans.local_file_ds import LocalFileDataSource as LFDS +from PyOpenWorm import connect +from PyOpenWorm.datasource import DataTranslator +from PyOpenWorm.context import Context + +class TestBitTorrentDataSourceDirLoader(unittest.TestCase): + def test_torrent_download1(self): + self.assertFalse(os.path.exists("d9da5ce947c6f1c127dfcdc2ede63320.torrent"), False) + self.assertFalse(os.path.exists("Merged_Nuclei_Stained_Worm.zip"), False) + content = BitTorrentDataSourceDirLoader("./") + ident = 'http://openworm.org/entities/ConnectomeCSVDataSource/Mark_Arnab_3000_connections' + content_path = content.load(ident) + self.assertTrue(os.path.exists("d9da5ce947c6f1c127dfcdc2ede63320.torrent"), True) + self.assertTrue(os.path.exists("Merged_Nuclei_Stained_Worm.zip"), True) + # Merged_Nuclei_Stained_Worm.zip will appear but its contents take a while to download + # watch the progress with - 'watch python3 torrent_cli.py' \ No newline at end of file