v1.0.0

First Stable Release

CHANGELOG.md

@@ -0,0 +1,35 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+------------
+## [Upcoming Release]
+
+-------
+
+## [1.0.0] - 2016-11-10
+First Stable release.
+### Added
+- Can now get images from `image` lists.
+- Support for command-line Arguments.
+- Added a progress bar.
+- Download resume capability.
+
+### Changed
+- Instead of entering the name of the person you enter the profile url.
+- Downloader is now a seperate script rather then an option.
+- Links are placed in a file in working dir; Instead of creating a seperate directory inside the working directory.
+- No interactive input; instead command line args are accepted.
+
+### Removed
+- Option to download images is removed from main file.
+
+------
+## [0.1.0] - 2016-04-20
+This is the first public release. **Pre Release**
+
+### Added
+- Retrives the links of of all photos from any `person` profile on `listal.com`.
+- Also, can download the photos from the retrived links using mulitple threads.

L-dl.py

@@ -0,0 +1,138 @@
+#
+# Listal Downloader
+#
+#
+
+import urllib.request,ssl
+import argparse
+import time
+import queue
+import threading
+import os,sys
+import better_exceptions
+#
+
+# Very Important. 
+ssl._create_default_https_context = ssl._create_unverified_context
+
+def download():
+    global broken, failed, ls
+    while not qq.empty():
+        mydata = threading.local()
+        mydata.name, mydata.url = qq.get()
+        mydata.keep_going, mydata.skip, mydata.retry = (True, False, 0)
+        #if mydata.name in ls:continue
+        while mydata.keep_going:
+            try:mydata.html = urllib.request.urlopen(mydata.url,timeout=120)
+            except urllib.error.HTTPError as HERR:
+                if HERR.code == 404 or HERR.code == 500:
+                    broken += 1
+                    mydata.keep_going = False
+                    mydata.skip = True
+            except:
+                mydata.retry += 1
+                if mydata.retry > 5:
+                    mydata.keep_going = False
+                    mydata.skip = True
+            break
+        if mydata.skip:continue
+        while True:
+            try:
+                mydata.image = mydata.html.read()
+                open(mydata.name,'wb').write(mydata.image)
+                break
+            except:
+                mydata.retry += 1
+                if mydata.retry > 10:break
+
+def mkqueue():
+    global total,ld,ls
+    fhand = open(os.path.join(ld,args.fname),'r')
+    links = []
+    for each in fhand:
+        if each.startswith('#') or len(each) < 10:continue
+        fname = each.strip().split('/')[-2].zfill(10) + "." + each.strip().split('.')[-1]
+        if fname not in ls:links.append((fname,each.strip()))
+    for each in sorted(set(links),reverse=True):qq.put(each)
+    total = qq.qsize()
+    print(str(total),"Files queued for download.")
+    fhand.close()
+
+def enqueue():
+    if qq.qsize() != 0:print("\n WARNING: Queue was not empty. ")
+    for name,url in failed:
+        qq.put((name,url))
+
+def init_threads():
+    for i in range(args.threads if args.threads < qq.qsize() else qq.qsize()):
+        t = threading.Thread(target=download)
+        threads.append(t)
+        t.start()
+
+def update_progress():
+    progress = 100 - int((100*qq.qsize()) / total)
+    te = time.strftime("%H:%M:%S",time.gmtime(time.time()-started))
+    pbar = "\r {:0>3}% [{:<50}] ({},{}) Time Elapsed : {} ".format(progress, '#'*int((progress/2)), (total-qq.qsize()), total, te)
+    sys.stdout.write(pbar)
+    sys.stdout.flush()
+
+def check_progress():
+    global t1,p1
+    t2 = time.time()
+    p2 = 100 - int((100*qq.qsize()) / total)
+    if t2-t1 < 25:pass
+    else:
+        if p2 - p1 >0:
+            t1 = t2
+            p1 = p2
+        else:quit()
+#
+
+parser = argparse.ArgumentParser(description='Start Tao Downloader.')
+parser.add_argument('fname', type=str,
+                    help='The File containing list of links.')
+parser.add_argument('--dir', dest='directory', type = str, default = None, required = False,
+                    help='The directory to download files in.')
+parser.add_argument('--threads', dest='threads', type = int, default = 10, required = False,
+                    help='No. of threads to use.')
+args = parser.parse_args()
+
+#
+qq = queue.Queue()
+started = time.time()
+threads = []
+links   = []
+failed  = []
+broken  = 00
+internal_error = 00
+ld = os.getcwd()
+
+
+
+if args.directory is not None:
+    if not os.path.exists(args.directory):os.makedirs(args.directory)
+    os.chdir(args.directory)
+ls = os.listdir(os.getcwd())
+#files = os.listdir(os.getcwd())
+
+mkqueue()
+init_threads()
+t1, p1 = 0,0
+while not qq.empty():
+    update_progress()
+    time.sleep(5)
+    #check_progress()
+for t in threads:t.join()
+
+if len(failed) > 0:
+    print("\n INFO : Download failed for {} Items. Trying Again ...".format(len(failed)))
+    enqueue()
+    failed.clear()
+    init_threads()
+    print("\n INFO:",len(failed),"Downloads Failed.")
+
+print(" \n ============================ \n    Time Taken : {} \n Files Downloaded : {} \n    Failed Downloads\
+    : {} \n    Broken Links : {}  \n ===============================================================".format(\
+    time.strftime("%H:%M:%S",time.gmtime(time.time()-started)), total-len(failed),len(failed),broken))
+
+#

Listal.py

@@ -1,6 +1,6 @@
 #   Listal.py
-#   08/11/2016
-#
+#   08/11/2016 - 2017-04-13
+#  
 
 import urllib.request, urllib.parse
 import bs4