From 313430c5766829b1ce2991ab5163f80cd7ec742a Mon Sep 17 00:00:00 2001 From: 0AwsD0 <64651813+0AwsD0@users.noreply.github.com> Date: Fri, 10 May 2024 01:14:54 +0200 Subject: [PATCH] fixes and comments for people that need change of wait time on card download --- download.py | 182 +++++++++++++++++++++++++++------------------------- todo.txt | 3 + 2 files changed, 96 insertions(+), 89 deletions(-) diff --git a/download.py b/download.py index e74ab8b..94e35bc 100644 --- a/download.py +++ b/download.py @@ -36,103 +36,102 @@ def basic_download(url, name, tags): #I need to let user config wait time and provide some values or idk explain it in documentation and readme.md #OR #I may try using watchdog or directory scan to check if file exists -> the card and waiting until it does so there is no need for download interval - it downloads only if previous card is downloaded - print("WARNING: Some cards weigh a lot, even over 25MB - if your intewrnet is slow CHANGE WAIT TIME between card downloads to even 10 seconds!") + print("WARNING: Some cards weigh a lot, even over 25MB - if your internet is slow - CHANGE WAIT TIME - between card downloads to even 20 seconds!") + print("The corresponding SLEEP functions in code have comments above, surrounded by #.") - try: - url = url - name = name - tags = tags - print("url: ", url, " name: ", name," tag: ", tags) - #selenium code below - options = webdriver.FirefoxOptions() - #options.add_argument("-headless") - #driver = webdriver.Firefox(options=options) - driver = webdriver.Firefox() - driver.get(url) + url = url + name = name + tags = tags + print("url: ", url, " name: ", name," tag: ", tags) + #selenium code below + #options = webdriver.FirefoxOptions() + #options.add_argument("-headless") + #driver = webdriver.Firefox(options=options) + driver = webdriver.Firefox() + driver.get(url) - # I know I can just make up the link to the website since it's uses GET method like "https://db.bepis.moe/koikatsu?name=aaaa&tag=bbbb" but I want to try use the way below - name_input = driver.find_element(By.ID, "name") - name_input.send_keys(name) - tag_input = driver.find_element(By.ID, "tag") - tag_input.send_keys(tags) - tag_input.submit() + # I know I can just make up the link to the website since it's uses GET method like "https://db.bepis.moe/koikatsu?name=aaaa&tag=bbbb" but I want to try use the way below + name_input = driver.find_element(By.ID, "name") + name_input.send_keys(name) + tag_input = driver.find_element(By.ID, "tag") + tag_input.send_keys(tags) + tag_input.submit() - print("Waiting 5 seconds for page to load...") - time.sleep(5) + print("Waiting 5 seconds for page to load...") + time.sleep(5) - #just set the loop to be true until the "Next" button is "disabled" than flip the flag to exit loop // if Next button DISABLED flag = 0 - i = 2 - i2 = 1 - flag = 1 + #just set the loop to be true until the "Next" button is "disabled" than flip the flag to exit loop // if Next button DISABLED flag = 0 + i = 2 + i2 = 1 + flag = 1 - #base url - the url2 is for adding '&page='+i and than reseting it back to normal before adding next &page - url = driver.current_url - except Exception: - print("Exception in basic_download() before loop: " + Exception) - print("Exiting in 5 seconds.") - time.sleep(5) - driver.quit() - exit(1) + #base url - the url2 is for adding '&page='+i and than reseting it back to normal before adding next &page + url = driver.current_url - while (flag == 1): + while (flag == 1): + try: try: - try: - while (i2 <= 24): - print(f"Downloading card number: {i2}") - download_selector = "document.querySelector('#inner-card-body > div:nth-child(" + str(i2) + ") > div > div > a.btn.btn-primary.btn-sm').click();" - time.sleep(4) - driver.execute_script(download_selector) - i2 += 1 - except Exception: - print("The download FINISHED!") - print("There were less than 24 cards on last page.") - print(">>OR There is slight chance, that website got down.") - # flag == 0 + go back to main() // or leave like that to exit program - print("Exiting in 5 seconds.") - time.sleep(5) - driver.quit() - exit(0) - next_button_script = """ - xpath = "//a[contains(text(),'Next')]"; - var matchingElement = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; - parentDiv = matchingElement.parentNode; - if(parentDiv.classList.contains('disabled') == true){ - var state = "Disabled"; - return state; - } - else{ - var state = "Enabled"; - return state; - } - """ - print("Getting 'Next' button state...") - button_state = driver.execute_script(next_button_script) - #sleep for script and mainly for last card/cards to download - time.sleep(5) - print("Button state = "+button_state) - if(button_state == "Disabled"): - print("Download FINISHED!") - # flag == 0 + go back to main() // or leave like that to exit program - print("-------------------") - print("Exiting in 5 seconds.") - driver.quit() + while (i2 <= 24): + print(f"Downloading card number: {i2}") + download_selector = "document.querySelector('#inner-card-body > div:nth-child(" + str(i2) + ") > div > div > a.btn.btn-primary.btn-sm').click();" + driver.execute_script(download_selector) + ###################################### + #BELOW TIME FOR CARD TO BE DOWNLOADED# + ###################################### time.sleep(5) - exit(0) - print("Getting url...") - url2 = url - print("Got url - adding page number...") - url2 += "&page=" + str(i) - print("Waiting 5 seconds for page to load...") - driver.get(url2) - time.sleep(5) - i2 = 1 - i += 1 + i2 += 1 except Exception: - print("The download failed. //loop "+Exception) + print("The download FINISHED!") + print("There were less than 24 cards on last page.") + print(">>OR There is slight chance, that website got down.") + # flag == 0 + go back to main() // or leave like that to exit program print("Exiting in 5 seconds.") time.sleep(5) driver.quit() - exit(1) + exit(0) + next_button_script = """ + xpath = "//a[contains(text(),'Next')]"; + var matchingElement = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + parentDiv = matchingElement.parentNode; + if(parentDiv.classList.contains('disabled') == true){ + var state = "Disabled"; + return state; + } + else{ + var state = "Enabled"; + return state; + } + """ + print("Getting 'Next' button state...") + button_state = driver.execute_script(next_button_script) + ######################################################################## + #sleep below for script and mainly for last card/cards to be downloaded# + ######################################################################## + time.sleep(10) + print("Button state = "+button_state) + if(button_state == "Disabled"): + print("Download FINISHED!") + # flag == 0 + go back to main() // or leave like that to exit program + print("-------------------") + print("Exiting in 5 seconds.") + driver.quit() + time.sleep(5) + exit(0) + print("Getting url...") + url2 = url + print("Got url - adding page number...") + url2 += "&page=" + str(i) + print("Waiting 5 seconds for page to load...") + driver.get(url2) + time.sleep(5) + i2 = 1 + i += 1 + except Exception: + print("The download failed. //loop "+Exception) + print("Exiting in 5 seconds.") + time.sleep(5) + driver.quit() + exit(1) #I tried to make below shorter since the download LOOP is same as in basic download, but the driver must be initialized in same method, so I can't do it without spawning additional unnecessary browsers #But in other end it's easier to deal with code that is simpler, even if there is more of it. ~Terry would approve @@ -168,7 +167,7 @@ def advanced_download(**data): #selenium code below - options = webdriver.FirefoxOptions() + #options = webdriver.FirefoxOptions() #options.add_argument("-headless") #driver = webdriver.Firefox(options=options) driver = webdriver.Firefox() @@ -240,8 +239,11 @@ def advanced_download(**data): while (i2 <= 24): print(f"Downloading card number: {i2}") download_selector = "document.querySelector('#inner-card-body > div:nth-child(" + str(i2) + ") > div > div > a.btn.btn-primary.btn-sm').click();" - time.sleep(4) driver.execute_script(download_selector) + ###################################### + #BELOW TIME FOR CARD TO BE DOWNLOADED# + ###################################### + time.sleep(5) i2 += 1 except Exception: print("The download FINISHED!") @@ -267,8 +269,10 @@ def advanced_download(**data): """ print("Getting 'Next' button state...") button_state = driver.execute_script(next_button_script) - #sleep for script and mainly for last card/cards to download - time.sleep(5) + ######################################################################## + #sleep below for script and mainly for last card/cards to be downloaded# + ######################################################################## + time.sleep(10) print("Button state = "+button_state) if(button_state == "Disabled"): print("Download FINISHED!") diff --git a/todo.txt b/todo.txt index 916f823..b94fe49 100644 --- a/todo.txt +++ b/todo.txt @@ -6,3 +6,6 @@ The plan is to even add auto download - you create file with all parameters and The program will do it for you! ~Thats the plan will see when or if I'm gonna implement that + ++ better card download -> checking if card is downloaded, and only then downloading next card ++ packing app into exe \ No newline at end of file