CanuteTheGreat/sgspider
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added fix for pop-up

Browse source
This commit is contained in:
Ronald Farrer 2021-12-29 18:36:21 -08:00
parent 43470c0e72
commit 063094941c
1 changed files with 19 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
sgspider.py Normal file → Executable file
View file

@ -5,6 +5,7 @@ import time
import re import re
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.firefox.options import Options from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.keys import Keys
import os import os
import configparser import configparser
@ -28,6 +29,8 @@ def login(credentials):
print("Loading front page and initiating login") print("Loading front page and initiating login")
driver.get("https://suicidegirls.com") driver.get("https://suicidegirls.com")
time.sleep(1) time.sleep(1)
webdriver.ActionChains(driver).send_keys(Keys.ESCAPE).perform()
time.sleep(1)
driver.find_element_by_id("login").click() driver.find_element_by_id("login").click()
time.sleep(1) time.sleep(1)
user = driver.find_element_by_name("username") user = driver.find_element_by_name("username")
@ -54,9 +57,9 @@ def getgirls():
print("Progress [", end='', flush=True) print("Progress [", end='', flush=True)
done = False done = False
cctr = 0 cctr = 0
albumctr = 0 pagectr = 0
while done == False: while done == False:
albumctr = albumctr + 1 pagectr = pagectr + 1
try: try:
driver.find_element_by_xpath("//a[@id = 'load-more']").click() driver.find_element_by_xpath("//a[@id = 'load-more']").click()
print('.', end='', flush=True) print('.', end='', flush=True)
@ -64,11 +67,11 @@ def getgirls():
except: except:
print('x', end='', flush=True) print('x', end='', flush=True)
cctr = cctr + 1 cctr = cctr + 1
time.sleep(10) time.sleep(1)
if cctr >= 10: if cctr >= 10:
done = True done = True
print("]\n") print("]\n")
print("Total albums found: " + str(albumctr)) print("Total pages loaded: " + str(pagectr))
print("Collecting the URLs for each album. This will take a LONG time!") print("Collecting the URLs for each album. This will take a LONG time!")
@ -120,8 +123,18 @@ def dlimgs(girl, album, url):
return return
print("File: " + str(filename) + " not downloaded, downloading now!") print("File: " + str(filename) + " not downloaded, downloading now!")
response = requests.get(url, stream=True) response = requests.get(url, stream=True)
with open(filename, 'wb') as out_file: timeout = 10
shutil.copyfileobj(response.raw, out_file) while True:
try:
with open(filename, 'wb') as out_file:
shutil.copyfileobj(response.raw, out_file)
break
except:
print("Encountered error writing file '" + str(filename) + "', sleeping " + str(timeout) + " seconds...")
time.sleep(timeout)
print("retrying...")
timeout = timeout + 10
pass
del response del response
def cleanup(): def cleanup():