CanuteTheGreat/sgspider
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added in some reminders to the user about waiting times.

Browse source
This commit is contained in:
Ronald Farrer 2021-04-20 10:48:32 -07:00
parent 5efbc1bc3b
commit 9b446c299c
1 changed files with 7 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
sgspider.py
View file

@ -43,7 +43,7 @@ def getgirls():
driver.get("https://www.suicidegirls.com/photos/sg/recent/all/") driver.get("https://www.suicidegirls.com/photos/sg/recent/all/")
time.sleep(1) time.sleep(1)
print("Starting to scroll through photos page.. this will take a *REALLY* LONG time!") print("Starting to scroll through photos page.. this will take a *REALLY* LONG time!")
print("Each '.' in the progress output represents a new page that has been loaded!") print("Each '.' in the progress output represents a new page that has been loaded and 'x' is a failure to load the next page.")
print("Please be cautious of memory usage!\n\n") print("Please be cautious of memory usage!\n\n")
print("Progress [", end='', flush=True) print("Progress [", end='', flush=True)
done = False done = False
@ -56,13 +56,16 @@ def getgirls():
print('.', end='', flush=True) print('.', end='', flush=True)
cctr = 0 cctr = 0
except: except:
print("reached end or next page failed!") print('x', end='', flush=True)
cctr = cctr + 1 cctr = cctr + 1
time.sleep(10)
if cctr >= 10: if cctr >= 10:
done = True done = True
print("]\n") print("]\n")
print("Total albums found: " + str(albumctr)) print("Total albums found: " + str(albumctr))
print("Collecting the URLs for each album. This will take a LONG time!")
urls = [] urls = []
elems = driver.find_elements_by_xpath("//a[@href]") elems = driver.find_elements_by_xpath("//a[@href]")
for elem in elems: for elem in elems:
@ -76,6 +79,8 @@ def getgirls():
return girls return girls
def getimgs(girls): def getimgs(girls):
print("collecting the URLs for the images. This will take a LONG time.")
for girl in girls: for girl in girls:
driver.get(girl) driver.get(girl)
urls = [] urls = []