mirror of
https://github.com/ClaytonWWilson/Scraper-for-theTVDB.com.git
synced 2025-12-15 17:28:46 +00:00
Changed variable names to lowercase hyphens
This commit is contained in:
parent
b2efb9cf0b
commit
9b90d83780
106
actions.py
106
actions.py
@ -28,8 +28,8 @@ def refreshToken():
|
|||||||
try:
|
try:
|
||||||
with open("login.json") as json_data:
|
with open("login.json") as json_data:
|
||||||
login = json.load(json_data)
|
login = json.load(json_data)
|
||||||
saveTime = dateutil.parser.parse(login["TIMESTAMP"])
|
save_time = dateutil.parser.parse(login["TIMESTAMP"])
|
||||||
curTime = datetime.datetime.now().replace(tzinfo=None) # TODO use UTC time?
|
cur_time = datetime.datetime.now().replace(tzinfo=None) # TODO use UTC time?
|
||||||
json_data.close()
|
json_data.close()
|
||||||
|
|
||||||
LOGIN_DATA = {
|
LOGIN_DATA = {
|
||||||
@ -38,7 +38,7 @@ def refreshToken():
|
|||||||
"username": login["USER_NAME"]
|
"username": login["USER_NAME"]
|
||||||
}
|
}
|
||||||
|
|
||||||
if checkTimestamp(saveTime, curTime):
|
if checkTimestamp(save_time, cur_time):
|
||||||
while True:
|
while True:
|
||||||
print("Your current token is still valid. Are you sure you want to grab a different one?")
|
print("Your current token is still valid. Are you sure you want to grab a different one?")
|
||||||
choice = input("(y/n) ")
|
choice = input("(y/n) ")
|
||||||
@ -70,13 +70,13 @@ def clearFolders(): # TODO implement this
|
|||||||
del_count = 0
|
del_count = 0
|
||||||
for folder in folders:
|
for folder in folders:
|
||||||
if os.path.exists(folder):
|
if os.path.exists(folder):
|
||||||
imageList = os.listdir(folder)
|
image_list = os.listdir(folder)
|
||||||
if len(imageList) != 0:
|
if len(image_list) != 0:
|
||||||
print("Clearing " + folder + "/")
|
print("Clearing " + folder + "/")
|
||||||
for x in imageList: # TODO check if folder is empty
|
for x in image_list: # TODO check if folder is empty
|
||||||
print("Deleting {}/{}".format(folder, x))
|
print("Deleting {}/{}".format(folder, x))
|
||||||
delPath = os.path.join(folder + "\\" + x)
|
del_path = os.path.join(folder + "\\" + x)
|
||||||
os.remove(delPath)
|
os.remove(del_path)
|
||||||
del_count += 1
|
del_count += 1
|
||||||
print()
|
print()
|
||||||
else:
|
else:
|
||||||
@ -89,64 +89,64 @@ def createFolder(folder): # TODO remove this
|
|||||||
os.makedirs(folder)
|
os.makedirs(folder)
|
||||||
|
|
||||||
|
|
||||||
def searchImages(idNum, keyType, authHeaders): # This is getting a list of file info for images in json format
|
def searchImages(id_num, keyType, authHeaders): # This is getting a list of file info for images in json format
|
||||||
queryUrl = "https://api.thetvdb.com/series/{}/images/query{}".format(str(idNum), keyType)
|
query_url = "https://api.thetvdb.com/series/{}/images/query{}".format(str(id_num), keyType)
|
||||||
response = requests.get(queryUrl, headers=authHeaders)
|
response = requests.get(query_url, headers=authHeaders)
|
||||||
if (checkStatus(response, True)):
|
if (checkStatus(response, True)):
|
||||||
return response
|
return response
|
||||||
else:
|
else:
|
||||||
quit()
|
quit()
|
||||||
|
|
||||||
def downloadImages(imageType, respObj, idNum): # TODO some images arent grabbed through the api. save the image number and make a try catch to get any missing images
|
def downloadImages(image_type, respObj, id_num): # TODO some images arent grabbed through the api. save the image number and make a try catch to get any missing images
|
||||||
parsed_respObj = json.loads(respObj.content)
|
parse_resp_obj = json.loads(respObj.content)
|
||||||
|
|
||||||
saveNameList = download(imageType, parsed_respObj)
|
save_name_list = download(image_type, parse_resp_obj)
|
||||||
|
|
||||||
searchRemainder(imageType, saveNameList, idNum)
|
searchRemainder(image_type, save_name_list, id_num)
|
||||||
|
|
||||||
def searchRemainder(imageType, saveNameList, idNum):#Finds any images missing from the api call in getImages
|
def searchRemainder(image_type, save_name_list, id_num):#Finds any images missing from the api call in getImages
|
||||||
numbers = []
|
numbers = []
|
||||||
print("Checking for missing images...") # TODO implement this method
|
print("Checking for missing images...") # TODO implement this method
|
||||||
if (imageType is "banner"): # TODO check upper and lower bounds
|
if (image_type is "banner"): # TODO check upper and lower bounds
|
||||||
print("this is a banner")
|
print("this is a banner")
|
||||||
#TODO deal with banners
|
#TODO deal with banners
|
||||||
else:
|
else:
|
||||||
for name in saveNameList:
|
for name in save_name_list:
|
||||||
if (name.rfind("-") != -1):
|
if (name.rfind("-") != -1):
|
||||||
hyphenIndex = name.rfind("-")
|
hyphen_index = name.rfind("-")
|
||||||
hyphenSuffix = name[hyphenIndex + 1:]
|
hyphen_suffix = name[hyphen_index + 1:]
|
||||||
filenum = hyphenSuffix.replace(".jpg", "")
|
file_num = hyphen_suffix.replace(".jpg", "")
|
||||||
numbers.append(int(filenum))
|
numbers.append(int(file_num))
|
||||||
else:
|
else:
|
||||||
print("I couldn't find a hyphen in: {}".format(name)) # Error checking
|
print("I couldn't find a hyphen in: {}".format(name)) # Error checking
|
||||||
numbers.sort
|
numbers.sort
|
||||||
missingList = findMissing(numbers)
|
missing_list = findMissing(numbers)
|
||||||
minNum = min(numbers)
|
min_num = min(numbers)
|
||||||
maxNum = max(numbers)
|
max_num = max(numbers)
|
||||||
|
|
||||||
tryMissing(missingList, minNum, maxNum, idNum, imageType)
|
tryMissing(missing_list, min_num, max_num, id_num, image_type)
|
||||||
|
|
||||||
def findMissing(numbers):
|
def findMissing(numbers):
|
||||||
start, end = numbers[0], numbers[-1]
|
start, end = numbers[0], numbers[-1]
|
||||||
return sorted(set(range(start, end + 1)).difference(numbers))
|
return sorted(set(range(start, end + 1)).difference(numbers))
|
||||||
|
|
||||||
def tryMissing(missingNums, minNum, maxNum, idNum, imageType):
|
def tryMissing(missing_nums, min_num, max_num, id_num, image_type):
|
||||||
if (imageType is "fanart"):
|
if (image_type is "fanart"):
|
||||||
startDirectory = "fanart/original/"
|
start_directory = "fanart/original/"
|
||||||
elif (imageType is "poster"):
|
elif (image_type is "poster"):
|
||||||
startDirectory = "posters/"
|
start_directory = "posters/"
|
||||||
|
|
||||||
for num in missingNums:
|
for num in missing_nums:
|
||||||
fileName = "{}{}-{}.jpg".format(startDirectory, str(idNum), str(num))
|
filename = "{}{}-{}.jpg".format(start_directory, str(id_num), str(num))
|
||||||
# fileName = "%s%s-%d.jpg" % startDirectory, idNum, missingNums[num]
|
# filename = "%s%s-%d.jpg" % start_directory, id_num, missing_nums[num]
|
||||||
# try:
|
# try:
|
||||||
print("Trying... {}".format(fileName))
|
print("Trying... {}".format(filename))
|
||||||
dlUrl = "https://www.thetvdb.com/banners/{}".format(fileName)
|
dl_url = "https://www.thetvdb.com/banners/{}".format(filename)
|
||||||
# print("url is: " + dlUrl)
|
# print("url is: " + dl_url)
|
||||||
response = requests.get(dlUrl)
|
response = requests.get(dl_url)
|
||||||
# print(response.status_code)
|
# print(response.status_code)
|
||||||
if (checkStatus(response, True) == True): # TODO there is an error occurring here when checking fanart
|
if (checkStatus(response, True) == True): # TODO there is an error occurring here when checking fanart
|
||||||
path = os.path.join("{}\\{}-{}.jpg".format(imageType, str(idNum), str(num)))
|
path = os.path.join("{}\\{}-{}.jpg".format(image_type, str(id_num), str(num)))
|
||||||
obj = open(path, "wb")
|
obj = open(path, "wb")
|
||||||
obj.write(response.content)
|
obj.write(response.content)
|
||||||
obj.close()
|
obj.close()
|
||||||
@ -157,35 +157,35 @@ def tryMissing(missingNums, minNum, maxNum, idNum, imageType):
|
|||||||
|
|
||||||
# except Exception as e:
|
# except Exception as e:
|
||||||
# print("response code: " + str(response.status_code))
|
# print("response code: " + str(response.status_code))
|
||||||
# print("Check: " + dlUrl)
|
# print("Check: " + dl_url)
|
||||||
# print(e)
|
# print(e)
|
||||||
|
|
||||||
# while minNum > 1: # Checking lower bounds
|
# while min_num > 1: # Checking lower bounds
|
||||||
# print("check lower")
|
# print("check lower")
|
||||||
|
|
||||||
def download(imageType, parsed_respObj):
|
def download(image_type, parse_resp_obj):
|
||||||
counter = 0
|
counter = 0
|
||||||
saveNameList = []
|
save_name_list = []
|
||||||
for imageObj in parsed_respObj["data"]:
|
for image_obj in parse_resp_obj["data"]:
|
||||||
fileName = parsed_respObj["data"][counter]["fileName"] # TODO the download method should start here, move everything else up to downloadImages
|
filename = parse_resp_obj["data"][counter]["filename"] # TODO the download method should start here, move everything else up to downloadImages
|
||||||
counter = counter + 1
|
counter = counter + 1
|
||||||
|
|
||||||
slashIndex = fileName.rfind("/") # This is used to slice the url at the beginning of the filename
|
slash_index = filename.rfind("/") # This is used to slice the url at the beginning of the filename
|
||||||
saveName = fileName[slashIndex + 1:] # For example 'https://thetvdb.com/banners/fanart/original/32451-3.jpg' --> '32451.jpg'
|
save_name = filename[slash_index + 1:] # For example 'https://thetvdb.com/banners/fanart/original/32451-3.jpg' --> '32451.jpg'
|
||||||
saveNameList.append(saveName)
|
save_name_list.append(save_name)
|
||||||
|
|
||||||
print("Downloading... {}".format(fileName))
|
print("Downloading... {}".format(filename))
|
||||||
dlUrl = "https://www.thetvdb.com/banners/{}".format(fileName)
|
dl_url = "https://www.thetvdb.com/banners/{}".format(filename)
|
||||||
response = requests.get(dlUrl) # TODO getting errors when checking 'new game'. Check to see if those images actually exist
|
response = requests.get(dl_url) # TODO getting errors when checking 'new game'. Check to see if those images actually exist
|
||||||
|
|
||||||
if (checkStatus(response, True)):
|
if (checkStatus(response, True)):
|
||||||
path = os.path.join(imageType + "\\", saveName)
|
path = os.path.join(image_type + "\\", save_name)
|
||||||
obj = open(path, "wb")
|
obj = open(path, "wb")
|
||||||
obj.write(response.content)
|
obj.write(response.content)
|
||||||
obj.close()
|
obj.close()
|
||||||
else:
|
else:
|
||||||
quit()
|
quit()
|
||||||
return saveNameList
|
return save_name_list
|
||||||
|
|
||||||
def installReqs():
|
def installReqs():
|
||||||
if is_pip_installed() == True:
|
if is_pip_installed() == True:
|
||||||
|
|||||||
@ -33,8 +33,8 @@ def checkStatus(response, v):
|
|||||||
else:
|
else:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def checkTimestamp(saveTime, curTime): # Returns true if the token is still valid
|
def checkTimestamp(save_time, cur_time): # Returns true if the token is still valid
|
||||||
if curTime - saveTime < datetime.timedelta(0, 86100, 0):
|
if cur_time - save_time < datetime.timedelta(0, 86100, 0):
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
|
|||||||
@ -26,7 +26,7 @@ while True:
|
|||||||
|
|
||||||
choice = input("> ").lower().strip()
|
choice = input("> ").lower().strip()
|
||||||
|
|
||||||
if choice == "1":
|
if choice == "1": # TODO catch KeyboardInterrupt at search
|
||||||
search()
|
search()
|
||||||
wait()
|
wait()
|
||||||
elif choice == "2":
|
elif choice == "2":
|
||||||
|
|||||||
48
search.py
48
search.py
@ -24,9 +24,9 @@ def search():
|
|||||||
print("There was an error checking your login. Try logging in again with 'Login/Change login'.")
|
print("There was an error checking your login. Try logging in again with 'Login/Change login'.")
|
||||||
return None
|
return None
|
||||||
else:
|
else:
|
||||||
saveTime = dateutil.parser.parse(login["TIMESTAMP"])
|
save_time = dateutil.parser.parse(login["TIMESTAMP"])
|
||||||
curTime = datetime.datetime.now().replace(tzinfo=None) # TODO use UTC time?
|
cur_time = datetime.datetime.now().replace(tzinfo=None) # TODO use UTC time?
|
||||||
if checkTimestamp(saveTime, curTime) == False:
|
if checkTimestamp(save_time, cur_time) == False:
|
||||||
refreshToken()
|
refreshToken()
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
print(ex)
|
print(ex)
|
||||||
@ -45,31 +45,31 @@ def search():
|
|||||||
}
|
}
|
||||||
|
|
||||||
keyword = input("Enter series to search: ") # Getting the search name and fixing
|
keyword = input("Enter series to search: ") # Getting the search name and fixing
|
||||||
sKeyword = urllib.parse.quote(keyword) # the url parse mistakes
|
s_keyword = urllib.parse.quote(keyword) # the url parse mistakes
|
||||||
|
|
||||||
sKeyword = sKeyword.replace("%21", "!") # TODO find a better way of doing this
|
s_keyword = s_keyword.replace("%21", "!") # TODO find a better way of doing this
|
||||||
sKeyword = sKeyword.replace("%2A", "*")
|
s_keyword = s_keyword.replace("%2A", "*")
|
||||||
sKeyword = sKeyword.replace("%28", "(")
|
s_keyword = s_keyword.replace("%28", "(")
|
||||||
sKeyword = sKeyword.replace("%29", ")")
|
s_keyword = s_keyword.replace("%29", ")")
|
||||||
sKeyword = sKeyword.replace("%27", "'")
|
s_keyword = s_keyword.replace("%27", "'")
|
||||||
sKeyword = sKeyword.replace("/", "%2F")
|
s_keyword = s_keyword.replace("/", "%2F")
|
||||||
sKeyword = sKeyword.replace("%7E", "~")
|
s_keyword = s_keyword.replace("%7E", "~")
|
||||||
|
|
||||||
searchUrl = "https://api.thetvdb.com/search/series?name={}".format(sKeyword)
|
search_url = "https://api.thetvdb.com/search/series?name={}".format(s_keyword)
|
||||||
response = requests.get(searchUrl, headers=authHeaders)
|
response = requests.get(search_url, headers=authHeaders)
|
||||||
|
|
||||||
if (checkStatus(response, True) == False):
|
if (checkStatus(response, True) == False):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
searchResults = json.loads(response.content)
|
search_results = json.loads(response.content)
|
||||||
|
|
||||||
title = -1
|
title = -1
|
||||||
print()
|
print()
|
||||||
clearScreen()
|
clearScreen()
|
||||||
while title < 0 or title > len(searchResults["data"]) - 1: # Looping until the user chooses
|
while title < 0 or title > len(search_results["data"]) - 1: # Looping until the user chooses
|
||||||
print("Results:") # a series from the printed list
|
print("Results:") # a series from the printed list
|
||||||
count = 1 # or they input '0' to cancel
|
count = 1 # or they input '0' to cancel
|
||||||
for result in searchResults["data"]:
|
for result in search_results["data"]:
|
||||||
print("\n{})\nSeries Name: {}".format(str(count), str(result["seriesName"])))
|
print("\n{})\nSeries Name: {}".format(str(count), str(result["seriesName"])))
|
||||||
print()
|
print()
|
||||||
desc = result["overview"]
|
desc = result["overview"]
|
||||||
@ -80,7 +80,7 @@ def search():
|
|||||||
print()
|
print()
|
||||||
title = int(input("Choose one by number or '0' to exit: ")) - 1 # Subtracting 1 so that the
|
title = int(input("Choose one by number or '0' to exit: ")) - 1 # Subtracting 1 so that the
|
||||||
print() # index can start from 0
|
print() # index can start from 0
|
||||||
if title < -1 or title > len(searchResults["data"]) - 1:
|
if title < -1 or title > len(search_results["data"]) - 1:
|
||||||
print("Error: Choose the number of an item listed. Or '0' to exit.")
|
print("Error: Choose the number of an item listed. Or '0' to exit.")
|
||||||
|
|
||||||
if (title == -1): # If the user inputs 0
|
if (title == -1): # If the user inputs 0
|
||||||
@ -88,14 +88,14 @@ def search():
|
|||||||
|
|
||||||
print()
|
print()
|
||||||
|
|
||||||
idNum = searchResults["data"][title]["id"] # Setting up the request urls
|
id_num = search_results["data"][title]["id"] # Setting up the request urls
|
||||||
fanart = searchImages(idNum, FAN_KEY_TYPE, authHeaders) # for banners, fanart, and posters
|
fanart = searchImages(id_num, FAN_KEY_TYPE, authHeaders) # for banners, fanart, and posters
|
||||||
poster = searchImages(idNum, POS_KEY_TYPE, authHeaders)
|
poster = searchImages(id_num, POS_KEY_TYPE, authHeaders)
|
||||||
banner = searchImages(idNum, BAN_KEY_TYPE, authHeaders)
|
banner = searchImages(id_num, BAN_KEY_TYPE, authHeaders)
|
||||||
|
|
||||||
clearFolders()
|
clearFolders()
|
||||||
downloadImages("fanart", fanart, idNum) # TODO find a better way to pass these variables. Constructor?
|
downloadImages("fanart", fanart, id_num) # TODO find a better way to pass these variables. Constructor?
|
||||||
downloadImages("poster", poster, idNum)
|
downloadImages("poster", poster, id_num)
|
||||||
downloadImages("banner", banner, idNum)
|
downloadImages("banner", banner, id_num)
|
||||||
print("\nAll downloads finished!\n")
|
print("\nAll downloads finished!\n")
|
||||||
return None
|
return None
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user