mirror of
https://github.com/ClaytonWWilson/Scraper-for-theTVDB.com.git
synced 2025-12-16 17:38:47 +00:00
The start of refactoring code
This commit is contained in:
parent
3bb7b78a96
commit
de2bc5c576
3
.gitignore
vendored
3
.gitignore
vendored
@ -2,3 +2,6 @@
|
|||||||
login.json
|
login.json
|
||||||
api info.txt
|
api info.txt
|
||||||
__pycache__
|
__pycache__
|
||||||
|
banner
|
||||||
|
fanart
|
||||||
|
poster
|
||||||
63
actions.py
63
actions.py
@ -65,6 +65,27 @@ def refreshToken():
|
|||||||
else:
|
else:
|
||||||
print("You need to log in first. Select Login/Change login.\n")
|
print("You need to log in first. Select Login/Change login.\n")
|
||||||
|
|
||||||
|
def download(series):
|
||||||
|
# Create downloads folder
|
||||||
|
if not os.path.exists("downloads"):
|
||||||
|
os.makedirs("downloads")
|
||||||
|
|
||||||
|
# Remove previously downloaded content for this series if it exists
|
||||||
|
if os.path.exists(os.path.join("downloads", series.folder_name)):
|
||||||
|
shutil.rmtree(os.path.join("downloads", series.folder_name))
|
||||||
|
|
||||||
|
# Create series folder
|
||||||
|
os.makedirs(os.path.join("downloads", series.folder_name))
|
||||||
|
|
||||||
|
api_path = "https://api.thetvdb.com/series/" + series.id
|
||||||
|
|
||||||
|
api_con = APIConnector()
|
||||||
|
res = api_con.send_http_req(api_path)
|
||||||
|
|
||||||
|
with open("out.json", "w") as out:
|
||||||
|
out.write(json.dumps(json.loads(res.content)))
|
||||||
|
|
||||||
|
|
||||||
def clearFolders(): # TODO implement this
|
def clearFolders(): # TODO implement this
|
||||||
folders = ["banner", "fanart", "poster"]
|
folders = ["banner", "fanart", "poster"]
|
||||||
del_count = 0
|
del_count = 0
|
||||||
@ -163,29 +184,29 @@ def tryMissing(missing_nums, min_num, max_num, id_num, image_type):
|
|||||||
# while min_num > 1: # Checking lower bounds
|
# while min_num > 1: # Checking lower bounds
|
||||||
# print("check lower")
|
# print("check lower")
|
||||||
|
|
||||||
def download(image_type, parse_resp_obj):
|
# def download(image_type, parse_resp_obj):
|
||||||
counter = 0
|
# counter = 0
|
||||||
save_name_list = []
|
# save_name_list = []
|
||||||
for image_obj in parse_resp_obj["data"]:
|
# for image_obj in parse_resp_obj["data"]:
|
||||||
filename = parse_resp_obj["data"][counter]["filename"] # TODO the download method should start here, move everything else up to downloadImages
|
# filename = parse_resp_obj["data"][counter]["filename"] # TODO the download method should start here, move everything else up to downloadImages
|
||||||
counter = counter + 1
|
# counter = counter + 1
|
||||||
|
|
||||||
slash_index = filename.rfind("/") # This is used to slice the url at the beginning of the filename
|
# slash_index = filename.rfind("/") # This is used to slice the url at the beginning of the filename
|
||||||
save_name = filename[slash_index + 1:] # For example 'https://thetvdb.com/banners/fanart/original/32451-3.jpg' --> '32451.jpg'
|
# save_name = filename[slash_index + 1:] # For example 'https://thetvdb.com/banners/fanart/original/32451-3.jpg' --> '32451.jpg'
|
||||||
save_name_list.append(save_name)
|
# save_name_list.append(save_name)
|
||||||
|
|
||||||
print("Downloading... {}".format(filename))
|
# print("Downloading... {}".format(filename))
|
||||||
dl_url = "https://www.thetvdb.com/banners/{}".format(filename)
|
# dl_url = "https://www.thetvdb.com/banners/{}".format(filename)
|
||||||
response = requests.get(dl_url) # TODO getting errors when checking 'new game'. Check to see if those images actually exist
|
# response = requests.get(dl_url) # TODO getting errors when checking 'new game'. Check to see if those images actually exist
|
||||||
|
|
||||||
if (checkStatus(response, True)):
|
# if (checkStatus(response, True)):
|
||||||
path = os.path.join(image_type + "\\", save_name)
|
# path = os.path.join(image_type + "\\", save_name)
|
||||||
obj = open(path, "wb")
|
# obj = open(path, "wb")
|
||||||
obj.write(response.content)
|
# obj.write(response.content)
|
||||||
obj.close()
|
# obj.close()
|
||||||
else:
|
# else:
|
||||||
quit()
|
# quit()
|
||||||
return save_name_list
|
# return save_name_list
|
||||||
|
|
||||||
def installReqs():
|
def installReqs():
|
||||||
if is_pip_installed() == True:
|
if is_pip_installed() == True:
|
||||||
@ -230,7 +251,7 @@ def update():
|
|||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
print("\nError: Git not found. It's either not installed or you did "
|
print("\nError: Git not found. It's either not installed or you did "
|
||||||
"not clone this using git. Install instructions are on the GitHub: "
|
"not clone this using git. Install instructions are on the GitHub: "
|
||||||
"https://github.com/ClaytonWWilson/Image-fetcher-for-theTVDB.com")
|
"https://github.com/ClaytonWWilson/Scraper-for-theTVDB.com")
|
||||||
return
|
return
|
||||||
if code == 0:
|
if code == 0:
|
||||||
print("\nUpdating complete.\n")
|
print("\nUpdating complete.\n")
|
||||||
|
|||||||
@ -33,7 +33,8 @@ def checkStatus(response, v):
|
|||||||
else:
|
else:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def checkTimestamp(save_time, cur_time): # Returns true if the token is still valid
|
# Returns true if the token is still valid
|
||||||
|
def checkTimestamp(save_time, cur_time):
|
||||||
if cur_time - save_time < datetime.timedelta(0, 86100, 0):
|
if cur_time - save_time < datetime.timedelta(0, 86100, 0):
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
|
|||||||
@ -4,6 +4,7 @@ from login import login
|
|||||||
from actions import wait
|
from actions import wait
|
||||||
from actions import clearScreen
|
from actions import clearScreen
|
||||||
from actions import clearFolders
|
from actions import clearFolders
|
||||||
|
from actions import download
|
||||||
from actions import installReqs
|
from actions import installReqs
|
||||||
from actions import refreshToken
|
from actions import refreshToken
|
||||||
from actions import update
|
from actions import update
|
||||||
@ -27,7 +28,9 @@ while True:
|
|||||||
choice = input("> ").lower().strip()
|
choice = input("> ").lower().strip()
|
||||||
|
|
||||||
if choice == "1": # TODO catch KeyboardInterrupt at search
|
if choice == "1": # TODO catch KeyboardInterrupt at search
|
||||||
search()
|
series = search()
|
||||||
|
if series != None:
|
||||||
|
download(series)
|
||||||
wait()
|
wait()
|
||||||
elif choice == "2":
|
elif choice == "2":
|
||||||
clearScreen()
|
clearScreen()
|
||||||
|
|||||||
37
login.py
37
login.py
@ -8,6 +8,28 @@ from actions import refreshToken
|
|||||||
from checks import checkTimestamp
|
from checks import checkTimestamp
|
||||||
from checks import getToken
|
from checks import getToken
|
||||||
|
|
||||||
|
class APIConnector:
|
||||||
|
def __init__(self):
|
||||||
|
with open("login.json", "r") as f:
|
||||||
|
self.login = json.loads(f)
|
||||||
|
self.auth_headers = {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
"Authorization": "Bearer " + login["TOKEN"]
|
||||||
|
}
|
||||||
|
|
||||||
|
def reload_login(self):
|
||||||
|
with open("login.json", "r") as f:
|
||||||
|
self.login = json.loads(f)
|
||||||
|
self.auth_headers = {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
"Authorization": "Bearer " + login["TOKEN"]
|
||||||
|
}
|
||||||
|
|
||||||
|
def send_http_req(api_path):
|
||||||
|
return requests.get(api_path, headers=self.auth_headers)
|
||||||
|
|
||||||
|
|
||||||
def login():
|
def login():
|
||||||
if os.path.exists("login.json") == False:
|
if os.path.exists("login.json") == False:
|
||||||
@ -27,17 +49,18 @@ def login():
|
|||||||
tmp_user_key = ""
|
tmp_user_key = ""
|
||||||
tmp_user_name = ""
|
tmp_user_name = ""
|
||||||
|
|
||||||
print("You can find your user key & request an API key while logged in at:\n"
|
print("Please enter your Username, Unique ID, and API Key.\n"
|
||||||
"https://www.thetvdb.com/?tab=userinfo\n"
|
"You can find all of this information while logged in at:\n"
|
||||||
|
"https://www.thetvdb.com/member/api\n"
|
||||||
"Press CTRL+C to cancel.\n")
|
"Press CTRL+C to cancel.\n")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
while tmp_api_key is "":
|
|
||||||
tmp_api_key = input("Enter your api key: ")
|
|
||||||
while tmp_user_key is "":
|
|
||||||
tmp_user_key = input("Enter your user key: ")
|
|
||||||
while tmp_user_name is "":
|
while tmp_user_name is "":
|
||||||
tmp_user_name = input("Enter your username: ")
|
tmp_user_name = input("Enter your Username: ")
|
||||||
|
while tmp_user_key is "":
|
||||||
|
tmp_user_key = input("Enter your Unique ID: ")
|
||||||
|
while tmp_api_key is "":
|
||||||
|
tmp_api_key = input("Enter your API Key: ")
|
||||||
except KeyboardInterrupt as e:
|
except KeyboardInterrupt as e:
|
||||||
print("\n")
|
print("\n")
|
||||||
return
|
return
|
||||||
|
|||||||
50
search.py
50
search.py
@ -3,6 +3,7 @@ import json
|
|||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
import dateutil
|
import dateutil
|
||||||
|
import re
|
||||||
import requests
|
import requests
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
|
|
||||||
@ -14,6 +15,18 @@ from actions import searchImages
|
|||||||
from checks import checkTimestamp
|
from checks import checkTimestamp
|
||||||
from checks import checkStatus
|
from checks import checkStatus
|
||||||
|
|
||||||
|
class Series:
|
||||||
|
def __init__(self, folder_name, id, url):
|
||||||
|
self.folder_name = folder_name
|
||||||
|
self.id = str(id)
|
||||||
|
self.url = url
|
||||||
|
|
||||||
|
# Clears out all illegal filename characters from the string
|
||||||
|
def create_folder_name(string):
|
||||||
|
string = string.strip().replace(' ', '_')
|
||||||
|
string = re.sub(r'(?u)[^-\w.]', '', string)
|
||||||
|
return string
|
||||||
|
|
||||||
|
|
||||||
def search():
|
def search():
|
||||||
try:
|
try:
|
||||||
@ -29,7 +42,7 @@ def search():
|
|||||||
if checkTimestamp(save_time, cur_time) == False:
|
if checkTimestamp(save_time, cur_time) == False:
|
||||||
refreshToken()
|
refreshToken()
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
print(ex)
|
# print(ex)
|
||||||
print("There was an error checking your login. Try logging in again with 'Login/Change login'.")
|
print("There was an error checking your login. Try logging in again with 'Login/Change login'.")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@ -44,10 +57,10 @@ def search():
|
|||||||
"Authorization": "Bearer " + login["TOKEN"]
|
"Authorization": "Bearer " + login["TOKEN"]
|
||||||
}
|
}
|
||||||
|
|
||||||
keyword = input("Enter series to search: ") # Getting the search name and fixing
|
keyword = input("Enter series name to search: ") # Getting the search name and fixing
|
||||||
s_keyword = urllib.parse.quote(keyword) # the url parse mistakes
|
s_keyword = urllib.parse.quote(keyword) # the url parse mistakes
|
||||||
|
|
||||||
s_keyword = s_keyword.replace("%21", "!") # TODO find a better way of doing this
|
s_keyword = s_keyword.replace("%21", "!") # TODO find a better way of doing this
|
||||||
s_keyword = s_keyword.replace("%2A", "*")
|
s_keyword = s_keyword.replace("%2A", "*")
|
||||||
s_keyword = s_keyword.replace("%28", "(")
|
s_keyword = s_keyword.replace("%28", "(")
|
||||||
s_keyword = s_keyword.replace("%29", ")")
|
s_keyword = s_keyword.replace("%29", ")")
|
||||||
@ -67,8 +80,8 @@ def search():
|
|||||||
print()
|
print()
|
||||||
clearScreen()
|
clearScreen()
|
||||||
while title < 0 or title > len(search_results["data"]) - 1: # Looping until the user chooses
|
while title < 0 or title > len(search_results["data"]) - 1: # Looping until the user chooses
|
||||||
print("Results:") # a series from the printed list
|
print("Results:") # a series from the printed list
|
||||||
count = 1 # or they input '0' to cancel
|
count = 1 # or they input '0' to cancel
|
||||||
for result in search_results["data"]:
|
for result in search_results["data"]:
|
||||||
print("\n{})\nSeries Name: {}".format(str(count), str(result["seriesName"])))
|
print("\n{})\nSeries Name: {}".format(str(count), str(result["seriesName"])))
|
||||||
print()
|
print()
|
||||||
@ -78,6 +91,8 @@ def search():
|
|||||||
print()
|
print()
|
||||||
count = count + 1
|
count = count + 1
|
||||||
print()
|
print()
|
||||||
|
#TODO this can crash with non integer inputs
|
||||||
|
#TODO they should also be able to ctrl-c to cancel search
|
||||||
title = int(input("Choose one by number or '0' to exit: ")) - 1 # Subtracting 1 so that the
|
title = int(input("Choose one by number or '0' to exit: ")) - 1 # Subtracting 1 so that the
|
||||||
print() # index can start from 0
|
print() # index can start from 0
|
||||||
if title < -1 or title > len(search_results["data"]) - 1:
|
if title < -1 or title > len(search_results["data"]) - 1:
|
||||||
@ -88,14 +103,17 @@ def search():
|
|||||||
|
|
||||||
print()
|
print()
|
||||||
|
|
||||||
id_num = search_results["data"][title]["id"] # Setting up the request urls
|
series = Series(create_folder_name(search_results["data"][title]["seriesName"]), search_results["data"][title]["id"], "https://www.thetvdb.com/series/" + search_results["data"][title]["slug"])
|
||||||
fanart = searchImages(id_num, FAN_KEY_TYPE, authHeaders) # for banners, fanart, and posters
|
return series
|
||||||
poster = searchImages(id_num, POS_KEY_TYPE, authHeaders)
|
|
||||||
banner = searchImages(id_num, BAN_KEY_TYPE, authHeaders)
|
|
||||||
|
|
||||||
clearFolders()
|
# id_num = search_results["data"][title]["id"] # Setting up the request urls
|
||||||
downloadImages("fanart", fanart, id_num) # TODO find a better way to pass these variables. Constructor?
|
# fanart = searchImages(id_num, FAN_KEY_TYPE, authHeaders) # for banners, fanart, and posters
|
||||||
downloadImages("poster", poster, id_num)
|
# poster = searchImages(id_num, POS_KEY_TYPE, authHeaders)
|
||||||
downloadImages("banner", banner, id_num)
|
# banner = searchImages(id_num, BAN_KEY_TYPE, authHeaders)
|
||||||
print("\nAll downloads finished!\n")
|
|
||||||
return None
|
# clearFolders()
|
||||||
|
# downloadImages("fanart", fanart, id_num) # TODO find a better way to pass these variables. Constructor?
|
||||||
|
# downloadImages("poster", poster, id_num)
|
||||||
|
# downloadImages("banner", banner, id_num)
|
||||||
|
# print("\nAll downloads finished!\n")
|
||||||
|
# return None
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user