ClaytonWWilson/Scraper-for-theTVDB.com
1
0
Fork 0
mirror of https://github.com/ClaytonWWilson/Scraper-for-theTVDB.com.git synced 2025-12-15 17:28:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

The start of refactoring code

Browse Source
This commit is contained in:
Clayton Wilson 2019-05-08 17:53:46 -04:00
parent 3bb7b78a96
commit de2bc5c576
6 changed files with 115 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -2,3 +2,6 @@
login.json
api info.txt
__pycache__
banner
fanart
poster

63
actions.py
View File

@ -65,6 +65,27 @@ def refreshToken():
else:
print("You need to log in first. Select Login/Change login.\n")
def download(series):
# Create downloads folder
if not os.path.exists("downloads"):
os.makedirs("downloads")
# Remove previously downloaded content for this series if it exists
if os.path.exists(os.path.join("downloads", series.folder_name)):
shutil.rmtree(os.path.join("downloads", series.folder_name))
# Create series folder
os.makedirs(os.path.join("downloads", series.folder_name))
api_path = "https://api.thetvdb.com/series/" + series.id
api_con = APIConnector()
res = api_con.send_http_req(api_path)
with open("out.json", "w") as out:
out.write(json.dumps(json.loads(res.content)))
def clearFolders(): # TODO implement this
folders = ["banner", "fanart", "poster"]
del_count = 0
@ -163,29 +184,29 @@ def tryMissing(missing_nums, min_num, max_num, id_num, image_type):
# while min_num > 1: # Checking lower bounds
# print("check lower")
def download(image_type, parse_resp_obj):
counter = 0
save_name_list = []
for image_obj in parse_resp_obj["data"]:
filename = parse_resp_obj["data"][counter]["filename"] # TODO the download method should start here, move everything else up to downloadImages
counter = counter + 1
# def download(image_type, parse_resp_obj):
# counter = 0
# save_name_list = []
# for image_obj in parse_resp_obj["data"]:
# filename = parse_resp_obj["data"][counter]["filename"] # TODO the download method should start here, move everything else up to downloadImages
# counter = counter + 1
slash_index = filename.rfind("/") # This is used to slice the url at the beginning of the filename
save_name = filename[slash_index + 1:] # For example 'https://thetvdb.com/banners/fanart/original/32451-3.jpg' --> '32451.jpg'
save_name_list.append(save_name)
# slash_index = filename.rfind("/") # This is used to slice the url at the beginning of the filename
# save_name = filename[slash_index + 1:] # For example 'https://thetvdb.com/banners/fanart/original/32451-3.jpg' --> '32451.jpg'
# save_name_list.append(save_name)
print("Downloading... {}".format(filename))
dl_url = "https://www.thetvdb.com/banners/{}".format(filename)
response = requests.get(dl_url) # TODO getting errors when checking 'new game'. Check to see if those images actually exist
# print("Downloading... {}".format(filename))
# dl_url = "https://www.thetvdb.com/banners/{}".format(filename)
# response = requests.get(dl_url) # TODO getting errors when checking 'new game'. Check to see if those images actually exist
if (checkStatus(response, True)):
path = os.path.join(image_type + "\\", save_name)
obj = open(path, "wb")
obj.write(response.content)
obj.close()
else:
quit()
return save_name_list
# if (checkStatus(response, True)):
# path = os.path.join(image_type + "\\", save_name)
# obj = open(path, "wb")
# obj.write(response.content)
# obj.close()
# else:
# quit()
# return save_name_list
def installReqs():
if is_pip_installed() == True:
@ -230,7 +251,7 @@ def update():
except FileNotFoundError:
print("\nError: Git not found. It's either not installed or you did "
"not clone this using git. Install instructions are on the GitHub: "
"https://github.com/ClaytonWWilson/Image-fetcher-for-theTVDB.com")
"https://github.com/ClaytonWWilson/Scraper-for-theTVDB.com")
return
if code == 0:
print("\nUpdating complete.\n")

3
checks.py
View File

@ -33,7 +33,8 @@ def checkStatus(response, v):
else:
return True
def checkTimestamp(save_time, cur_time): # Returns true if the token is still valid
# Returns true if the token is still valid
def checkTimestamp(save_time, cur_time):
if cur_time - save_time < datetime.timedelta(0, 86100, 0):
return True
else:

5
launcher.py
View File

@ -4,6 +4,7 @@ from login import login
from actions import wait
from actions import clearScreen
from actions import clearFolders
from actions import download
from actions import installReqs
from actions import refreshToken
from actions import update
@ -27,7 +28,9 @@ while True:
choice = input("> ").lower().strip()
if choice == "1": # TODO catch KeyboardInterrupt at search
search()
series = search()
if series != None:
download(series)
wait()
elif choice == "2":
clearScreen()

37
login.py
View File

@ -8,6 +8,28 @@ from actions import refreshToken
from checks import checkTimestamp
from checks import getToken
class APIConnector:
def __init__(self):
with open("login.json", "r") as f:
self.login = json.loads(f)
self.auth_headers = {
"Content-Type": "application/json",
"Accept": "application/json",
"Authorization": "Bearer " + login["TOKEN"]
}
def reload_login(self):
with open("login.json", "r") as f:
self.login = json.loads(f)
self.auth_headers = {
"Content-Type": "application/json",
"Accept": "application/json",
"Authorization": "Bearer " + login["TOKEN"]
}
def send_http_req(api_path):
return requests.get(api_path, headers=self.auth_headers)
def login():
if os.path.exists("login.json") == False:
@ -27,17 +49,18 @@ def login():
tmp_user_key = ""
tmp_user_name = ""
print("You can find your user key & request an API key while logged in at:\n"
"https://www.thetvdb.com/?tab=userinfo\n"
print("Please enter your Username, Unique ID, and API Key.\n"
"You can find all of this information while logged in at:\n"
"https://www.thetvdb.com/member/api\n"
"Press CTRL+C to cancel.\n")
try:
while tmp_api_key is "":
tmp_api_key = input("Enter your api key: ")
while tmp_user_key is "":
tmp_user_key = input("Enter your user key: ")
while tmp_user_name is "":
tmp_user_name = input("Enter your username: ")
tmp_user_name = input("Enter your Username: ")
while tmp_user_key is "":
tmp_user_key = input("Enter your Unique ID: ")
while tmp_api_key is "":
tmp_api_key = input("Enter your API Key: ")
except KeyboardInterrupt as e:
print("\n")
return

50
search.py
View File

@ -3,6 +3,7 @@ import json
import datetime
import dateutil
import re
import requests
import urllib.parse
@ -14,6 +15,18 @@ from actions import searchImages
from checks import checkTimestamp
from checks import checkStatus
class Series:
def __init__(self, folder_name, id, url):
self.folder_name = folder_name
self.id = str(id)
self.url = url
# Clears out all illegal filename characters from the string
def create_folder_name(string):
string = string.strip().replace(' ', '_')
string = re.sub(r'(?u)[^-\w.]', '', string)
return string
def search():
try:
@ -29,7 +42,7 @@ def search():
if checkTimestamp(save_time, cur_time) == False:
refreshToken()
except Exception as ex:
print(ex)
# print(ex)
print("There was an error checking your login. Try logging in again with 'Login/Change login'.")
return None
@ -44,10 +57,10 @@ def search():
"Authorization": "Bearer " + login["TOKEN"]
}
keyword = input("Enter series to search: ") # Getting the search name and fixing
s_keyword = urllib.parse.quote(keyword) # the url parse mistakes
keyword = input("Enter series name to search: ") # Getting the search name and fixing
s_keyword = urllib.parse.quote(keyword) # the url parse mistakes
s_keyword = s_keyword.replace("%21", "!") # TODO find a better way of doing this
s_keyword = s_keyword.replace("%21", "!") # TODO find a better way of doing this
s_keyword = s_keyword.replace("%2A", "*")
s_keyword = s_keyword.replace("%28", "(")
s_keyword = s_keyword.replace("%29", ")")
@ -67,8 +80,8 @@ def search():
print()
clearScreen()
while title < 0 or title > len(search_results["data"]) - 1: # Looping until the user chooses
print("Results:") # a series from the printed list
count = 1 # or they input '0' to cancel
print("Results:") # a series from the printed list
count = 1 # or they input '0' to cancel
for result in search_results["data"]:
print("\n{})\nSeries Name: {}".format(str(count), str(result["seriesName"])))
print()
@ -78,6 +91,8 @@ def search():
print()
count = count + 1
print()
#TODO this can crash with non integer inputs
#TODO they should also be able to ctrl-c to cancel search
title = int(input("Choose one by number or '0' to exit: ")) - 1 # Subtracting 1 so that the
print() # index can start from 0
if title < -1 or title > len(search_results["data"]) - 1:
@ -88,14 +103,17 @@ def search():
print()
id_num = search_results["data"][title]["id"] # Setting up the request urls
fanart = searchImages(id_num, FAN_KEY_TYPE, authHeaders) # for banners, fanart, and posters
poster = searchImages(id_num, POS_KEY_TYPE, authHeaders)
banner = searchImages(id_num, BAN_KEY_TYPE, authHeaders)
series = Series(create_folder_name(search_results["data"][title]["seriesName"]), search_results["data"][title]["id"], "https://www.thetvdb.com/series/" + search_results["data"][title]["slug"])
return series
clearFolders()
downloadImages("fanart", fanart, id_num) # TODO find a better way to pass these variables. Constructor?
downloadImages("poster", poster, id_num)
downloadImages("banner", banner, id_num)
print("\nAll downloads finished!\n")
return None
# id_num = search_results["data"][title]["id"] # Setting up the request urls
# fanart = searchImages(id_num, FAN_KEY_TYPE, authHeaders) # for banners, fanart, and posters
# poster = searchImages(id_num, POS_KEY_TYPE, authHeaders)
# banner = searchImages(id_num, BAN_KEY_TYPE, authHeaders)
# clearFolders()
# downloadImages("fanart", fanart, id_num) # TODO find a better way to pass these variables. Constructor?
# downloadImages("poster", poster, id_num)
# downloadImages("banner", banner, id_num)
# print("\nAll downloads finished!\n")
# return None