mirror of
https://github.com/ClaytonWWilson/Scraper-for-theTVDB.com.git
synced 2025-12-15 17:28:46 +00:00
Refactoring - Added downloading for actors.
This commit is contained in:
parent
754df95b70
commit
d65ffefa70
1
.gitignore
vendored
1
.gitignore
vendored
@ -2,6 +2,7 @@
|
|||||||
login.json
|
login.json
|
||||||
api info.txt
|
api info.txt
|
||||||
__pycache__
|
__pycache__
|
||||||
|
downloads
|
||||||
banner
|
banner
|
||||||
fanart
|
fanart
|
||||||
poster
|
poster
|
||||||
@ -1,9 +1,11 @@
|
|||||||
import json
|
import json
|
||||||
import os.path
|
import os.path
|
||||||
import datetime
|
import datetime
|
||||||
|
import requests
|
||||||
import dateutil.parser
|
import dateutil.parser
|
||||||
|
|
||||||
|
from utils import clearScreen
|
||||||
|
|
||||||
|
|
||||||
def login():
|
def login():
|
||||||
if os.path.exists("login.json") == False:
|
if os.path.exists("login.json") == False:
|
||||||
@ -82,6 +84,48 @@ def getToken(data):#TODO add a timeout and try catch to all requests
|
|||||||
else:
|
else:
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
def refreshToken():
|
||||||
|
if os.path.exists("login.json"):
|
||||||
|
try:
|
||||||
|
with open("login.json") as json_data:
|
||||||
|
login = json.load(json_data)
|
||||||
|
save_time = dateutil.parser.parse(login["TIMESTAMP"])
|
||||||
|
cur_time = datetime.datetime.now().replace(tzinfo=None) # TODO use UTC time?
|
||||||
|
json_data.close()
|
||||||
|
|
||||||
|
LOGIN_DATA = {
|
||||||
|
"apikey": login["API_KEY"],
|
||||||
|
"userkey": login["USER_KEY"],
|
||||||
|
"username": login["USER_NAME"]
|
||||||
|
}
|
||||||
|
|
||||||
|
if checkTimestamp(save_time, cur_time):
|
||||||
|
while True:
|
||||||
|
print("Your current token is still valid. Are you sure you want to grab a different one?")
|
||||||
|
choice = input("(y/n) ")
|
||||||
|
if choice is "n":
|
||||||
|
break
|
||||||
|
elif choice is "y":
|
||||||
|
login["TOKEN"] = getToken(LOGIN_DATA) # TODO find a better way to run this on both paths
|
||||||
|
login["TIMESTAMP"] = str(datetime.datetime.now().replace(tzinfo=None))
|
||||||
|
obj = open("login.json", "w")
|
||||||
|
obj.write(json.dumps(login))
|
||||||
|
obj.close()
|
||||||
|
print("\nNew token acquired!\n")
|
||||||
|
break
|
||||||
|
clearScreen()
|
||||||
|
else:
|
||||||
|
login["TOKEN"] = getToken(LOGIN_DATA)
|
||||||
|
login["TIMESTAMP"] = str(datetime.datetime.now().replace(tzinfo=None))
|
||||||
|
obj = open("login.json", "w")
|
||||||
|
obj.write(json.dumps(login))
|
||||||
|
obj.close()
|
||||||
|
print("New token acquired!\n")
|
||||||
|
except Exception as e:
|
||||||
|
print("You need to log in first. Select Login/Change login.\n") # TODO make a set of constants for error codes
|
||||||
|
else:
|
||||||
|
print("You need to log in first. Select Login/Change login.\n")
|
||||||
|
|
||||||
def checkStatus(response, v):
|
def checkStatus(response, v):
|
||||||
if (response.status_code != 200):
|
if (response.status_code != 200):
|
||||||
if (v == True):
|
if (v == True):
|
||||||
|
|||||||
@ -36,10 +36,12 @@ while True:
|
|||||||
login()
|
login()
|
||||||
wait()
|
wait()
|
||||||
elif choice == "4":
|
elif choice == "4":
|
||||||
installReqs()
|
# installReqs()
|
||||||
|
print("Not implemented")
|
||||||
wait()
|
wait()
|
||||||
elif choice == "5":
|
elif choice == "5":
|
||||||
update()
|
# update()
|
||||||
|
print("Not implemented")
|
||||||
wait()
|
wait()
|
||||||
elif choice == "0":
|
elif choice == "0":
|
||||||
exit()
|
exit()
|
||||||
|
|||||||
87
main.py
87
main.py
@ -7,6 +7,7 @@ import json
|
|||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from utils import APIConnector
|
from utils import APIConnector
|
||||||
|
from utils import create_file_name
|
||||||
|
|
||||||
|
|
||||||
# TODO add counters for number of images downloaded and deleted
|
# TODO add counters for number of images downloaded and deleted
|
||||||
@ -14,48 +15,8 @@ from utils import APIConnector
|
|||||||
def wait():
|
def wait():
|
||||||
input("Press enter to continue.")
|
input("Press enter to continue.")
|
||||||
|
|
||||||
def refreshToken():
|
|
||||||
if os.path.exists("login.json"):
|
|
||||||
try:
|
|
||||||
with open("login.json") as json_data:
|
|
||||||
login = json.load(json_data)
|
|
||||||
save_time = dateutil.parser.parse(login["TIMESTAMP"])
|
|
||||||
cur_time = datetime.datetime.now().replace(tzinfo=None) # TODO use UTC time?
|
|
||||||
json_data.close()
|
|
||||||
|
|
||||||
LOGIN_DATA = {
|
|
||||||
"apikey": login["API_KEY"],
|
|
||||||
"userkey": login["USER_KEY"],
|
|
||||||
"username": login["USER_NAME"]
|
|
||||||
}
|
|
||||||
|
|
||||||
if checkTimestamp(save_time, cur_time):
|
|
||||||
while True:
|
|
||||||
print("Your current token is still valid. Are you sure you want to grab a different one?")
|
|
||||||
choice = input("(y/n) ")
|
|
||||||
if choice is "n":
|
|
||||||
break
|
|
||||||
elif choice is "y":
|
|
||||||
login["TOKEN"] = getToken(LOGIN_DATA) # TODO find a better way to run this on both paths
|
|
||||||
login["TIMESTAMP"] = str(datetime.datetime.now().replace(tzinfo=None))
|
|
||||||
obj = open("login.json", "w")
|
|
||||||
obj.write(json.dumps(login))
|
|
||||||
obj.close()
|
|
||||||
print("\nNew token acquired!\n")
|
|
||||||
break
|
|
||||||
clearScreen()
|
|
||||||
else:
|
|
||||||
login["TOKEN"] = getToken(LOGIN_DATA)
|
|
||||||
login["TIMESTAMP"] = str(datetime.datetime.now().replace(tzinfo=None))
|
|
||||||
obj = open("login.json", "w")
|
|
||||||
obj.write(json.dumps(login))
|
|
||||||
obj.close()
|
|
||||||
print("New token acquired!\n")
|
|
||||||
except Exception as e:
|
|
||||||
print("You need to log in first. Select Login/Change login.\n") # TODO make a set of constants for error codes
|
|
||||||
else:
|
|
||||||
print("You need to log in first. Select Login/Change login.\n")
|
|
||||||
|
|
||||||
|
# Downloads all data for a series
|
||||||
def download(series):
|
def download(series):
|
||||||
# Create downloads folder
|
# Create downloads folder
|
||||||
if not os.path.exists("downloads"):
|
if not os.path.exists("downloads"):
|
||||||
@ -68,17 +29,49 @@ def download(series):
|
|||||||
# Create series folder
|
# Create series folder
|
||||||
os.makedirs(os.path.join("downloads", series.folder_name))
|
os.makedirs(os.path.join("downloads", series.folder_name))
|
||||||
|
|
||||||
api_path = "https://api.thetvdb.com/series/" + series.id
|
|
||||||
|
|
||||||
api_con = APIConnector()
|
api_con = APIConnector()
|
||||||
|
|
||||||
|
# Download series text data to info.json
|
||||||
|
api_path = "https://api.thetvdb.com/series/{}".format(series.id)
|
||||||
|
res = api_con.send_http_req(api_path)
|
||||||
|
|
||||||
|
info_path = os.path.join("downloads", series.folder_name, "info.json")
|
||||||
|
|
||||||
|
with open(info_path, 'wb') as f:
|
||||||
|
f.write(res.content)
|
||||||
|
|
||||||
|
# Make a folder for actors
|
||||||
|
actors_folder_path = os.path.join("downloads", series.folder_name, "actors")
|
||||||
|
os.makedirs(actors_folder_path)
|
||||||
|
|
||||||
|
# Download actors to actors.json
|
||||||
|
api_path = "https://api.thetvdb.com/series/{}/actors".format(series.id)
|
||||||
res = api_con.send_http_req(api_path)
|
res = api_con.send_http_req(api_path)
|
||||||
|
|
||||||
with open("out.json", "w") as out:
|
actors_path = os.path.join("downloads", series.folder_name, "actors", "actors.json")
|
||||||
out.write(json.dumps(json.loads(res.content)))
|
|
||||||
|
with open(actors_path, 'wb') as f:
|
||||||
|
f.write(res.content)
|
||||||
|
|
||||||
|
# Make folder for actor profile images
|
||||||
|
actors_profile_folder_path = os.path.join("downloads", series.folder_name, "actors", "profiles")
|
||||||
|
os.makedirs(actors_profile_folder_path)
|
||||||
|
|
||||||
|
# Download each actor's profile picture and save it as their name
|
||||||
|
for actor in json.loads(res.content)["data"]:
|
||||||
|
name = create_file_name(actor["name"])
|
||||||
|
|
||||||
|
# Check if there is an image for the actor
|
||||||
|
if actor["image"] != "":
|
||||||
|
print("downloading " + actor["image"])
|
||||||
|
img_res = requests.get("https://www.thetvdb.com/banners/" + actor["image"])
|
||||||
|
with open(os.path.join(actors_profile_folder_path, name + '_' + str(actor["id"]) + ".jpg"), 'wb') as f:
|
||||||
|
f.write(img_res.content)
|
||||||
|
else:
|
||||||
|
# Use a default image if one does not exist on theTVDB.com
|
||||||
|
shutil.copyfile(os.path.join("resources", "default_person.jpg"), os.path.join(actors_profile_folder_path, name + '_' + str(actor["id"]) + ".jpg"))
|
||||||
|
|
||||||
|
|
||||||
def createFolder(folder): # TODO remove this
|
|
||||||
os.makedirs(folder)
|
|
||||||
|
|
||||||
|
|
||||||
def searchImages(id_num, keyType, authHeaders): # This is getting a list of file info for images in json format
|
def searchImages(id_num, keyType, authHeaders): # This is getting a list of file info for images in json format
|
||||||
|
|||||||
10
search.py
10
search.py
@ -9,20 +9,16 @@ import urllib.parse
|
|||||||
|
|
||||||
from utils import clearFolders
|
from utils import clearFolders
|
||||||
from utils import clearScreen
|
from utils import clearScreen
|
||||||
|
from utils import create_file_name
|
||||||
from authentication import checkTimestamp
|
from authentication import checkTimestamp
|
||||||
from authentication import checkStatus
|
from authentication import checkStatus
|
||||||
|
from authentication import refreshToken
|
||||||
|
|
||||||
class Series:
|
class Series:
|
||||||
def __init__(self, folder_name, id, url):
|
def __init__(self, folder_name, id, url):
|
||||||
self.folder_name = folder_name
|
self.folder_name = folder_name
|
||||||
self.id = str(id)
|
self.id = str(id)
|
||||||
self.url = url
|
self.url = url
|
||||||
|
|
||||||
# Clears out all illegal filename characters from the string
|
|
||||||
def create_folder_name(string):
|
|
||||||
string = string.strip().replace(' ', '_')
|
|
||||||
string = re.sub(r'(?u)[^-\w.]', '', string)
|
|
||||||
return string
|
|
||||||
|
|
||||||
|
|
||||||
def search():
|
def search():
|
||||||
@ -100,7 +96,7 @@ def search():
|
|||||||
|
|
||||||
print()
|
print()
|
||||||
|
|
||||||
series = Series(create_folder_name(search_results["data"][title]["seriesName"]), search_results["data"][title]["id"], "https://www.thetvdb.com/series/" + search_results["data"][title]["slug"])
|
series = Series(create_file_name(search_results["data"][title]["seriesName"]), search_results["data"][title]["id"], "https://www.thetvdb.com/series/" + search_results["data"][title]["slug"])
|
||||||
return series
|
return series
|
||||||
|
|
||||||
# id_num = search_results["data"][title]["id"] # Setting up the request urls
|
# id_num = search_results["data"][title]["id"] # Setting up the request urls
|
||||||
|
|||||||
11
utils.py
11
utils.py
@ -1,5 +1,6 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
|
||||||
@ -43,7 +44,7 @@ def clearFolders(): # TODO implement this
|
|||||||
else:
|
else:
|
||||||
print("'{}' is already empty".format(folder))
|
print("'{}' is already empty".format(folder))
|
||||||
else:
|
else:
|
||||||
createFolder(folder)
|
os.makedirs(folder)
|
||||||
print("Deleted {} images.\n".format(del_count))
|
print("Deleted {} images.\n".format(del_count))
|
||||||
|
|
||||||
def clearScreen():
|
def clearScreen():
|
||||||
@ -51,4 +52,10 @@ def clearScreen():
|
|||||||
if IS_WINDOWS:
|
if IS_WINDOWS:
|
||||||
os.system("cls")
|
os.system("cls")
|
||||||
else:
|
else:
|
||||||
os.system("clear")
|
os.system("clear")
|
||||||
|
|
||||||
|
# Clears out all illegal filename characters from the string
|
||||||
|
def create_file_name(string):
|
||||||
|
string = string.strip().replace(' ', '_')
|
||||||
|
string = re.sub(r'(?u)[^-\w.]', '', string)
|
||||||
|
return string
|
||||||
Loading…
Reference in New Issue
Block a user