ClaytonWWilson/Scraper-for-theTVDB.com
1
0
Fork 0
mirror of https://github.com/ClaytonWWilson/Scraper-for-theTVDB.com.git synced 2025-12-15 17:28:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactoring - Added downloading for actors.

Browse Source
This commit is contained in:
Clayton Wilson 2019-05-10 00:57:08 -04:00
parent 754df95b70
commit d65ffefa70
6 changed files with 102 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,6 +2,7 @@
login.json
api info.txt
__pycache__
downloads
banner
fanart
poster

46
authentication.py
View File

@ -1,9 +1,11 @@
import json
import os.path
import datetime
import requests
import dateutil.parser
from utils import clearScreen
def login():
if os.path.exists("login.json") == False:
@ -82,6 +84,48 @@ def getToken(data):#TODO add a timeout and try catch to all requests
else:
return ""
def refreshToken():
if os.path.exists("login.json"):
try:
with open("login.json") as json_data:
login = json.load(json_data)
save_time = dateutil.parser.parse(login["TIMESTAMP"])
cur_time = datetime.datetime.now().replace(tzinfo=None) # TODO use UTC time?
json_data.close()
LOGIN_DATA = {
"apikey": login["API_KEY"],
"userkey": login["USER_KEY"],
"username": login["USER_NAME"]
}
if checkTimestamp(save_time, cur_time):
while True:
print("Your current token is still valid. Are you sure you want to grab a different one?")
choice = input("(y/n) ")
if choice is "n":
break
elif choice is "y":
login["TOKEN"] = getToken(LOGIN_DATA) # TODO find a better way to run this on both paths
login["TIMESTAMP"] = str(datetime.datetime.now().replace(tzinfo=None))
obj = open("login.json", "w")
obj.write(json.dumps(login))
obj.close()
print("\nNew token acquired!\n")
break
clearScreen()
else:
login["TOKEN"] = getToken(LOGIN_DATA)
login["TIMESTAMP"] = str(datetime.datetime.now().replace(tzinfo=None))
obj = open("login.json", "w")
obj.write(json.dumps(login))
obj.close()
print("New token acquired!\n")
except Exception as e:
print("You need to log in first. Select Login/Change login.\n") # TODO make a set of constants for error codes
else:
print("You need to log in first. Select Login/Change login.\n")
def checkStatus(response, v):
if (response.status_code != 200):
if (v == True):

6
launcher.py
View File

@ -36,10 +36,12 @@ while True:
login()
wait()
elif choice == "4":
installReqs()
# installReqs()
print("Not implemented")
wait()
elif choice == "5":
update()
# update()
print("Not implemented")
wait()
elif choice == "0":
exit()

87
main.py
View File

@ -7,6 +7,7 @@ import json
import subprocess
from utils import APIConnector
from utils import create_file_name
# TODO add counters for number of images downloaded and deleted
@ -14,48 +15,8 @@ from utils import APIConnector
def wait():
input("Press enter to continue.")
def refreshToken():
if os.path.exists("login.json"):
try:
with open("login.json") as json_data:
login = json.load(json_data)
save_time = dateutil.parser.parse(login["TIMESTAMP"])
cur_time = datetime.datetime.now().replace(tzinfo=None) # TODO use UTC time?
json_data.close()
LOGIN_DATA = {
"apikey": login["API_KEY"],
"userkey": login["USER_KEY"],
"username": login["USER_NAME"]
}
if checkTimestamp(save_time, cur_time):
while True:
print("Your current token is still valid. Are you sure you want to grab a different one?")
choice = input("(y/n) ")
if choice is "n":
break
elif choice is "y":
login["TOKEN"] = getToken(LOGIN_DATA) # TODO find a better way to run this on both paths
login["TIMESTAMP"] = str(datetime.datetime.now().replace(tzinfo=None))
obj = open("login.json", "w")
obj.write(json.dumps(login))
obj.close()
print("\nNew token acquired!\n")
break
clearScreen()
else:
login["TOKEN"] = getToken(LOGIN_DATA)
login["TIMESTAMP"] = str(datetime.datetime.now().replace(tzinfo=None))
obj = open("login.json", "w")
obj.write(json.dumps(login))
obj.close()
print("New token acquired!\n")
except Exception as e:
print("You need to log in first. Select Login/Change login.\n") # TODO make a set of constants for error codes
else:
print("You need to log in first. Select Login/Change login.\n")
# Downloads all data for a series
def download(series):
# Create downloads folder
if not os.path.exists("downloads"):
@ -68,17 +29,49 @@ def download(series):
# Create series folder
os.makedirs(os.path.join("downloads", series.folder_name))
api_path = "https://api.thetvdb.com/series/" + series.id
api_con = APIConnector()
# Download series text data to info.json
api_path = "https://api.thetvdb.com/series/{}".format(series.id)
res = api_con.send_http_req(api_path)
with open("out.json", "w") as out:
out.write(json.dumps(json.loads(res.content)))
info_path = os.path.join("downloads", series.folder_name, "info.json")
with open(info_path, 'wb') as f:
f.write(res.content)
# Make a folder for actors
actors_folder_path = os.path.join("downloads", series.folder_name, "actors")
os.makedirs(actors_folder_path)
# Download actors to actors.json
api_path = "https://api.thetvdb.com/series/{}/actors".format(series.id)
res = api_con.send_http_req(api_path)
actors_path = os.path.join("downloads", series.folder_name, "actors", "actors.json")
with open(actors_path, 'wb') as f:
f.write(res.content)
# Make folder for actor profile images
actors_profile_folder_path = os.path.join("downloads", series.folder_name, "actors", "profiles")
os.makedirs(actors_profile_folder_path)
# Download each actor's profile picture and save it as their name
for actor in json.loads(res.content)["data"]:
name = create_file_name(actor["name"])
# Check if there is an image for the actor
if actor["image"] != "":
print("downloading " + actor["image"])
img_res = requests.get("https://www.thetvdb.com/banners/" + actor["image"])
with open(os.path.join(actors_profile_folder_path, name + '_' + str(actor["id"]) + ".jpg"), 'wb') as f:
f.write(img_res.content)
else:
# Use a default image if one does not exist on theTVDB.com
shutil.copyfile(os.path.join("resources", "default_person.jpg"), os.path.join(actors_profile_folder_path, name + '_' + str(actor["id"]) + ".jpg"))
def createFolder(folder): # TODO remove this
os.makedirs(folder)
def searchImages(id_num, keyType, authHeaders): # This is getting a list of file info for images in json format

10
search.py
View File

@ -9,8 +9,10 @@ import urllib.parse
from utils import clearFolders
from utils import clearScreen
from utils import create_file_name
from authentication import checkTimestamp
from authentication import checkStatus
from authentication import refreshToken
class Series:
def __init__(self, folder_name, id, url):
@ -18,12 +20,6 @@ class Series:
self.id = str(id)
self.url = url
# Clears out all illegal filename characters from the string
def create_folder_name(string):
string = string.strip().replace(' ', '_')
string = re.sub(r'(?u)[^-\w.]', '', string)
return string
def search():
try:
@ -100,7 +96,7 @@ def search():
print()
series = Series(create_folder_name(search_results["data"][title]["seriesName"]), search_results["data"][title]["id"], "https://www.thetvdb.com/series/" + search_results["data"][title]["slug"])
series = Series(create_file_name(search_results["data"][title]["seriesName"]), search_results["data"][title]["id"], "https://www.thetvdb.com/series/" + search_results["data"][title]["slug"])
return series
# id_num = search_results["data"][title]["id"] # Setting up the request urls

9
utils.py
View File

@ -1,5 +1,6 @@
import json
import os
import re
import requests
@ -43,7 +44,7 @@ def clearFolders(): # TODO implement this
else:
print("'{}' is already empty".format(folder))
else:
createFolder(folder)
os.makedirs(folder)
print("Deleted {} images.\n".format(del_count))
def clearScreen():
@ -52,3 +53,9 @@ def clearScreen():
os.system("cls")
else:
os.system("clear")
# Clears out all illegal filename characters from the string
def create_file_name(string):
string = string.strip().replace(' ', '_')
string = re.sub(r'(?u)[^-\w.]', '', string)
return string