diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a05f10..0d0645a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ ## 1.x (Terminal) +### 1.9 + +Spending four hours squashing these bugs is probably better than spending four hours downloading the same things over and over again when it crashes. + +##### Added + + - Songs downloaded from YouTube now get ID3 tags (title, artist, etc). + +##### Changed + + - Tracks are now cached as soon as they're downloaded, which makes the program much more crash-resistant. + - Playlist files are now updated as the songs are added to iTunes, which makes the program much more crash-resistant and accurate. + ### 1.8 Something went wrong along the way... but now it should be fixed! diff --git a/ideemyouworthy/downloadfinished_messageinterface.py b/ideemyouworthy/downloadfinished_messageinterface.py index 157d0ad..03f49a8 100644 --- a/ideemyouworthy/downloadfinished_messageinterface.py +++ b/ideemyouworthy/downloadfinished_messageinterface.py @@ -42,11 +42,11 @@ def send(self, message, value = None): self.master_track_file.write_text(json.dumps(master_track_dict, indent = 4, ensure_ascii = False), encoding = "utf-8") break - if len(self.queue_manager.queue) == 0 and not self.track_manager.has_finished_queue: - self.track_manager.has_finished_queue = True - - if self.youtube_manager is None: - self.track_manager.finished_queue(self.downloaded_tracks, self.new_playlists, self.playlist_changes, self.use_itunes) - else: - self.youtube_manager.update_objects(self.downloaded_tracks, self.new_playlists, self.playlist_changes, self.use_itunes, self.track_manager) - self.youtube_manager.start_download_process() + if len(self.queue_manager.queue) == 0 and not self.track_manager.has_finished_queue: + self.track_manager.has_finished_queue = True + + if self.youtube_manager is None: + self.track_manager.finished_queue(self.downloaded_tracks, self.new_playlists, self.playlist_changes, self.use_itunes) + else: + self.youtube_manager.update_objects(self.downloaded_tracks, self.new_playlists, self.playlist_changes, self.use_itunes, self.track_manager) + self.youtube_manager.start_download_process() diff --git a/ideemyouworthy/main.py b/ideemyouworthy/main.py index 093446c..4a0d063 100644 --- a/ideemyouworthy/main.py +++ b/ideemyouworthy/main.py @@ -19,14 +19,14 @@ # START TESTING -# delete_path = Path(Path.cwd().parent / "playlists") -# if delete_path.exists(): shutil.rmtree(delete_path) -# -# delete_path = Path(Path.cwd().parent / "music") -# if delete_path.exists(): shutil.rmtree(delete_path) -# -# delete_path = Path(Path.cwd().parent / "cache" / "track_master_list.json") -# if delete_path.exists(): os.remove(delete_path) +delete_path = Path(Path.cwd().parent / "playlists") +if delete_path.exists(): shutil.rmtree(delete_path) + +delete_path = Path(Path.cwd().parent / "music") +if delete_path.exists(): shutil.rmtree(delete_path) + +delete_path = Path(Path.cwd().parent / "cache" / "track_master_list.json") +if delete_path.exists(): os.remove(delete_path) # END TESTING @@ -37,7 +37,9 @@ account_manager.login_spotify() music_directory = str(Path.cwd().parents[0] / "music") -youtube_manager = YoutubeManager(logger, account_manager.spotify_manager, music_directory) + +youtube_tag_dict = collections.OrderedDict() +youtube_manager = YoutubeManager(logger, account_manager.spotify_manager, music_directory, youtube_tag_dict) playlist_manager = PlaylistManager(logger, account_manager) @@ -102,6 +104,8 @@ queue_list.append("https://www.deezer.com/en/track/" + str(deezer_id[0])) else: + youtube_tag_dict[track] = track_manager.get_track_data(track) + search_string = youtube_manager.get_search_string(split_uri[2]) first_result = youtube_manager.search(search_string) youtube_list.append(first_result) @@ -111,9 +115,11 @@ logger.info("Downloading " + str(len(queue_list)) + " deezer tracks") logger.info("Downloading " + str(len(youtube_list)) + " YouTube tracks") - if len(youtube_list) != 0: + youtube_num = len(youtube_list) + + if youtube_num != 0: youtube_manager.url_list = youtube_list - youtube_manager.youtube_tracks_to_download = len(youtube_list) + youtube_manager.youtube_tracks_to_download = youtube_num message_interface.youtube_manager = youtube_manager if len(queue_list) != 0: @@ -123,6 +129,9 @@ youtube_manager.update_objects(downloaded_tracks, new_playlists, playlist_changes, use_itunes, track_manager) youtube_manager.start_download_process() + if youtube_num != 0: + youtube_manager.add_tags() + else: logger.info("Downloading 0 tracks") diff --git a/ideemyouworthy/playlistmanager.py b/ideemyouworthy/playlistmanager.py index 2917227..60f8750 100644 --- a/ideemyouworthy/playlistmanager.py +++ b/ideemyouworthy/playlistmanager.py @@ -8,7 +8,7 @@ class PlaylistManager: - + # todo: don't update playlist files until it's actually downloaded def __init__(self, logger, account_manager): self.logger = logger @@ -172,7 +172,7 @@ def create_m3u(self): with playlist_m3u.open("a") as append_file: for track in playlist_tracks: - try: # TODO: I think this is fixing the exception if the track isn't in the master list, aka isn't on deezer + try: track_file_path = master_track_dict[track]["download_location"] append_file.write(track_file_path + "\n") except: diff --git a/ideemyouworthy/trackmanager.py b/ideemyouworthy/trackmanager.py index 92e17dc..565f45f 100644 --- a/ideemyouworthy/trackmanager.py +++ b/ideemyouworthy/trackmanager.py @@ -34,6 +34,16 @@ def get_playlist_tracks(self, playlist_id, fields): tracks.extend(results['items']) return tracks + def get_track_data(self, uri): + track = self.spotify_manager.track(uri) + track_data = {} + track_data["name"] = track["name"] + track_data["track_number"] = track["track_number"] + track_data["album"] = track["album"]["name"] + track_data["artist"] = track["artists"][0]["name"] + + return track_data + def find_new_tracks(self, new_playlists): playlist_changes = collections.OrderedDict() @@ -64,10 +74,14 @@ def find_new_tracks(self, new_playlists): playlist_changes[playlist] = playlist_differences self.logger.info(playlist + ": Found " + str(len(playlist_differences)) + " new tracks") - playlist_file_path.write_text(json.dumps(new_playlist_songs, indent = 4, ensure_ascii = False), encoding = "utf-8") - return playlist_changes + def add_track_to_playlist(self, playlist_name, track_uri): + playlist_file_path = Path.cwd().parents[0] / "playlists" / (playlist_name + ".json") + playlist_dict = json.loads(playlist_file_path.read_text(encoding = "utf-8")) + playlist_dict[track_uri] = None + playlist_file_path.write_text(json.dumps(playlist_dict, indent = 4, ensure_ascii = False), encoding = "utf-8") + def clear_duplicate_downloads(self, playlist_changes): master_track_dict = json.loads(self.master_track_file.read_text(encoding = "utf-8")) master_track_set = util.dictToSet(master_track_dict) @@ -125,6 +139,8 @@ def finished_queue(self, downloaded_tracks, new_playlists, playlist_changes, use for track in playlist_changes[playlist]: if track in old_master_track_dict and isinstance(old_master_track_dict[track], dict): # (this will be true unless there was a downloading error) itunes_playlists_dict[playlist].AddFile(old_master_track_dict[track]["download_location"]) + self.add_track_to_playlist(playlist, track) + print("Added track " + track + " to playlist " + playlist) self.logger.info("Finished updating iTunes") else: @@ -160,8 +176,10 @@ def fix_itunes(self, itunes_playlists_dict, playlist_edits): self.logger.info(missing_track + " could not be added to iTunes. File path length: " + file_path_length) self.store_problematic_track("(" + file_path_length + ") " + missing_track) - self.logger.info(playlist + ": Removed " + str(extra_count) + " extra tracks") - self.logger.info(playlist + ": Added " + str(missing_count) + " missing tracks") + if extra_count > 0: + self.logger.info(playlist + ": Removed " + str(extra_count) + " extra tracks") + if missing_count > 0: + self.logger.info(playlist + ": Added " + str(missing_count) + " missing tracks") # This verifies that the iTunes and cached playlists agree def verify_itunes(self): @@ -187,7 +205,11 @@ def verify_itunes(self): for playlist in itunes_playlists_dict: playlist_file_path = Path.cwd().parents[0] / "playlists" / (playlist + ".json") - official_playlist_dict = json.loads(playlist_file_path.read_text()) # todo: this doesn't allow other playlists in itunes + + if not playlist_file_path.exists(): + continue + + official_playlist_dict = json.loads(playlist_file_path.read_text()) official_locations = set() for track in official_playlist_dict: diff --git a/ideemyouworthy/youtubemanager.py b/ideemyouworthy/youtubemanager.py index 64961ff..4521a91 100644 --- a/ideemyouworthy/youtubemanager.py +++ b/ideemyouworthy/youtubemanager.py @@ -1,18 +1,18 @@ import json +import os import re -import subprocess from pathlib import Path -import os, sys import mutagen -from tinytag import TinyTag from youtube_search import YoutubeSearch +from mutagen.id3 import ID3, TIT2, TRCK, TALB, TPE1 +from mutagen import File import youtube_dl class YoutubeManager: - def __init__(self, logger, spotify_manager, music_directory): + def __init__(self, logger, spotify_manager, music_directory, youtube_tag_dict): self.logger = logger self.spotify_manager = spotify_manager @@ -27,6 +27,8 @@ def __init__(self, logger, spotify_manager, music_directory): self.youtube_tracks_to_download = None self.current_download_count = 0 + self.youtube_tag_dict = youtube_tag_dict + ydl_opts = { 'format': 'bestaudio/best', 'postprocessors': [{ @@ -97,45 +99,24 @@ def continue_download_process(self): def progress_hook(self, response): if response["status"] == "finished": file_name = response["filename"] - file_name = re.search("(.*\.)([a-zA-Z1-9]*)$", file_name).group(1) + "mp3" - - print("[" + str(self.current_download_count) + "/" + str(self.youtube_tracks_to_download) + "] Downloaded " + file_name) + fake_path = re.search("(.*\.)([a-zA-Z1-9]*)$", file_name).group(1) + "mp3" - # spotify_uri = None - # - # # downloaded_tracks uses full spotify uri - # full_tags_dict = json.loads(self.youtube_tag_cache_file.read_text()) - # tags_dict = full_tags_dict[spotify_uri] - # tags_dict["title"] - # tags_dict["album_artist"] + spotify_uri = None for track in self.downloaded_tracks: if self.downloaded_tracks[track] == self.current_download_url: - self.downloaded_tracks[track] = {"youtube_url": self.current_download_url, "download_location": file_name} - - master_track_dict = json.loads(self.master_track_file.read_text(encoding = "utf-8")) - master_track_dict[track] = self.downloaded_tracks[track] - self.master_track_file.write_text(json.dumps(master_track_dict, indent = 4, ensure_ascii = False), encoding = "utf-8") + spotify_uri = track break - # spotify_uri = track - - # from mutagen.easyid3 import EasyID3 - # from mutagen.id3 import ID3 - # from mutagen import File, MutagenError - # try: - # tagged_file = EasyID3(file_name) - # except mutagen.id3.ID3NoHeaderError: - # tagged_file = File(file_name) - # print(tagged_file.items()) - # tagged_file.add_tags() - # tagged_file.save() - # tagged_file = EasyID3(file_name) - # - # tagged_file["title"] = file_name - # tagged_file.save() - - # TODO: Add tags here + self.youtube_tag_dict[spotify_uri]["filepath"] = fake_path + + print("[" + str(self.current_download_count) + "/" + str(self.youtube_tracks_to_download) + "] Downloaded " + fake_path) + + self.downloaded_tracks[spotify_uri] = {"youtube_url": self.current_download_url, "download_location": fake_path} + + master_track_dict = json.loads(self.master_track_file.read_text(encoding = "utf-8")) + master_track_dict[spotify_uri] = self.downloaded_tracks[spotify_uri] + self.master_track_file.write_text(json.dumps(master_track_dict, indent = 4, ensure_ascii = False), encoding = "utf-8") if len(self.url_list) != 0: self.continue_download_process() @@ -144,6 +125,55 @@ def progress_hook(self, response): self.logger.info("Finished YouTube downloads") self.track_manager.finished_queue(self.downloaded_tracks, self.new_playlists, self.playlist_changes, self.use_itunes) + def add_tags(self): + + def remove(value, deletechars): + for c in deletechars: + value = value.replace(c, '') + return value + + for uri in self.youtube_tag_dict: + + if "filepath" in self.youtube_tag_dict[uri]: + file_path = Path(self.youtube_tag_dict[uri]["filepath"]) + + try: + tagged_file = ID3() + except mutagen.id3.ID3NoHeaderError: + tagged_file = File() + tagged_file.add_tags() + tagged_file.save() + tagged_file = ID3() + + if self.youtube_tag_dict[uri]["name"]: + tagged_file["TIT2"] = TIT2(encoding = 3, text = self.youtube_tag_dict[uri]["name"]) + if self.youtube_tag_dict[uri]["track_number"]: + try: + tagged_file["TRCK"] = TRCK(encoding = 3, text = str(self.youtube_tag_dict[uri]["track_number"])) + except: + tagged_file["TRCK"] = TRCK(encoding = 3, text = u"1") + if self.youtube_tag_dict[uri]["album"]: + tagged_file["TALB"] = TALB(encoding = 3, text = self.youtube_tag_dict[uri]["album"]) + if self.youtube_tag_dict[uri]["artist"]: + tagged_file["TPE1"] = TPE1(encoding = 3, text = self.youtube_tag_dict[uri]["artist"]) + + tagged_file.save(file_path) + + while True: + try: + file_path.rename(file_path) + except: + continue + break + + new_path = Path(file_path.parent / Path(remove(f"{self.youtube_tag_dict[uri]['artist']} - {self.youtube_tag_dict[uri]['name']}.mp3", '\/:*?"<>|'))) + + os.rename(file_path, new_path) + + master_track_dict = json.loads(self.master_track_file.read_text(encoding = "utf-8")) + master_track_dict[uri]["download_location"] = str(new_path) + self.master_track_file.write_text(json.dumps(master_track_dict, indent = 4, ensure_ascii = False), encoding = "utf-8") + def update_objects(self, downloaded_tracks, new_playlists, playlist_changes, use_itunes, track_manager): self.downloaded_tracks = downloaded_tracks self.new_playlists = new_playlists