From 4d36985c33ca79f98722d72d976ad402eba83fe0 Mon Sep 17 00:00:00 2001 From: Rui NARCISO Date: Tue, 4 Jun 2024 16:26:25 +0200 Subject: [PATCH] added ability to separate the folders for albums and shared albums --- src/gphotos_sync/DatabaseMedia.py | 2 ++ src/gphotos_sync/GoogleAlbumsRow.py | 7 ++++- src/gphotos_sync/GoogleAlbumsSync.py | 39 +++++++++++++++++++----- src/gphotos_sync/GooglePhotosDownload.py | 22 ++++++++----- src/gphotos_sync/LocalData.py | 4 +-- src/gphotos_sync/LocalFilesMedia.py | 10 ++++++ src/gphotos_sync/Settings.py | 1 + src/gphotos_sync/__main__.py | 7 +++++ src/gphotos_sync/authorize.py | 5 ++- src/gphotos_sync/sql/gphotos_create.sql | 3 +- 10 files changed, 79 insertions(+), 21 deletions(-) diff --git a/src/gphotos_sync/DatabaseMedia.py b/src/gphotos_sync/DatabaseMedia.py index 1c35c04d..c348486d 100644 --- a/src/gphotos_sync/DatabaseMedia.py +++ b/src/gphotos_sync/DatabaseMedia.py @@ -53,6 +53,7 @@ def __init__( _create_date: datetime = Utils.MINIMUM_DATE, _downloaded: bool = False, _location: str = "", + _is_shared_album: bool = False, ): super(DatabaseMedia, self).__init__() self._id = _id @@ -69,6 +70,7 @@ def __init__( self._create_date = _create_date self._downloaded = _downloaded self._location = _location + self._is_shared_album = _is_shared_album # this is used to replace meta data that has been extracted from the # file system and overrides that provided by Google API diff --git a/src/gphotos_sync/GoogleAlbumsRow.py b/src/gphotos_sync/GoogleAlbumsRow.py index bd972e2c..8f7713ed 100644 --- a/src/gphotos_sync/GoogleAlbumsRow.py +++ b/src/gphotos_sync/GoogleAlbumsRow.py @@ -26,6 +26,7 @@ class GoogleAlbumsRow(DbRow): "EndDate": datetime, "SyncDate": datetime, "Downloaded": bool, + "IsSharedAlbum": bool, } # All properties on this class are dynamically added from the above @@ -37,6 +38,7 @@ def to_media(self) -> DatabaseMedia: # type:ignore _filename=self.AlbumName, # type:ignore _size=self.Size, # type:ignore _create_date=self.EndDate, # type:ignore + _is_shared_album=self.IsSharedAlbum, # type:ignore ) return db_media @@ -45,7 +47,9 @@ def from_media(cls, album) -> GoogleAlbumMedia: # type:ignore pass @classmethod - def from_parm(cls, album_id, filename, size, start, end) -> "GoogleAlbumsRow": + def from_parm( + cls, album_id, filename, size, start, end, is_shared + ) -> "GoogleAlbumsRow": new_row = cls.make( RemoteId=album_id, AlbumName=filename, @@ -54,5 +58,6 @@ def from_parm(cls, album_id, filename, size, start, end) -> "GoogleAlbumsRow": EndDate=end, SyncDate=Utils.date_to_string(datetime.now()), Downloaded=0, + IsSharedAlbum=is_shared, ) return new_row diff --git a/src/gphotos_sync/GoogleAlbumsSync.py b/src/gphotos_sync/GoogleAlbumsSync.py index 847802cf..3baa5ca4 100644 --- a/src/gphotos_sync/GoogleAlbumsSync.py +++ b/src/gphotos_sync/GoogleAlbumsSync.py @@ -43,9 +43,17 @@ def __init__( """ self._photos_folder = settings.photos_path self._albums_folder = settings.albums_path + self._shared_albums_folder = settings.shared_albums_path self._root_folder: Path = root_folder - self._links_root = self._root_folder / self._albums_folder + if not self._albums_folder.is_absolute(): + self._albums_root = self._root_folder / self._albums_folder + else: + self._albums_root = self._albums_folder + if not self._shared_albums_folder.is_absolute(): + self._shared_albums_root = self._root_folder / self._shared_albums_folder + else: + self._shared_albums_root = self._shared_albums_folder self._photos_root = self._root_folder / self._photos_folder self._db: LocalData = db self._api: RestClient = api @@ -224,7 +232,12 @@ def index_albums_type( # write the album data down now we know the contents' # date range gar = GoogleAlbumsRow.from_parm( - album.id, album.filename, album.size, first_date, last_date + album.id, + album.filename, + album.size, + first_date, + last_date, + {"albums": False, "sharedAlbums": True}.get(item_key), ) self._db.put_row(gar, update=indexed_album) @@ -239,7 +252,7 @@ def index_albums_type( log.warning("Indexed %d %s", count, description) def album_folder_name( - self, album_name: str, start_date: datetime, end_date: datetime + self, album_name: str, start_date: datetime, end_date: datetime, shared: bool ) -> Path: album_name = get_check().valid_file_name(album_name) if self._omit_album_date: @@ -259,7 +272,11 @@ def album_folder_name( fmt = self.path_format or "{0} {1}" rel_path = str(Path(year) / fmt.format(month, album_name)) - link_folder: Path = self._links_root / rel_path + link_folder: Path + if shared: + link_folder = self._shared_albums_root / rel_path + else: + link_folder = self._albums_root / rel_path return link_folder def create_album_content_links(self): @@ -271,10 +288,13 @@ def create_album_content_links(self): # always re-create all album links - it is quite fast and a good way # to ensure consistency # especially now that we have --album-date-by-first-photo - if self._links_root.exists(): + if self._albums_root.exists(): log.debug("removing previous album links tree") - shutil.rmtree(self._links_root) - re_download = not self._links_root.exists() + shutil.rmtree(self._albums_root) + if self._shared_albums_root.exists(): + log.debug("removing previous shared album links tree") + shutil.rmtree(self._shared_albums_root) + re_download = not self._albums_root.exists() for ( path, @@ -284,6 +304,7 @@ def create_album_content_links(self): end_date_str, rid, created, + sharedAlbum, ) in self._db.get_album_files( album_invert=self._album_invert, download_again=re_download ): @@ -308,7 +329,9 @@ def create_album_content_links(self): full_file_name = self._root_folder / path / file_name - link_folder: Path = self.album_folder_name(album_name, start_date, end_date) + link_folder: Path = self.album_folder_name( + album_name, start_date, end_date, sharedAlbum + ) if self._no_album_sorting: link_filename = "{}".format(file_name) diff --git a/src/gphotos_sync/GooglePhotosDownload.py b/src/gphotos_sync/GooglePhotosDownload.py index 3f3ec5cd..50e52910 100644 --- a/src/gphotos_sync/GooglePhotosDownload.py +++ b/src/gphotos_sync/GooglePhotosDownload.py @@ -282,13 +282,16 @@ def do_download_file(self, base_url: str, media_item: DatabaseMedia): response.close() t_path.rename(local_full_path) create_date = Utils.safe_timestamp(media_item.create_date) - os.utime( - str(local_full_path), - ( - Utils.safe_timestamp(media_item.modify_date).timestamp(), - create_date.timestamp(), - ), - ) + try: + os.utime( + str(local_full_path), + ( + Utils.safe_timestamp(media_item.modify_date).timestamp(), + create_date.timestamp(), + ), + ) + except (PermissionError,): + log.debug("Could not set times for downloaded file") if _use_win_32: file_handle = win32file.CreateFile( str(local_full_path), @@ -301,7 +304,10 @@ def do_download_file(self, base_url: str, media_item: DatabaseMedia): ) win32file.SetFileTime(file_handle, *(create_date,) * 3) file_handle.close() - os.chmod(str(local_full_path), 0o666 & ~self.current_umask) + try: + os.chmod(str(local_full_path), 0o666 & ~self.current_umask) + except (PermissionError,): + log.debug("Could not set file access rights for downloaded file") except KeyboardInterrupt: log.debug("User cancelled download thread") raise diff --git a/src/gphotos_sync/LocalData.py b/src/gphotos_sync/LocalData.py index 0c01bab3..7695b83f 100644 --- a/src/gphotos_sync/LocalData.py +++ b/src/gphotos_sync/LocalData.py @@ -346,8 +346,8 @@ def get_album_files( query = """ SELECT SyncFiles.Path, SyncFiles.Filename, Albums.AlbumName, - Albums.StartDate, Albums.EndDate, Albums.RemoteId, SyncFiles.CreateDate - FROM AlbumFiles + Albums.StartDate, Albums.EndDate, Albums.RemoteId, SyncFiles.CreateDate, + Albums.IsSharedAlbum FROM AlbumFiles INNER JOIN SyncFiles ON AlbumFiles.DriveRec=SyncFiles.RemoteId INNER JOIN Albums ON AlbumFiles.AlbumRec=Albums.RemoteId WHERE Albums.RemoteId LIKE ? diff --git a/src/gphotos_sync/LocalFilesMedia.py b/src/gphotos_sync/LocalFilesMedia.py index d8af8e32..53666b5f 100644 --- a/src/gphotos_sync/LocalFilesMedia.py +++ b/src/gphotos_sync/LocalFilesMedia.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # coding: utf8 +import logging import re from datetime import datetime from json import loads @@ -10,10 +11,13 @@ from typing import Any, Dict, List, Optional, Union import exif +from plum.exceptions import UnpackError from . import Utils from .BaseMedia import BaseMedia +log = logging.getLogger(__name__) + JSONValue = Union[str, int, float, bool, None, Dict[str, Any], List[Any]] JSONType = Union[Dict[str, JSONValue], List[JSONValue]] @@ -138,6 +142,12 @@ def get_exif(self): self.got_meta = True except (IOError, AssertionError): self.got_meta = False + except (UnpackError, ValueError): + log.error( + "Problem reading exif data from file: %s", + str(self.relative_folder / self.filename), + ) + self.got_meta = False @property def uid(self) -> str: diff --git a/src/gphotos_sync/Settings.py b/src/gphotos_sync/Settings.py index 4a5238f4..2624f412 100644 --- a/src/gphotos_sync/Settings.py +++ b/src/gphotos_sync/Settings.py @@ -23,6 +23,7 @@ class Settings: use_flat_path: bool albums_path: Path + shared_albums_path: Path album_index: bool omit_album_date: bool album_invert: bool diff --git a/src/gphotos_sync/__main__.py b/src/gphotos_sync/__main__.py index f25cc729..96dece7b 100644 --- a/src/gphotos_sync/__main__.py +++ b/src/gphotos_sync/__main__.py @@ -174,6 +174,12 @@ def __init__(self): "Defaults to the 'albums' in the local download folders", default="albums", ) + parser.add_argument( + "--shared-albums-path", + help="Specify a folder for the shared albums " + "Defaults to the 'sharedAlbums' in the local download folders", + default="sharedAlbums", + ) parser.add_argument( "--photos-path", help="Specify a folder for the photo files. " @@ -373,6 +379,7 @@ def setup(self, args: Namespace, db_path: Path): archived=args.archived, photos_path=Path(args.photos_path), albums_path=Path(args.albums_path), + shared_albums_path=Path(args.shared_albums_path), use_flat_path=args.use_flat_path, max_retries=int(args.max_retries), max_threads=int(args.max_threads), diff --git a/src/gphotos_sync/authorize.py b/src/gphotos_sync/authorize.py index 1e26c20b..32577522 100644 --- a/src/gphotos_sync/authorize.py +++ b/src/gphotos_sync/authorize.py @@ -74,7 +74,10 @@ def load_token(self) -> Optional[str]: def save_token(self, token: str): with self.token_file.open("w") as stream: dump(token, stream) - self.token_file.chmod(0o600) + try: + self.token_file.chmod(0o600) + except (PermissionError,): + log.warning("Could not change permissions of the token file") def authorize(self): """Initiates OAuth2 authentication and authorization flow""" diff --git a/src/gphotos_sync/sql/gphotos_create.sql b/src/gphotos_sync/sql/gphotos_create.sql index 9f7337c8..8220802f 100644 --- a/src/gphotos_sync/sql/gphotos_create.sql +++ b/src/gphotos_sync/sql/gphotos_create.sql @@ -9,7 +9,8 @@ create table Albums StartDate INT, EndDate INT, SyncDate INT, - Downloaded INT DEFAULT 0 + Downloaded INT DEFAULT 0, + IsSharedAlbum BOOL ) ; DROP INDEX IF EXISTS Albums_RemoteId_uindex;