Merge pull request #68 from gilesknap/filesystem-search

Simplifying dependencies

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 matrix:
   include:
-    - python: 3.6.7
+    - python: 3.7.2
       dist: xenial
       sudo: true
 

Pipfile

@@ -5,22 +5,20 @@ name = "pypi"
 
 [packages]
 appdirs = "*"
-requests = "*"
-six = "*"
 requests_oauthlib = "*"
 # pin a beta version which fixes vunerability:
 # https://nvd.nist.gov/vuln/detail/CVE-2017-18342
 # todo won't accept >= for some reason so will need changing in future
 pyyaml = "==4.2b1"
-# pyyaml = "*"
-piexif = "*"
-python-magic = "*"
+exif = "*"
+selenium = "*"
 gphotos-sync = {path = "."}
-coverage = "*"
 
 [dev-packages]
 mock = "*"
 pytest = "*"
+coverage = "*"
+gphotos-sync = {path = "."}
 
 [requires]
-python_version = "3.6"
+python_version = "3.7"

README.rst

@@ -20,7 +20,7 @@ After doing a full sync you will have 2 directories off of the specified root:
   of the file (this matches the approach used in the official Google tool for Windows).
 
 * **albums** - contains a folder hierarchy representing the set of albums  and shared albums in your library. All
-  the files are symlinks to content in one of the other folders. The folder names  will be
+  the files are symlinks to content in the photos folder. The folder names  will be
   'albums/YYYY/MM Original Album Name'.
 
 In addition there will be further folders when using the --compare-folder option.  The option is used to make a
@@ -46,8 +46,8 @@ NOTES:
   extract metadata from video files and revert to relying on Google Photos meta data and file modified date (this is
   a much less reliable way to match video files, but the results should be OK if the backup folder
   was originally created using gphotos-sync).
-* If the library contains two separate items that have the same exif UID then this will result in seeing one
-  pair of duplicates, plus one of those duplicates will appear in the extra_files list.
+* If you have shared albums and have clicked 'add to library' on items from others' libraries then you will have two
+  copies of those items and they will show as duplicates too.
 
 Known Issues
 ------------

gphotos/BadIds.py

@@ -1,17 +1,11 @@
 from pathlib import Path
 from yaml import safe_load, safe_dump, YAMLError
-from typing import NamedTuple, Dict
+from typing import Dict
 import logging
 
 log = logging.getLogger(__name__)
 
 
-# sadly, I cant use this with safe_load / safe_dump
-class Item(NamedTuple):
-    path: str
-    product_url: str
-
-
 class BadIds:
     """ keeps a list of media items with ID in a YAML file. The YAML file
     allows a user to easily investigate their list of media items that have

gphotos/BaseMedia.py

@@ -119,3 +119,4 @@ def mime_type(self) -> str:
     @property
     def url(self) -> str:
         raise NotImplementedError
+

gphotos/DatabaseMedia.py

@@ -46,7 +46,8 @@ def __init__(self,
                  _description: str = None,
                  _date: datetime = None,
                  _create_date: datetime = None,
-                 _downloaded: bool = False):
+                 _downloaded: bool = False,
+                 _location: str = None):
         super(DatabaseMedia, self).__init__()
         # add all of the arguments as attributes on this object
         self.__dict__.update(locals())
@@ -59,8 +60,15 @@ def update_extra_meta(self, uid, create_date, size):
         self._create_date = create_date
         self._size = size
 
+    @property
+    def location(self) -> str:
+        """
+        image GPS information
+        """
+        return self._location
+
     # ----- BaseMedia base class override Properties below -----
-    @ property
+    @property
     def size(self) -> int:
         return self._size
 

gphotos/GoogleAlbumsSync.py

@@ -31,8 +31,10 @@ def __init__(self, api: RestClient, root_folder: Path, db: LocalData,
             db: local database for indexing
         """
         self._root_folder: Path = root_folder
-        self._links_root = self._root_folder / 'albums'
-        self._photos_root = self._root_folder / 'photos'
+        self._photos_folder = Path('photos')
+        self._albums_folder = Path('albums')
+        self._links_root = self._root_folder / self._albums_folder
+        self._photos_root = self._root_folder / self._photos_folder
         self._db: LocalData = db
         self._api: RestClient = api
         self.flush = flush
@@ -65,14 +67,30 @@ def fetch_album_contents(self, album_id: str,
                 self._db.put_album_file(album_id, media_item.id)
                 last_date = max(media_item.create_date, last_date)
                 first_date = min(media_item.create_date, first_date)
+
                 # this adds other users photos from shared albums
-                log.debug('Adding album media item %s %s %s',
-                          media_item.relative_path, media_item.filename,
-                          media_item.duplicate_number)
+                # Todo - This will cause two copies of a file to appear for
+                #  those shared items you have imported into your own library.
+                #  They will have different RemoteIds, one will point to your
+                #  library copy (you own) and one to the shared item in the
+                #  the folder. Currently with the meta data available it would
+                #  be impossible to eliminate these without eliminating other
+                #  cases where date and filename (TITLE) match
                 if add_media_items:
-                    media_item.set_path_by_date(self._photos_root)
+                    media_item.set_path_by_date(self._photos_folder)
+                    (num, row) = self._db.file_duplicate_no(
+                        str(media_item.filename),
+                        str(media_item.relative_folder),
+                        media_item.id)
+                    # we just learned if there were any duplicates in the db
+                    media_item.duplicate_number = num
+
+                    log.debug('Adding album media item %s %s %s',
+                              media_item.relative_path, media_item.filename,
+                              media_item.duplicate_number)
                     self._db.put_row(
                         GooglePhotosRow.from_media(media_item), False)
+
             next_page = items_json.get('nextPageToken')
             if next_page:
                 body = self.make_search_parameters(album_id=album_id,

gphotos/GooglePhotosDownload.py

@@ -50,9 +50,8 @@ def __init__(self, api: RestClient, root_folder: Path, db: LocalData):
 
         # attributes to be set after init
         # thus in theory one instance could so multiple indexes
-        # those with _ must be set through their set_ function
-        self._start_date: datetime = None
-        self._end_date: datetime = None
+        self.start_date: datetime = None
+        self.end_date: datetime = None
         self.retry_download: bool = False
         self.video_timeout: int = 2000
         self.image_timeout: int = 60
@@ -73,12 +72,6 @@ def __init__(self, api: RestClient, root_folder: Path, db: LocalData):
             'https://', HTTPAdapter(max_retries=retries,
                                     pool_maxsize=self.MAX_THREADS))
 
-    def set_start_date(self, val: str):
-        self._start_date = Utils.string_to_date(val)
-
-    def set_end_date(self, val: str):
-        self._end_date = Utils.string_to_date(val)
-
     def download_photo_media(self):
         """
         here we batch up our requests to get base url for downloading media.
@@ -101,8 +94,8 @@ def grouper(
             for media_items_block in grouper(
                     self._db.get_rows_by_search(
                         GooglePhotosRow,
-                        start_date=self._start_date,
-                        end_date=self._end_date,
+                        start_date=self.start_date,
+                        end_date=self.end_date,
                         skip_downloaded=not self.retry_download)):
                 batch = {}