Merge pull request #200 from gilesknap/download-after-cancel

Download after cancel

.gitattributes

@@ -0,0 +1 @@
+*/_version_git.py export-subst

.travis.yml

@@ -9,45 +9,45 @@ cache: pip
 
 matrix:
   include:
-  - os: osx
-    language: sh
-    python: '3.7'
-    before_install:
-    - pip3 install mock
-    script:
-    - python3 -m pytest --log-format='%(asctime)s.%(msecs)03d %(filename)-25s %(lineno)4d %(levelname)-8s %(message)s' --log-date-format='%Y-%m-%d %H:%M:%S' --log-level debug
-  - os: windows
-    language: sh
-    python: '3.7'
-    before_install:
-    - choco install python --version 3.7.4
-    - export PATH="/c/Python37:/c/Python37/Scripts:$PATH"
-    - python -m pip install --upgrade pip wheel
-    - python -m pip install mock
-  - os: linux
-    python: '3.6'
-  - os: linux
-    python: '3.7'
-    after_success:
-       - codecov --token=dbadb6ba-0032-4812-bcc2-67ef7bafdfed
-    deploy:
-      - provider: pypi
-        user: gilesknap
-        password:
-          secure: scyeqVy+HMOpQ1uTGwhFZRmmGy/IIg0EayrD8EpjJ8RZQpMBv/AkCzBTF8jtSPglggnT7JPD1TSfgOO4INw1NWFvgGVlOEMvhf4HkC49Mmf8m3z5ZIxUehgUfwhtrwUn9mWCNvdmr5IfaMnFsrX63kukYZOlKvbAZ8qAhX6MHYlzGW9NwKq9XLEx0jj10W2rmPB+yiAvhRixGk7Gx53c6BrBO/Acrlw9uMm+zJkyIv77Lzvp3kRRqthwxo9qDikdJgVUtmihinANVTUYGGeTGgMfKaknIlQjHoKuE6qgWZsqIW3lDHXCCSvXrr9+ukfE6+kpw9luqKzk3IzcD5QKxsJfA8OPLDgwdtYCyTZw5mhVajlEZAlVRQR3F6/87OqsFRLYU0iLxQwD4AGHSub9BYW9Momntd/vpEJVx165dpyStrRf9bgd0Gwk7shMlwzY2pqMcoPYEC4ewu8nAQeWH/BQXHD+8D855S1ib38HyWMkgp3m3F4KRO3CZpXDIIFMyWi1G+IrANEBXhxpd7lELBaaqldAAeYUSrD8qd9nJXul9vbYIorbsQRJK7J9rSkXLT8oai1dv1PL5PNidRHYZp+9FGeA68WHdGsLPd85cEf9M2xA5LXw4TEGctAFLOfr96bDtSlqSni94lbUlXVFQuX49Fc2JYwv5R+gFTupMVc=
-        on:
-          tags: true
-      - provider: script
-        script: bash ./build-docker.sh
-        on:
-          tags: true
+    - os: osx
+      language: sh
+      python: "3.7"
+      before_install:
+        - pip3 install mock
+      script:
+        - python3 -m pytest --log-format='%(asctime)s.%(msecs)03d %(filename)-25s %(lineno)4d %(levelname)-8s %(message)s' --log-date-format='%Y-%m-%d %H:%M:%S' --log-level debug
+    - os: windows
+      language: sh
+      python: "3.7"
+      before_install:
+        - choco install python --version 3.7.4
+        - export PATH="/c/Python37:/c/Python37/Scripts:$PATH"
+        - python -m pip install --upgrade pip wheel
+        - python -m pip install mock
+    - os: linux
+      python: "3.6"
+    - os: linux
+      python: "3.7"
+      after_success:
+        - codecov --token=dbadb6ba-0032-4812-bcc2-67ef7bafdfed
+      deploy:
+        - provider: pypi
+          user: gilesknap
+          password:
+            secure: "rHIvz5+DFNGNK4aHNT8UB2D5JeEDvIM1X5HkXbJ9jVgMTV2ODuUxo3WFkyU1ti6EKW+Ex8j2+kS/qOb0c3FoQfQ7gKm73VrjuYOsgIJ1qiB6zXDmyGOKTBzNuZdhA3jF6WLH9/G1qi63sqhw8Mqbz5SRN7v8m1V0twRmcroOnko49fLi2mQRxyyggFP7ukiM7M7X/aJgHtSCP+mBzbyMWwDvsl6BuiyDSYXCXnmFXXO3Jx07rkBgJ/GqNm0gGgnctlZxs5OrrLBxDXGuneR5MFFKu3RbASk8dKvG+h0RaHg7ZmdgH80wR2Z/tIW46OqUYAN+0wVMIWOQCw7a0kvV1zq1A0EmKNAyXdro2cY5GfLxNpvKT6vkTU58/aZANKrx0nEqwcstfLDWcxznGBtRAvjE0ZvcnDCxYjkTDg9DNFGC3aJFnFJ2vS+VZ2bdXe7ElVQnAGHJ+tpJlE/+U8IXUEdvu4jvcK3ZWJ0StqefOtIapP3ECSHIXo9/i/znUgaw5NejkdilBgg43Hn/3Nc7DM4Z1WxRMm+7R/+woFcv/EMdTIAzXbXhlqnJR8+Xv0W8/AjXH8R9IzAtiFol5B6hjtdiGDsqPbEFvWZTxuzR9ZwmZTxDtvvtyNYCVICIm+fB4F1ZL/D7ImxONmu1CI6gVm2DR/UFnjrhzx3slXoqBuM="
+          on:
+            tags: true
+        - provider: script
+          script: bash ./build-docker.sh
+          on:
+            tags: true
 
 install:
-- pip3 install .
-- pip3 install codecov
-- pip3 install pytest
-- pip3 install pytest-cov
+  - pip3 install .
+  - pip3 install codecov
+  - pip3 install pytest
+  - pip3 install pytest-cov
 script:
-- python -m pytest --cov=gphotos --log-format='%(asctime)s.%(msecs)03d %(filename)-25s
-  %(lineno)4d %(levelname)-8s %(message)s' --log-date-format='%Y-%m-%d %H:%M:%S' --log-level
-  debug
+  - python -m pytest --cov=gphotos --log-format='%(asctime)s.%(msecs)03d %(filename)-25s
+    %(lineno)4d %(levelname)-8s %(message)s' --log-date-format='%Y-%m-%d %H:%M:%S' --log-level
+    debug

.vscode/launch.json

@@ -1,36 +1,50 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Test gPhotos trace",
-            "type": "python",
-            "request": "launch",
-            "program": "${workspaceFolder}/gphotos-sync",
-            "args": [
-                "/tmp/test-gphotos",
-                "--secret",
-                "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json",
-                "--log-level",
-                "trace"
-            ],
-            "console": "integratedTerminal"
-        },
-        {
-            "name": "Test gPhotos debug",
-            "type": "python",
-            "request": "launch",
-            "program": "${workspaceFolder}/gphotos-sync",
-            "args": [
-                "/tmp/test-gphotos",
-                "--secret",
-                "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json",
-                "--log-level",
-                "debug"
-            ],
-            "console": "integratedTerminal"
-        }
-    ]
-}
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Test gPhotos trace",
+      "type": "python",
+      "request": "launch",
+      "program": "${workspaceFolder}/gphotos-sync",
+      "args": [
+        "/tmp/test-gphotos",
+        "--secret",
+        "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json",
+        "--log-level",
+        "trace"
+      ],
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Test gPhotos debug",
+      "type": "python",
+      "request": "launch",
+      "program": "${workspaceFolder}/gphotos-sync",
+      "args": [
+        "/tmp/test-gphotos",
+        "--secret",
+        "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json",
+        "--log-level",
+        "debug"
+      ],
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Test gPhotos in /tmp/gphotos pipenv",
+      "type": "python",
+      "request": "launch",
+      "program": "/home/giles/.local/share/virtualenvs/tmp-XVr6zr33/bin/gphotos-sync",
+      "args": [
+        "/tmp/test-gphotos",
+        "--secret",
+        "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json",
+        "--log-level",
+        "debug"
+      ],
+      "console": "integratedTerminal"
+    }
+  ]
+}

.vscode/settings.json

@@ -16,5 +16,6 @@
   ],
   "python.linting.flake8Enabled": true,
   "python.linting.pylintEnabled": false,
-  "python.linting.enabled": true
+  "python.linting.enabled": true,
+  "python.formatting.provider": "black"
 }

Pipfile

@@ -9,15 +9,14 @@ exif = "*"
 appdirs = "*"
 requests_oauthlib = "*"
 pyyaml = ">=4.2b1"
-gphotos-sync = {path = "."}
 pywin32 = {markers = "platform_system == 'Windows'",version = "*"}
 psutil = "*"
 
 [dev-packages]
+pytest = ">=5.0.1"
 mock = "*"
 coverage = "*"
 pytest-cov = "*"
-pytest = "==5.0.1"
 flake8 = "*"
 black = "*"
 rope = "*"

gphotos/Checks.py

@@ -4,7 +4,6 @@
 import re
 import shutil
 import subprocess
-from os.path import sep
 from pathlib import Path
 
 from psutil import disk_partitions
@@ -20,22 +19,29 @@ class Checks:
     fix_whitespace_ending = re.compile("([ \t]+$)")
     fix_unicode = re.compile(r"[^\x00-\x7F]")
 
+    # these filesystem types will have NTFS style filename restrictions
     windows_fs = ["fat", "ntfs", "9p"]
     WINDOWS_MAX_PATH = 248
 
-    def __init__(self, root_path: Path):
+    def __init__(self, root_path: Path, max_filename, ntfs):
         self.root_path: Path = root_path
         self._root_str: str = str(root_path).lower()
-        self.is_linux: bool = self._check_linux_filesystem()
+        if ntfs:
+            self.is_linux: bool = False
+        else:
+            self.is_linux: bool = self._check_linux_filesystem()
         self.is_symlink: bool = self._symlinks_supported()
         self.is_unicode: bool = self._unicode_filenames()
         self.is_case_sensitive: bool = self._check_case_sensitive()
         self.max_path: int = self._get_max_path_length()
-        self.max_filename: int = self._get_max_filename_length()
+        if max_filename > 0:
+            self.max_filename: int = max_filename
+        else:
+            self.max_filename: int = self._get_max_filename_length()
 
     def _check_linux_filesystem(self) -> bool:
         filesystem_type = ""
-        for part in disk_partitions():
+        for part in disk_partitions(True):
             if part.mountpoint == "/":
                 filesystem_type = part.fstype
                 continue
@@ -169,9 +175,9 @@ def valid_file_name(self, s: str) -> str:
 
 
 # ugly global stuff to avoid passing Checks object everywhere
-def do_check(root: Path):
+def do_check(root: Path, max_filename=0, ntfs=None):
     global root_folder
-    root_folder = Checks(root)
+    root_folder = Checks(root, max_filename, ntfs)
     return root_folder
 
 

gphotos/GooglePhotosDownload.py

@@ -153,6 +153,7 @@ def grouper(
                 self.files_download_failed,
                 self.files_download_skipped,
             )
+        return self.files_downloaded
 
     def download_batch(self, batch: Mapping[str, DatabaseMedia]):
         """ Downloads a batch of media items collected in download_photo_media.

gphotos/GooglePhotosIndex.py

@@ -136,7 +136,7 @@ def to_dict(self):
             log.debug("mediaItems.search with body:\n{}".format(body))
             return self._api.mediaItems.search.execute(body).json()
 
-    def index_photos_media(self) -> bool:
+    def index_photos_media(self) -> int:
         log.warning("Indexing Google Photos Files ...")
         total_listed = 0
 
@@ -226,7 +226,7 @@ def index_photos_media(self) -> bool:
             self._db.set_scan_date(last_date=self.latest_download)
 
         log.warning(f"indexed {self.files_indexed} items")
-        return self.files_indexed > 0
+        return self.files_indexed
 
     def get_extra_meta(self):
         count = 0