From bcc8efcc91557afb8c6ac3547404baf667de0d14 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 2 Feb 2020 23:32:54 +0000 Subject: [PATCH 01/14] scan all mounts to find filesystem root --- gphotos/Checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gphotos/Checks.py b/gphotos/Checks.py index 6e6ab86b..1de12dfa 100644 --- a/gphotos/Checks.py +++ b/gphotos/Checks.py @@ -4,7 +4,6 @@ import re import shutil import subprocess -from os.path import sep from pathlib import Path from psutil import disk_partitions @@ -20,6 +19,7 @@ class Checks: fix_whitespace_ending = re.compile("([ \t]+$)") fix_unicode = re.compile(r"[^\x00-\x7F]") + # these filesystem types will have NTFS style filename restrictions windows_fs = ["fat", "ntfs", "9p"] WINDOWS_MAX_PATH = 248 From 8f3f2b0c78f1d63b2822b37928dacabf328f75ea Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 2 Feb 2020 23:33:36 +0000 Subject: [PATCH 02/14] add _version_git version numbering --- .vscode/launch.json | 70 ++++++++++++------------ Pipfile | 2 +- gphotos/Main.py | 9 +-- gphotos/__init__.py | 6 ++ gphotos/_version_git.py | 82 ++++++++++++++++++++++++++++ setup.py | 18 +++++- test/test_credentials/.gphotos.token | 2 +- 7 files changed, 142 insertions(+), 47 deletions(-) create mode 100644 gphotos/_version_git.py diff --git a/.vscode/launch.json b/.vscode/launch.json index b98621ae..8e5ef574 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,36 +1,36 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Test gPhotos trace", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/gphotos-sync", - "args": [ - "/tmp/test-gphotos", - "--secret", - "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json", - "--log-level", - "trace" - ], - "console": "integratedTerminal" - }, - { - "name": "Test gPhotos debug", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/gphotos-sync", - "args": [ - "/tmp/test-gphotos", - "--secret", - "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json", - "--log-level", - "debug" - ], - "console": "integratedTerminal" - } - ] -} \ No newline at end of file + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Test gPhotos trace", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/gphotos-sync", + "args": [ + "/tmp/test-gphotos", + "--secret", + "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json", + "--log-level", + "trace" + ], + "console": "integratedTerminal" + }, + { + "name": "Test gPhotos debug", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/gphotos-sync", + "args": [ + "/tmp/test-gphotos", + "--secret", + "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json", + "--log-level", + "debug" + ], + "console": "integratedTerminal" + } + ] +} diff --git a/Pipfile b/Pipfile index dd442fbc..d75e98bc 100644 --- a/Pipfile +++ b/Pipfile @@ -14,10 +14,10 @@ pywin32 = {markers = "platform_system == 'Windows'",version = "*"} psutil = "*" [dev-packages] +pytest = ">=5.0.1" mock = "*" coverage = "*" pytest-cov = "*" -pytest = "==5.0.1" flake8 = "*" black = "*" rope = "*" diff --git a/gphotos/Main.py b/gphotos/Main.py index c7ffbb4d..487d2197 100644 --- a/gphotos/Main.py +++ b/gphotos/Main.py @@ -6,7 +6,6 @@ from datetime import datetime from pathlib import Path -import pkg_resources from appdirs import AppDirs from gphotos import Utils from gphotos.authorize import Authorize @@ -19,11 +18,7 @@ from gphotos.Logging import setup_logging from gphotos.restclient import RestClient from gphotos.Settings import Settings -from pkg_resources import DistributionNotFound - - -# todo add toms versioneer clone -__version__ = "2.11.beta-2" +from gphotos._version_git import __version__ if os.name != "nt": import fcntl @@ -51,8 +46,6 @@ def __init__(self): ) except TypeError: version_string = "(version not available)" - except DistributionNotFound: - version_string = "(version not available under unit tests)" parser = ArgumentParser( epilog=version_string, description="Google Photos download tool" diff --git a/gphotos/__init__.py b/gphotos/__init__.py index e69de29b..3c8edbef 100644 --- a/gphotos/__init__.py +++ b/gphotos/__init__.py @@ -0,0 +1,6 @@ +try: + # In a release there will be a static version file written by setup.py + from ._version_static import __version__ +except ImportError: + # Otherwise get the release number from git describe + from ._version_git import __version__ # noqa: F401 diff --git a/gphotos/_version_git.py b/gphotos/_version_git.py new file mode 100644 index 00000000..fd24eff4 --- /dev/null +++ b/gphotos/_version_git.py @@ -0,0 +1,82 @@ +# Compute a version number from a git repo or archive + +# This file is released into the public domain. Generated by: +# versiongit-0.1+31.a44853a (https://github.com/dls-controls/versiongit) +import os +import re +from subprocess import check_output, STDOUT + +# These will be filled in if git archive is run +GIT_ARCHIVE_REF_NAMES = "$Format:%D$" +GIT_ARCHIVE_HASH = "$Format:%h$" + + +def get_version_from_git(path=None): + """Try to parse version from git describe, fallback to git archive tags""" + if path is None: + # If no path to git repo, choose the directory this file is in + path = os.path.dirname(os.path.abspath(__file__)) + tag, plus, sha1, dirty, error = "0", "unknown", "error", "", None + if not GIT_ARCHIVE_HASH.startswith("$"): + # git archive has written a sha1 for us to use + sha1 = GIT_ARCHIVE_HASH + for ref_name in GIT_ARCHIVE_REF_NAMES.split(", "): + if ref_name.startswith("tag: "): + # On a git archive tag + tag, plus = ref_name[5:], "0" + else: + git_cmd = "git -C %s describe --tags --dirty --always --long" % path + # output is TAG-NUM-gHEX[-dirty] or HEX[-dirty] + try: + out = check_output(git_cmd.split(), stderr=STDOUT).decode().strip() + except Exception as e: + error = e + else: + if out.endswith("-dirty"): + out = out[:-6] + dirty = ".dirty" + if "-" in out: + # There is a tag, extract it and the other pieces + match = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", out) + tag, plus, sha1 = match.groups() + else: + # No tag, just sha1 + plus, sha1 = "untagged", out + # Replace dashes in tag for dots + tag = tag.replace("-", ".") + if plus != "0" or dirty: + # Not on a tag, add additional info + tag = "%(tag)s+%(plus)s.%(sha1)s%(dirty)s" % locals() + return tag, error, sha1 + + +__version__, git_error, git_sha1 = get_version_from_git() + + +def get_cmdclass(build_py=None, sdist=None): + """Create cmdclass dict to pass to setuptools.setup that will write a + _version_static.py file in our resultant sdist, wheel or egg""" + if build_py is None: + from setuptools.command.build_py import build_py + if sdist is None: + from setuptools.command.sdist import sdist + + def make_version_static(base_dir, pkg): + # Only place _version_static in the root directory of a module + pkg = pkg.split(".")[0] + with open(os.path.join(base_dir, pkg, "_version_static.py"), "w") as f: + f.write("__version__ = %r\n" % __version__) + + class BuildPy(build_py): + def run(self): + build_py.run(self) + for pkg in self.packages: + make_version_static(self.build_lib, pkg) + + class Sdist(sdist): + def make_release_tree(self, base_dir, files): + sdist.make_release_tree(self, base_dir, files) + for pkg in self.distribution.packages: + make_version_static(base_dir, pkg) + + return dict(build_py=BuildPy, sdist=Sdist) diff --git a/setup.py b/setup.py index 475f10ef..515ce70a 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,15 @@ #!/usr/bin/env python3 +import sys from setuptools import setup, find_packages, os +# Place the directory containing _version_git on the path +for path, _, filenames in os.walk(os.path.dirname(os.path.abspath(__file__))): + if "_version_git.py" in filenames: + sys.path.append(path) + break +from gphotos._version_git import get_cmdclass, __version__ # noqa E402 + module_name = "gphotos-sync" install_reqs = [ @@ -14,8 +22,13 @@ ] develop_reqs = [ - "pytest", + "pytest>=5.0.1", "mock", + "coverage", + "pytest", + "flake8", + "black", + "rope", ] if os.name == "nt": @@ -26,7 +39,8 @@ setup( name=module_name, - version="2.11.beta-2", + cmdclass=get_cmdclass(), + version=__version__, python_requires=">=3.6", license="MIT", platforms=["Linux", "Windows", "Mac"], diff --git a/test/test_credentials/.gphotos.token b/test/test_credentials/.gphotos.token index 1ff55302..f1249330 100644 --- a/test/test_credentials/.gphotos.token +++ b/test/test_credentials/.gphotos.token @@ -1 +1 @@ -{"access_token": "ya29.ImG7B_Kdq8yO7t0BQGYacrJ2e-E_wpialBAQqjkelBCacgDEa5Uh_LVYMaIf3E7tlJtRZMzurJsZb6Vy1xoaG0j7fZeFHyhD4pfMRNjPWonjH46iQjGsUfLsK03lxCmnKpLC", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.readonly", "https://www.googleapis.com/auth/photoslibrary.sharing"], "token_type": "Bearer", "expires_at": 1580342705.889631, "refresh_token": "1//03RScSJb3QlT_CgYIARAAGAMSNwF-L9IrGYvHJGx8gkwwgr_UpJW4e_HUM9-4jOvk_C2HI06R1GnXoH7yNt8vd5F4_7OVvrqkxl8"} \ No newline at end of file +{"access_token": "ya29.ImG8B95EQiBKbUHQULndtrp8IEQZpHhg6bJ4h_ZdMt2gXKI_80-nlIWCErIIZ18Le9ie-IvzjQwwCVFBGI2MvL6MEB-ml4rpUqj264l2O5a0sGH0uzZJdcsuA6iijJavoJx9", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.sharing", "https://www.googleapis.com/auth/photoslibrary.readonly"], "token_type": "Bearer", "expires_at": 1580689512.2334697, "refresh_token": "1//03RScSJb3QlT_CgYIARAAGAMSNwF-L9IrGYvHJGx8gkwwgr_UpJW4e_HUM9-4jOvk_C2HI06R1GnXoH7yNt8vd5F4_7OVvrqkxl8"} \ No newline at end of file From 658703323bcf39e106f363e2c71c875f9e86bde8 Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 3 Feb 2020 00:21:02 +0000 Subject: [PATCH 03/14] scan all mounts for filesystem type (retry) --- Pipfile.lock | 33 +++++++++++++-------------------- gphotos/Checks.py | 2 +- setup.py | 2 +- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 0e8e13fa..61d8b6b0 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7214e080984f9d0e4da9c886b4637b3cd92b2af39bb719d6b416892e40f4cf93" + "sha256": "445e0352ffa2e066e4f999e3d0b75cc9b5c7dba11e2e596b6d658ab7a85740e3" }, "pipfile-spec": 6, "requires": { @@ -165,13 +165,6 @@ "index": "pypi", "version": "==1.4.3" }, - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, "attrs": { "hashes": [ "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", @@ -249,11 +242,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", - "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" + "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302", + "sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b" ], "markers": "python_version < '3.8'", - "version": "==1.4.0" + "version": "==1.5.0" }, "mccabe": { "hashes": [ @@ -264,18 +257,18 @@ }, "mock": { "hashes": [ - "sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3", - "sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8" + "sha256:39d6bbbe86dfc7ef7fa21a1b34b67991f46548e7486af5799b79de0ab98c119e", + "sha256:b1bc607824a30bb4c3315a833d7817c1f01e0658c85710236d537e699b853493" ], "index": "pypi", - "version": "==3.0.5" + "version": "==4.0.0b1" }, "more-itertools": { "hashes": [ - "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39", - "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288" + "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", + "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" ], - "version": "==8.1.0" + "version": "==8.2.0" }, "packaging": { "hashes": [ @@ -328,11 +321,11 @@ }, "pytest": { "hashes": [ - "sha256:6ef6d06de77ce2961156013e9dff62f1b2688aa04d0dc244299fe7d67e09370d", - "sha256:a736fed91c12681a7b34617c8fcefe39ea04599ca72c608751c31d89579a3f77" + "sha256:0d5fe9189a148acc3c3eb2ac8e1ac0742cb7618c084f3d228baaec0c254b318d", + "sha256:ff615c761e25eb25df19edddc0b970302d2a9091fbce0e7213298d85fb61fef6" ], "index": "pypi", - "version": "==5.0.1" + "version": "==5.3.5" }, "pytest-cov": { "hashes": [ diff --git a/gphotos/Checks.py b/gphotos/Checks.py index 1de12dfa..52ffe624 100644 --- a/gphotos/Checks.py +++ b/gphotos/Checks.py @@ -35,7 +35,7 @@ def __init__(self, root_path: Path): def _check_linux_filesystem(self) -> bool: filesystem_type = "" - for part in disk_partitions(): + for part in disk_partitions(True): if part.mountpoint == "/": filesystem_type = part.fstype continue diff --git a/setup.py b/setup.py index 515ce70a..1cf9bb33 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ license="MIT", platforms=["Linux", "Windows", "Mac"], description="Google Photos and Albums backup tool", - packages=find_packages(exclude=("tests.*", "tests", "etc.*", "etc")), + packages=find_packages(), entry_points={"console_scripts": ["gphotos-sync = gphotos.Main:main"]}, long_description=long_description, install_requires=install_reqs, From 41701cdcf1882e4627543a2ac29e1ab6179266a5 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 3 Feb 2020 09:28:27 +0000 Subject: [PATCH 04/14] Import _version_git from absolute dir --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1cf9bb33..bfe520e0 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,8 @@ if "_version_git.py" in filenames: sys.path.append(path) break -from gphotos._version_git import get_cmdclass, __version__ # noqa E402 + +from _version_git import get_cmdclass, __version__ # noqa E402 module_name = "gphotos-sync" From 3a2fc740e84e929c51e8cc79c16c55f29ae018ae Mon Sep 17 00:00:00 2001 From: giles Date: Tue, 4 Feb 2020 16:20:52 +0000 Subject: [PATCH 05/14] tidy dependencies to fix _version_git --- Pipfile | 3 ++- Pipfile.lock | 5 ++++- setup.py | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index d75e98bc..91cf62de 100644 --- a/Pipfile +++ b/Pipfile @@ -9,9 +9,9 @@ exif = "*" appdirs = "*" requests_oauthlib = "*" pyyaml = ">=4.2b1" -gphotos-sync = {path = "."} pywin32 = {markers = "platform_system == 'Windows'",version = "*"} psutil = "*" +gphotos-sync = {path = "."} [dev-packages] pytest = ">=5.0.1" @@ -21,6 +21,7 @@ pytest-cov = "*" flake8 = "*" black = "*" rope = "*" +gphotos-sync = {path = "."} [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 61d8b6b0..acb77c1d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "445e0352ffa2e066e4f999e3d0b75cc9b5c7dba11e2e596b6d658ab7a85740e3" + "sha256": "d590231ff88fb7addd3fb31476e414582506d2023b04482f5e291311122ab7d9" }, "pipfile-spec": 6, "requires": { @@ -240,6 +240,9 @@ "index": "pypi", "version": "==3.7.9" }, + "gphotos-sync": { + "path": "." + }, "importlib-metadata": { "hashes": [ "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302", diff --git a/setup.py b/setup.py index bfe520e0..580308aa 100755 --- a/setup.py +++ b/setup.py @@ -11,6 +11,8 @@ from _version_git import get_cmdclass, __version__ # noqa E402 +print(f"installing version {__version__}") + module_name = "gphotos-sync" install_reqs = [ @@ -38,6 +40,8 @@ with open("README.rst", "rb") as f: long_description = f.read().decode("utf-8") +packages = [x for x in find_packages() if x.startswith("gphotos")] + setup( name=module_name, cmdclass=get_cmdclass(), @@ -46,7 +50,7 @@ license="MIT", platforms=["Linux", "Windows", "Mac"], description="Google Photos and Albums backup tool", - packages=find_packages(), + packages=packages, entry_points={"console_scripts": ["gphotos-sync = gphotos.Main:main"]}, long_description=long_description, install_requires=install_reqs, From 4d9b334bcec9ff4d075229922f7cd646b84b5113 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 9 Feb 2020 18:40:11 +0000 Subject: [PATCH 06/14] patch popen to avoid flickering console for background processes under Windows (issue (#196) --- .vscode/launch.json | 14 ++++++++++++++ gphotos/Main.py | 13 ++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 8e5ef574..528e7cd0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -31,6 +31,20 @@ "debug" ], "console": "integratedTerminal" + }, + { + "name": "Test gPhotos in /tmp/gphotos pipenv", + "type": "python", + "request": "launch", + "program": "/home/giles/.local/share/virtualenvs/tmp-XVr6zr33/bin/gphotos-sync", + "args": [ + "/tmp/test-gphotos", + "--secret", + "/home/giles/github//gphotos-sync/test/test_credentials/client_secret.json", + "--log-level", + "debug" + ], + "console": "integratedTerminal" } ] } diff --git a/gphotos/Main.py b/gphotos/Main.py index 487d2197..d5b8af10 100644 --- a/gphotos/Main.py +++ b/gphotos/Main.py @@ -20,7 +20,18 @@ from gphotos.Settings import Settings from gphotos._version_git import __version__ -if os.name != "nt": +if os.name == "nt": + import subprocess + orig_Popen = subprocess.Popen + + def Popen_patch(*args, **kargs): + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + kargs["startupinfo"] = startupinfo + return orig_Popen(*args, **kargs) + + subprocess.Popen = Popen_patch +else: import fcntl APP_NAME = "gphotos-sync" From 38b1237a0966acd1077529fb3796063489638789 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 9 Feb 2020 22:05:51 +0000 Subject: [PATCH 07/14] change popen patch to class --- gphotos/Main.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gphotos/Main.py b/gphotos/Main.py index d5b8af10..15ebfde1 100644 --- a/gphotos/Main.py +++ b/gphotos/Main.py @@ -18,18 +18,18 @@ from gphotos.Logging import setup_logging from gphotos.restclient import RestClient from gphotos.Settings import Settings -from gphotos._version_git import __version__ +from gphotos import __version__ if os.name == "nt": import subprocess orig_Popen = subprocess.Popen - def Popen_patch(*args, **kargs): - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - kargs["startupinfo"] = startupinfo - return orig_Popen(*args, **kargs) - + class Popen_patch(subprocess.Popen): + def __init__(self, *args, **kargs): + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + kargs["startupinfo"] = startupinfo + super().__init__(*args, **kargs) subprocess.Popen = Popen_patch else: import fcntl From d098b2c9305456c7a9cf9f787c1c4e13976b4269 Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 10 Feb 2020 17:25:21 +0000 Subject: [PATCH 08/14] fix __version_git versioining --- .gitattributes | 1 + gphotos/_version_git.py | 3 ++- test/test_credentials/.gphotos.token | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..41a13933 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*/_version_git.py export-subst diff --git a/gphotos/_version_git.py b/gphotos/_version_git.py index fd24eff4..c48c6854 100644 --- a/gphotos/_version_git.py +++ b/gphotos/_version_git.py @@ -1,7 +1,7 @@ # Compute a version number from a git repo or archive # This file is released into the public domain. Generated by: -# versiongit-0.1+31.a44853a (https://github.com/dls-controls/versiongit) +# versiongit-0.1+66.e5e165d (https://github.com/dls-controls/versiongit) import os import re from subprocess import check_output, STDOUT @@ -43,6 +43,7 @@ def get_version_from_git(path=None): # No tag, just sha1 plus, sha1 = "untagged", out # Replace dashes in tag for dots + # Remove this line when we stop supporting python 2.7 tag = tag.replace("-", ".") if plus != "0" or dirty: # Not on a tag, add additional info diff --git a/test/test_credentials/.gphotos.token b/test/test_credentials/.gphotos.token index f1249330..91a67051 100644 --- a/test/test_credentials/.gphotos.token +++ b/test/test_credentials/.gphotos.token @@ -1 +1 @@ -{"access_token": "ya29.ImG8B95EQiBKbUHQULndtrp8IEQZpHhg6bJ4h_ZdMt2gXKI_80-nlIWCErIIZ18Le9ie-IvzjQwwCVFBGI2MvL6MEB-ml4rpUqj264l2O5a0sGH0uzZJdcsuA6iijJavoJx9", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.sharing", "https://www.googleapis.com/auth/photoslibrary.readonly"], "token_type": "Bearer", "expires_at": 1580689512.2334697, "refresh_token": "1//03RScSJb3QlT_CgYIARAAGAMSNwF-L9IrGYvHJGx8gkwwgr_UpJW4e_HUM9-4jOvk_C2HI06R1GnXoH7yNt8vd5F4_7OVvrqkxl8"} \ No newline at end of file +{"access_token": "ya29.ImG9B5GZTuFrX38cxQVK0xe54xOMh_cNzrCM4JXqIafmXXKNDlbQr3hug2wiIBPs5rGizisEcWIpEI3lb4J2XCBebX3sYDextTcSjxlgcKoQHnVMjUC8Ld0QS95ZdV8QzAeL", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.sharing", "https://www.googleapis.com/auth/photoslibrary.readonly"], "token_type": "Bearer", "expires_at": 1581359304.3966794, "refresh_token": "1//03RScSJb3QlT_CgYIARAAGAMSNwF-L9IrGYvHJGx8gkwwgr_UpJW4e_HUM9-4jOvk_C2HI06R1GnXoH7yNt8vd5F4_7OVvrqkxl8"} \ No newline at end of file From bf470ae02a6a83dfb4f49c26f957e6c40afb89d3 Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 17 Feb 2020 06:33:57 +0000 Subject: [PATCH 09/14] update to new _version_git.py --- .vscode/settings.json | 3 ++- Pipfile | 2 -- Pipfile.lock | 17 +++++++---------- gphotos/_version_git.py | 17 ++++++++++++----- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 241cb2ff..2510c605 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,5 +16,6 @@ ], "python.linting.flake8Enabled": true, "python.linting.pylintEnabled": false, - "python.linting.enabled": true + "python.linting.enabled": true, + "python.formatting.provider": "black" } diff --git a/Pipfile b/Pipfile index 91cf62de..87a5219c 100644 --- a/Pipfile +++ b/Pipfile @@ -11,7 +11,6 @@ requests_oauthlib = "*" pyyaml = ">=4.2b1" pywin32 = {markers = "platform_system == 'Windows'",version = "*"} psutil = "*" -gphotos-sync = {path = "."} [dev-packages] pytest = ">=5.0.1" @@ -21,7 +20,6 @@ pytest-cov = "*" flake8 = "*" black = "*" rope = "*" -gphotos-sync = {path = "."} [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index acb77c1d..0bdfc684 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d590231ff88fb7addd3fb31476e414582506d2023b04482f5e291311122ab7d9" + "sha256": "445e0352ffa2e066e4f999e3d0b75cc9b5c7dba11e2e596b6d658ab7a85740e3" }, "pipfile-spec": 6, "requires": { @@ -240,9 +240,6 @@ "index": "pypi", "version": "==3.7.9" }, - "gphotos-sync": { - "path": "." - }, "importlib-metadata": { "hashes": [ "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302", @@ -260,11 +257,11 @@ }, "mock": { "hashes": [ - "sha256:39d6bbbe86dfc7ef7fa21a1b34b67991f46548e7486af5799b79de0ab98c119e", - "sha256:b1bc607824a30bb4c3315a833d7817c1f01e0658c85710236d537e699b853493" + "sha256:2a572b715f09dd2f0a583d8aeb5bb67d7ed7a8fd31d193cf1227a99c16a67bc3", + "sha256:5e48d216809f6f393987ed56920305d8f3c647e6ed35407c1ff2ecb88a9e1151" ], "index": "pypi", - "version": "==4.0.0b1" + "version": "==4.0.1" }, "more-itertools": { "hashes": [ @@ -422,10 +419,10 @@ }, "zipp": { "hashes": [ - "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28", - "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98" + "sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50", + "sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e" ], - "version": "==2.1.0" + "version": "==2.2.0" } } } diff --git a/gphotos/_version_git.py b/gphotos/_version_git.py index c48c6854..5b7943d9 100644 --- a/gphotos/_version_git.py +++ b/gphotos/_version_git.py @@ -1,7 +1,3 @@ -# Compute a version number from a git repo or archive - -# This file is released into the public domain. Generated by: -# versiongit-0.1+66.e5e165d (https://github.com/dls-controls/versiongit) import os import re from subprocess import check_output, STDOUT @@ -51,7 +47,13 @@ def get_version_from_git(path=None): return tag, error, sha1 -__version__, git_error, git_sha1 = get_version_from_git() +try: + # When installing from sdist there will already be a _version_static.py + # and during setup it will be on the python path (see setup.py: os.walk) + from _version_static import __version__ # type: ignore +except ImportError: + # Otherwise get the release number from git describe + __version__, git_error, git_sha1 = get_version_from_git() def get_cmdclass(build_py=None, sdist=None): @@ -67,6 +69,11 @@ def make_version_static(base_dir, pkg): pkg = pkg.split(".")[0] with open(os.path.join(base_dir, pkg, "_version_static.py"), "w") as f: f.write("__version__ = %r\n" % __version__) + static_version = os.path.join(base_dir, pkg, "_version_static.py") + # when installing from sdist the _version_static.py will already exist + if not os.path.exists(static_version): + with open(static_version, "w") as f: + f.write("__version__ = %r\n" % __version__) class BuildPy(build_py): def run(self): From a056a55aedb4d7ce0cc136ebc7956327f5bc3419 Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 17 Feb 2020 08:05:14 +0000 Subject: [PATCH 10/14] always try download - to cope with a previously cancelled download --- gphotos/GooglePhotosDownload.py | 1 + gphotos/GooglePhotosIndex.py | 4 +-- gphotos/Main.py | 46 ++++++++++++++++++---------- test/test_credentials/.gphotos.token | 2 +- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/gphotos/GooglePhotosDownload.py b/gphotos/GooglePhotosDownload.py index b8540dc3..2ba957ce 100644 --- a/gphotos/GooglePhotosDownload.py +++ b/gphotos/GooglePhotosDownload.py @@ -153,6 +153,7 @@ def grouper( self.files_download_failed, self.files_download_skipped, ) + return self.files_downloaded def download_batch(self, batch: Mapping[str, DatabaseMedia]): """ Downloads a batch of media items collected in download_photo_media. diff --git a/gphotos/GooglePhotosIndex.py b/gphotos/GooglePhotosIndex.py index ee6a5f21..0a7a1608 100644 --- a/gphotos/GooglePhotosIndex.py +++ b/gphotos/GooglePhotosIndex.py @@ -136,7 +136,7 @@ def to_dict(self): log.debug("mediaItems.search with body:\n{}".format(body)) return self._api.mediaItems.search.execute(body).json() - def index_photos_media(self) -> bool: + def index_photos_media(self) -> int: log.warning("Indexing Google Photos Files ...") total_listed = 0 @@ -226,7 +226,7 @@ def index_photos_media(self) -> bool: self._db.set_scan_date(last_date=self.latest_download) log.warning(f"indexed {self.files_indexed} items") - return self.files_indexed > 0 + return self.files_indexed def get_extra_meta(self): count = 0 diff --git a/gphotos/Main.py b/gphotos/Main.py index 15ebfde1..ffdbf3b3 100644 --- a/gphotos/Main.py +++ b/gphotos/Main.py @@ -22,6 +22,7 @@ if os.name == "nt": import subprocess + orig_Popen = subprocess.Popen class Popen_patch(subprocess.Popen): @@ -30,6 +31,7 @@ def __init__(self, *args, **kargs): startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW kargs["startupinfo"] = startupinfo super().__init__(*args, **kargs) + subprocess.Popen = Popen_patch else: import fcntl @@ -312,29 +314,38 @@ def setup(self, args: Namespace, db_path: Path): ) def do_sync(self, args: Namespace): - new_files = True + files_downloaded = 0 with self.data_store: if not args.skip_index: if not args.skip_files and not args.album: - new_files = self.google_photos_idx.index_photos_media() - # if there are no new files and no arguments that specify specific - # scan requirements, then we have done all we need to do + self.google_photos_idx.index_photos_media() + + if not args.index_only: + if not args.skip_files: + files_downloaded = self.google_photos_down.download_photo_media() + if ( - new_files - or args.rescan - or args.retry_download - or args.start_date - or args.album - ): - if not args.skip_albums and not args.skip_index: - self.google_albums_sync.index_album_media() + not args.skip_albums + and not args.skip_index + and (files_downloaded > 0 or args.skip_files or args.rescan) + ) or args.album is not None: + self.google_albums_sync.index_album_media() + # run download again to pick up files indexed in albums only if not args.index_only: if not args.skip_files: - self.google_photos_down.download_photo_media() - if not args.skip_albums: - self.google_albums_sync.create_album_content_links() - if args.do_delete: - self.google_photos_idx.check_for_removed() + files_downloaded = ( + self.google_photos_down.download_photo_media() + ) + + if not args.index_only: + if ( + not args.skip_albums + and (files_downloaded > 0 or args.skip_files or args.rescan) + or args.album is not None + ): + self.google_albums_sync.create_album_content_links() + if args.do_delete: + self.google_photos_idx.check_for_removed() if args.compare_folder: if not args.skip_index: @@ -372,6 +383,7 @@ def main(self, test_args: dict = None): root_folder.mkdir(parents=True, mode=0o700) setup_logging(args.log_level, args.logfile, root_folder) + log.warning(f"gphotos-sync {__version__} {start_time}") args = self.fs_checks(root_folder, args) diff --git a/test/test_credentials/.gphotos.token b/test/test_credentials/.gphotos.token index 91a67051..8ab4033e 100644 --- a/test/test_credentials/.gphotos.token +++ b/test/test_credentials/.gphotos.token @@ -1 +1 @@ -{"access_token": "ya29.ImG9B5GZTuFrX38cxQVK0xe54xOMh_cNzrCM4JXqIafmXXKNDlbQr3hug2wiIBPs5rGizisEcWIpEI3lb4J2XCBebX3sYDextTcSjxlgcKoQHnVMjUC8Ld0QS95ZdV8QzAeL", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.sharing", "https://www.googleapis.com/auth/photoslibrary.readonly"], "token_type": "Bearer", "expires_at": 1581359304.3966794, "refresh_token": "1//03RScSJb3QlT_CgYIARAAGAMSNwF-L9IrGYvHJGx8gkwwgr_UpJW4e_HUM9-4jOvk_C2HI06R1GnXoH7yNt8vd5F4_7OVvrqkxl8"} \ No newline at end of file +{"access_token": "ya29.ImG9B4byH8b2x8RAKFa-H0UA3aIGgOHS-2ptVZDUXuCWAeq0B6x2zTr_XPQZTv4ZEFrXPOjHroMA8XXY3Vv1Sc-fv8hEh2dhCaPou3d-eHjd32HIIpKTVE5pV8TtbLAD1YlR", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.readonly", "https://www.googleapis.com/auth/photoslibrary.sharing"], "token_type": "Bearer", "expires_at": 1581929942.2690644, "refresh_token": "1//03RScSJb3QlT_CgYIARAAGAMSNwF-L9IrGYvHJGx8gkwwgr_UpJW4e_HUM9-4jOvk_C2HI06R1GnXoH7yNt8vd5F4_7OVvrqkxl8"} \ No newline at end of file From 089404fc39fcdfe787b825221dd4d17faab27fd8 Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 17 Feb 2020 08:15:28 +0000 Subject: [PATCH 11/14] pypi password --- .travis.yml | 78 ++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a88540a..59e8c5bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,45 +9,45 @@ cache: pip matrix: include: - - os: osx - language: sh - python: '3.7' - before_install: - - pip3 install mock - script: - - python3 -m pytest --log-format='%(asctime)s.%(msecs)03d %(filename)-25s %(lineno)4d %(levelname)-8s %(message)s' --log-date-format='%Y-%m-%d %H:%M:%S' --log-level debug - - os: windows - language: sh - python: '3.7' - before_install: - - choco install python --version 3.7.4 - - export PATH="/c/Python37:/c/Python37/Scripts:$PATH" - - python -m pip install --upgrade pip wheel - - python -m pip install mock - - os: linux - python: '3.6' - - os: linux - python: '3.7' - after_success: - - codecov --token=dbadb6ba-0032-4812-bcc2-67ef7bafdfed - deploy: - - provider: pypi - user: gilesknap - password: - secure: scyeqVy+HMOpQ1uTGwhFZRmmGy/IIg0EayrD8EpjJ8RZQpMBv/AkCzBTF8jtSPglggnT7JPD1TSfgOO4INw1NWFvgGVlOEMvhf4HkC49Mmf8m3z5ZIxUehgUfwhtrwUn9mWCNvdmr5IfaMnFsrX63kukYZOlKvbAZ8qAhX6MHYlzGW9NwKq9XLEx0jj10W2rmPB+yiAvhRixGk7Gx53c6BrBO/Acrlw9uMm+zJkyIv77Lzvp3kRRqthwxo9qDikdJgVUtmihinANVTUYGGeTGgMfKaknIlQjHoKuE6qgWZsqIW3lDHXCCSvXrr9+ukfE6+kpw9luqKzk3IzcD5QKxsJfA8OPLDgwdtYCyTZw5mhVajlEZAlVRQR3F6/87OqsFRLYU0iLxQwD4AGHSub9BYW9Momntd/vpEJVx165dpyStrRf9bgd0Gwk7shMlwzY2pqMcoPYEC4ewu8nAQeWH/BQXHD+8D855S1ib38HyWMkgp3m3F4KRO3CZpXDIIFMyWi1G+IrANEBXhxpd7lELBaaqldAAeYUSrD8qd9nJXul9vbYIorbsQRJK7J9rSkXLT8oai1dv1PL5PNidRHYZp+9FGeA68WHdGsLPd85cEf9M2xA5LXw4TEGctAFLOfr96bDtSlqSni94lbUlXVFQuX49Fc2JYwv5R+gFTupMVc= - on: - tags: true - - provider: script - script: bash ./build-docker.sh - on: - tags: true + - os: osx + language: sh + python: "3.7" + before_install: + - pip3 install mock + script: + - python3 -m pytest --log-format='%(asctime)s.%(msecs)03d %(filename)-25s %(lineno)4d %(levelname)-8s %(message)s' --log-date-format='%Y-%m-%d %H:%M:%S' --log-level debug + - os: windows + language: sh + python: "3.7" + before_install: + - choco install python --version 3.7.4 + - export PATH="/c/Python37:/c/Python37/Scripts:$PATH" + - python -m pip install --upgrade pip wheel + - python -m pip install mock + - os: linux + python: "3.6" + - os: linux + python: "3.7" + after_success: + - codecov --token=dbadb6ba-0032-4812-bcc2-67ef7bafdfed + deploy: + - provider: pypi + user: gilesknap + password: + secure: "rHIvz5+DFNGNK4aHNT8UB2D5JeEDvIM1X5HkXbJ9jVgMTV2ODuUxo3WFkyU1ti6EKW+Ex8j2+kS/qOb0c3FoQfQ7gKm73VrjuYOsgIJ1qiB6zXDmyGOKTBzNuZdhA3jF6WLH9/G1qi63sqhw8Mqbz5SRN7v8m1V0twRmcroOnko49fLi2mQRxyyggFP7ukiM7M7X/aJgHtSCP+mBzbyMWwDvsl6BuiyDSYXCXnmFXXO3Jx07rkBgJ/GqNm0gGgnctlZxs5OrrLBxDXGuneR5MFFKu3RbASk8dKvG+h0RaHg7ZmdgH80wR2Z/tIW46OqUYAN+0wVMIWOQCw7a0kvV1zq1A0EmKNAyXdro2cY5GfLxNpvKT6vkTU58/aZANKrx0nEqwcstfLDWcxznGBtRAvjE0ZvcnDCxYjkTDg9DNFGC3aJFnFJ2vS+VZ2bdXe7ElVQnAGHJ+tpJlE/+U8IXUEdvu4jvcK3ZWJ0StqefOtIapP3ECSHIXo9/i/znUgaw5NejkdilBgg43Hn/3Nc7DM4Z1WxRMm+7R/+woFcv/EMdTIAzXbXhlqnJR8+Xv0W8/AjXH8R9IzAtiFol5B6hjtdiGDsqPbEFvWZTxuzR9ZwmZTxDtvvtyNYCVICIm+fB4F1ZL/D7ImxONmu1CI6gVm2DR/UFnjrhzx3slXoqBuM=" + on: + tags: true + - provider: script + script: bash ./build-docker.sh + on: + tags: true install: -- pip3 install . -- pip3 install codecov -- pip3 install pytest -- pip3 install pytest-cov + - pip3 install . + - pip3 install codecov + - pip3 install pytest + - pip3 install pytest-cov script: -- python -m pytest --cov=gphotos --log-format='%(asctime)s.%(msecs)03d %(filename)-25s - %(lineno)4d %(levelname)-8s %(message)s' --log-date-format='%Y-%m-%d %H:%M:%S' --log-level - debug + - python -m pytest --cov=gphotos --log-format='%(asctime)s.%(msecs)03d %(filename)-25s + %(lineno)4d %(levelname)-8s %(message)s' --log-date-format='%Y-%m-%d %H:%M:%S' --log-level + debug From 9595049d06c65c32dc334123bf2205c79ff13eec Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 17 Feb 2020 18:59:53 +0000 Subject: [PATCH 12/14] add --ntfs and --max-filename --- gphotos/Checks.py | 16 +++++++++++----- gphotos/Main.py | 14 +++++++++++++- test/test-units/test_units.py | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/gphotos/Checks.py b/gphotos/Checks.py index 52ffe624..65f28960 100644 --- a/gphotos/Checks.py +++ b/gphotos/Checks.py @@ -23,15 +23,21 @@ class Checks: windows_fs = ["fat", "ntfs", "9p"] WINDOWS_MAX_PATH = 248 - def __init__(self, root_path: Path): + def __init__(self, root_path: Path, max_filename, ntfs): self.root_path: Path = root_path self._root_str: str = str(root_path).lower() - self.is_linux: bool = self._check_linux_filesystem() + if ntfs: + self.is_linux: bool = False + else: + self.is_linux: bool = self._check_linux_filesystem() self.is_symlink: bool = self._symlinks_supported() self.is_unicode: bool = self._unicode_filenames() self.is_case_sensitive: bool = self._check_case_sensitive() self.max_path: int = self._get_max_path_length() - self.max_filename: int = self._get_max_filename_length() + if max_filename > 0: + self.max_filename: int = max_filename + else: + self.max_filename: int = self._get_max_filename_length() def _check_linux_filesystem(self) -> bool: filesystem_type = "" @@ -169,9 +175,9 @@ def valid_file_name(self, s: str) -> str: # ugly global stuff to avoid passing Checks object everywhere -def do_check(root: Path): +def do_check(root: Path, max_filename=0, ntfs=None): global root_folder - root_folder = Checks(root) + root_folder = Checks(root, max_filename, ntfs) return root_folder diff --git a/gphotos/Main.py b/gphotos/Main.py index ffdbf3b3..009c717d 100644 --- a/gphotos/Main.py +++ b/gphotos/Main.py @@ -238,6 +238,18 @@ def __init__(self): action="store_true", help="show progress of indexing and downloading in warning log", ) + parser.add_argument( + "--max-filename", + help="Set the maxiumum filename length for target filesystem." + "This overrides the automatic detection.", + default=0, + ) + parser.add_argument( + "--ntfs", + action="store_true", + help="Declare that the target filesystem is ntfs (or ntfs like)." + "This overrides the automatic detection.", + ) parser.add_help = True def setup(self, args: Namespace, db_path: Path): @@ -360,7 +372,7 @@ def start(self, args: Namespace): def fs_checks(root_folder: Path, args: dict): Utils.minimum_date(root_folder) # store the root folder filesystem checks globally for all to inspect - do_check(root_folder) + do_check(root_folder, int(args.max_filename), bool(args.ntfs)) # check if symlinks are supported if not get_check().is_symlink: diff --git a/test/test-units/test_units.py b/test/test-units/test_units.py index c09a9baa..1220baee 100644 --- a/test/test-units/test_units.py +++ b/test/test-units/test_units.py @@ -4,6 +4,7 @@ from os import name as os_name from pathlib import Path from unittest import TestCase +import test.test_setup as ts import gphotos.authorize as auth from gphotos.Checks import do_check, get_check @@ -138,3 +139,20 @@ def test_os_filesystem(self): else: do_check(test_data) self.assertTrue(get_check().is_linux) + + def test_fs_overrides(self): + s = ts.SetupDbAndCredentials() + args = ["--ntfs", "--max-filename", "30"] + s.test_setup("test_fs_overrides", args=args, trash_db=True, trash_files=True) + s.gp.fs_checks(s.root, s.parsed_args) + self.assertFalse(get_check().is_linux) + self.assertEquals(get_check().max_filename, 30) + + if os_name != "nt": + args = [] + s.test_setup( + "test_fs_overrides", args=args, trash_db=True, trash_files=True + ) + s.gp.fs_checks(s.root, s.parsed_args) + self.assertTrue(get_check().is_linux) + self.assertEquals(get_check().max_filename, 255) From 9cd624297b95eb4a8a00200fa7cde93df3add431 Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 17 Feb 2020 19:13:01 +0000 Subject: [PATCH 13/14] harden test test_google_albums_sync --- test/test-units/test_errors.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test-units/test_errors.py b/test/test-units/test_errors.py index 7823b6c3..a188ee34 100644 --- a/test/test-units/test_errors.py +++ b/test/test-units/test_errors.py @@ -183,11 +183,11 @@ def download_faves(self, expected=4, no_response=False, trash=True): ) s.gp.start(s.parsed_args) - db = LocalData(s.root) - # Total of 1 out of media items - db.cur.execute("SELECT COUNT() FROM SyncFiles") - count = db.cur.fetchone() - self.assertEqual(expected, count[0]) + with LocalData(s.root) as db: + # Total of 1 out of media items + db.cur.execute("SELECT COUNT() FROM SyncFiles") + count = db.cur.fetchone() + self.assertEqual(expected, count[0]) class DummyResponse: @staticmethod From 1603dbc1f8ca55b0a2904956837c1671b8684d9b Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 17 Feb 2020 20:58:19 +0000 Subject: [PATCH 14/14] explicit delete of .sqlite.previous on --flush-index #199 --- gphotos/LocalData.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/gphotos/LocalData.py b/gphotos/LocalData.py index d636679b..a64ffecd 100644 --- a/gphotos/LocalData.py +++ b/gphotos/LocalData.py @@ -1,10 +1,10 @@ -#!/usr/bin/env python3 -# coding: utf8 -from pathlib import Path +import logging import platform import sqlite3 as lite -from sqlite3.dbapi2 import Connection, Cursor from datetime import datetime +from os import unlink +from pathlib import Path +from sqlite3.dbapi2 import Connection, Cursor from typing import Iterator, Type # todo this module could be tidied quite a bit @@ -12,13 +12,11 @@ # also the generic functions seem a bit ugly and could do with rework import gphotos.Queries as Queries from gphotos import Utils +from gphotos.DatabaseMedia import DatabaseMedia +from gphotos.DbRow import DbRow from gphotos.GoogleAlbumsRow import GoogleAlbumsRow -from gphotos.LocalFilesRow import LocalFilesRow from gphotos.GooglePhotosRow import GooglePhotosRow -from gphotos.DbRow import DbRow -from gphotos.DatabaseMedia import DatabaseMedia - -import logging +from gphotos.LocalFilesRow import LocalFilesRow log = logging.getLogger(__name__) @@ -44,7 +42,7 @@ def __init__(self, root_folder: Path, flush_index: bool = False): clean_db = True elif flush_index: clean_db = True - self.db_file.rename(self.db_file.parent / (self.db_file.name + ".previous")) + self.backup_sql_file() self.con: Connection = lite.connect(str(self.db_file), check_same_thread=False) self.con.row_factory = lite.Row @@ -59,6 +57,12 @@ def __init__(self, root_folder: Path, flush_index: bool = False): def __enter__(self): return self + def backup_sql_file(self): + backup = self.db_file.parent / (self.db_file.name + ".previous") + if backup.exists(): + backup.unlink() + self.db_file.rename(backup) + def __exit__(self, exc_type, exc_val, exc_tb): """ Always clean up and close the connection when this object is destroyed. """ @@ -84,7 +88,7 @@ def check_schema_version(self): ) self.con.commit() self.con.close() - self.db_file.rename(self.db_file.parent / (self.db_file.name + ".previous")) + self.backup_sql_file() self.con = lite.connect(str(self.db_file)) self.con.row_factory = lite.Row self.cur = self.con.cursor()