diff --git a/zrsync/zrsync.py b/zrsync/zrsync.py index eb2a412..c55949c 100755 --- a/zrsync/zrsync.py +++ b/zrsync/zrsync.py @@ -61,9 +61,9 @@ def __init__(self, name, source_dir, server_addr, server_port, target_dir, ssh, self.ssh = ssh self.no_delete = no_delete self.initial_only = initial_only - self._error_tmp = dict() - self._move_tmp = dict() - self._previous_stat = dict() + self._error_tmp = {} + self._move_tmp = {} + self._previous_stat = {} def do_connect(self): self._socket = self._context.socket(zmq.REQ) @@ -162,17 +162,17 @@ def watch_dir(self, base_dir): self.logger.debug("HANDLE_EVENTS: %r", HANDLE_EVENTS) mask = reduce(lambda x, y: x | y, [getattr(inotify.constants, ev) for ev in HANDLE_EVENTS], 0) for handler in self.logger.handlers: - if not handler in inotify.adapters._LOGGER.handlers: + if handler not in inotify.adapters._LOGGER.handlers: inotify.adapters._LOGGER.addHandler(handler) inotify.adapters._LOGGER.setLevel(self.logger.level) notifyier = inotify.adapters.InotifyTree(base_dir, mask=mask) for event in notifyier.event_gen(): if event is not None: header, type_names, watch_path, filename = event - if not any([tn in HANDLE_EVENTS for tn in type_names]): + if all(tn not in HANDLE_EVENTS for tn in type_names): self.logger.debug("ignoring event type_names=%r", type_names) continue - if any([re.search(regex, filename) for regex in FILE_IGNORE_REGEX]): + if any(re.search(regex, filename) for regex in FILE_IGNORE_REGEX): self.logger.debug("ignoring filename=%r", filename) continue self.logger.debug("WD=(%d) MASK=(%d) COOKIE=(%d) LEN=(%d) MASK->NAMES=%s WATCH-PATH=[%s] FILENAME=[%s]", @@ -208,8 +208,8 @@ def handle_fs_event(self, action, path, filename, cookie): self.logger.debug("File %r vanished, ignoring.", local_filename) return True - if "IN_ISDIR" in action: - if "IN_CREATE" in action: + if "IN_CREATE" in action: + if "IN_ISDIR" in action: # This should be just a newly created, _empty_ directory, but # inotify fails to add watches for newly created directories # fast enough to catch for example 'mkdir -p'. @@ -220,73 +220,18 @@ def handle_fs_event(self, action, path, filename, cookie): return False else: request["cmd"] = "mkdir" - elif "IN_DELETE" in action: - if self.no_delete: - self.logger.debug("Not deleting %r: no_delete is set.", local_filename) - return True - else: - request["cmd"] = "rmdir" - else: - if "IN_CREATE" in action: + else: self._previous_stat = request.copy() if request["st_size"] == 0: request["cmd"] = "truncate" else: request.add_full(local_filename).add_parent_time(local_filename).add_hash(local_filename) - elif "IN_ATTRIB" in action: - # prevent unnecessary request by detecting IN_ATTRIB following - # IN_CREATE and IN_CLOSE_WRITE without stat change - if request.has_same_stat(self._previous_stat): - return True - else: - self.logger.debug("***** IN_ATTRIB in action: request.has_same_stat(self._previous_stat) = False") - self.logger.debug("***** request.to_dict()=%r", request.to_dict()) - self.logger.debug("***** self._previous_stat=%r", self._previous_stat) - self._previous_stat = request.copy() - request["cmd"] = "touch" - elif "IN_CLOSE_WRITE" in action: - # prevent unnecessary request in case of an empty file - if request.has_same_stat(self._previous_stat): - self.logger.debug("***** IN_CLOSE_WRITE in action: request.has_same_stat(self._previous_stat) = True") - return True - if request["st_size"] == 0: - if request.has_same_stat(self._previous_stat): - self.logger.debug("***** IN_CLOSE_WRITE in action: [st_size] = 0 AND request.has_same_stat(self._previous_stat) = True") - return True - else: - self.logger.debug("***** IN_CLOSE_WRITE in action: [st_size] = 0 AND request.has_same_stat(self._previous_stat) = False") - self.logger.debug("***** request.to_dict()=%r", request.to_dict()) - self.logger.debug("***** self._previous_stat=%r", self._previous_stat) - self._previous_stat = request.copy() - request["cmd"] = "truncate" - elif request["st_blocks"] < MIN_BLOCK_DIFF_SIZE: - request.add_full(local_filename).request.add_parent_time(local_filename).add_hash(local_filename) - else: - request["cmd"] = "sig" - elif "IN_DELETE" in action: - if self.no_delete: - self.logger.debug("Not deleting %r: no_delete is set.", local_filename) - return True - else: - request["cmd"] = "rm" - elif "IN_MOVED_FROM" in action: - if request.cmd == "rm": - # file was moved away, noticed by request.add_stat() above - pass - else: - self._move_tmp[cookie] = remote_filename - return True - elif "IN_MOVED_TO" in action: - mv_src = self._move_tmp.pop("cookie", None) - if mv_src: - request.update({ - "cmd": "mv", - "from": mv_src - }) - else: - # moved in from outside of inotify-observed tree - request.add_full(local_filename).add_parent_time(local_filename).add_hash(local_filename) - + elif "IN_DELETE" in action: + if self.no_delete: + self.logger.debug("Not deleting %r: no_delete is set.", local_filename) + return True + else: + request["cmd"] = "rmdir" if request.cmd == "none": raise RuntimeError("request.cmd was not set with action='{}' path='{}' filename='{}' cookie='{}'.".format(action, path, filename, cookie)) @@ -454,7 +399,7 @@ def main(args): log_level=args["log_level"] ) jobs = zip(args[""], args[""]) - processes = list() + processes = [] for src, trg in jobs: p_name = "zrsync client {}".format(src) kwargs.update(dict( diff --git a/zrsync/zrsyncd.py b/zrsync/zrsyncd.py index 2106d37..6650f83 100755 --- a/zrsync/zrsyncd.py +++ b/zrsync/zrsyncd.py @@ -88,10 +88,7 @@ def __repr__(self): pprint.pformat(self.shorten_data(self))) def __setitem__(self, key, value): - if key == "data": - self._request[key] = self.encode_binary(value) - else: - self._request[key] = value + self._request[key] = self.encode_binary(value) if key == "data" else value def add_diff(self, signature, filename=None): if not filename: @@ -198,17 +195,31 @@ def encode_binary(data): def has_same_stat(self, other): if not other: return False - for key in ["st_mode", "st_uid", "st_gid", "st_size", "st_atime", "st_mtime", "st_blocks"]: - if self[key] != other[key]: - return False - return True + return all( + self[key] == other[key] + for key in [ + "st_mode", + "st_uid", + "st_gid", + "st_size", + "st_atime", + "st_mtime", + "st_blocks", + ] + ) def is_valid(self): if not all([isinstance(self.status, int) and (self.status == 0 or self.get("reason")), isinstance(self.cmd, str), isinstance(self.name, str)]): return False if self.cmd == "patch": - if self.status == 0 and not (self.get("result") == "done" or all([self.get(x) for x in ["data", "parent_atime", "parent_mtime"]])): + if ( + self.status == 0 + and self.get("result") != "done" + and not all( + self.get(x) for x in ["data", "parent_atime", "parent_mtime"] + ) + ): return False elif self.cmd == "sync_dir": if not (isinstance(self.get("tree_diff"), list) or @@ -344,10 +355,7 @@ def receive(self): request.get("result") or request.get("from", ""), *request.size) self.logger.debug("Received %s", request) except TypeError as exc: - if isinstance(message, dict): - msg = ZRequest.shorten_data(message) - else: - msg = message + msg = ZRequest.shorten_data(message) if isinstance(message, dict) else message raise MessageFormatError("Request has bad format: {}".format(exc), exception=exc, ori_message=msg) return request @@ -396,9 +404,8 @@ def stat(cls, path, hash_it=False, parent_times=False): ) if stat.S_ISLNK(res["st_mode"]): res["lnk_target"] = os.readlink(path) - else: - if hash_it: - res["sha256"] = hash_file(path) + elif hash_it: + res["sha256"] = hash_file(path) if parent_times: p_stat = os.lstat(os.path.dirname(os.path.normpath(path))) res["parent_atime"] = p_stat.st_atime @@ -445,7 +452,7 @@ def sync_tree(self, src_tree, no_delete): :return: list: of dicts (results of cmp_node()) """ self.logger.debug("sync_tree %r", src_tree["me"]["name"]) - self.diff_tree = dict() + self.diff_tree = {} # purge files/dirs not in source self.logger.debug("deleting files (%r)...", src_tree["me"]["name"]) @@ -465,21 +472,20 @@ def sync_tree(self, src_tree, no_delete): # walk src_tree, collect requests self.logger.debug("creating / modifying files (%r)...", src_tree["me"]["name"]) - requests = list() + requests = [] my_nodes = [src_tree["me"]] my_nodes.extend(src_tree["files"]) for node in my_nodes: if node["cmd"] == "ignore": self.logger.debug("Ignoring %r (cmd=ignore).", node["name"]) continue - res = self.cmp_node(node) - if res: + if res := self.cmp_node(node): requests.append(res) for adir in src_tree["dirs"]: requests.extend(self.sync_tree(adir, no_delete)) - if not adir["me"]["cmd"] == "ignore": + if adir["me"]["cmd"] != "ignore": os.utime(adir["me"]["name"], (adir["me"]["st_atime"], adir["me"]["st_mtime"])) - if not src_tree["me"]["cmd"] == "ignore": + if src_tree["me"]["cmd"] != "ignore": os.utime(src_tree["me"]["name"], (src_tree["me"]["st_atime"], src_tree["me"]["st_mtime"])) self.logger.debug("done (%r).", src_tree["me"]["name"]) @@ -582,26 +588,22 @@ def cmp_node(self, node): self.logger.warning("mtime difference is %d microseconds", (src_d - dst_d).microseconds) if stat.S_ISDIR(node["st_mode"]): warn = self.utime(file_path, node["st_atime"], node["st_mtime"]) + elif node["st_size"] == 0: + self.logger.info("truncate %r", file_path) + with open(file_path, "w") as fp: + fp.truncate(0) + warn = self.utime(file_path, node["st_atime"], node["st_mtime"]) + return dict(cmd="truncate", result="done", name=file_path, warn=warn) + elif node["st_blocks"] < MIN_BLOCK_DIFF_SIZE: + # not worth diffing on both ends and patching + self.logger.info("req full %r", file_path) + return dict(cmd="full", name=file_path) else: - if node["st_size"] == 0: - self.logger.info("truncate %r", file_path) - with open(file_path, "w") as fp: - fp.truncate(0) - warn = self.utime(file_path, node["st_atime"], node["st_mtime"]) - return dict(cmd="truncate", result="done", name=file_path, warn=warn) - elif node["st_blocks"] < MIN_BLOCK_DIFF_SIZE: - # not worth diffing on both ends and patching - self.logger.info("req full %r", file_path) - return dict(cmd="full", name=file_path) - else: - self.logger.info("req diff %r", file_path) - sig = self.get_signature(file_path) - return dict(cmd="diff", name=file_path, data=ZRequest.encode_binary(sig.read())) + self.logger.info("req diff %r", file_path) + sig = self.get_signature(file_path) + return dict(cmd="diff", name=file_path, data=ZRequest.encode_binary(sig.read())) else: raise RuntimeError("We should not be here.") - else: - # src_st == dst_st - pass except OSError as exc: self.logger.exception("Uncaught OSError. node=%r st=%r args=%r errno=%r filename=%r message=%r " "strerror=%r -> %r", node, st, exc.args, exc.errno, exc.filename, exc.message, @@ -690,11 +692,15 @@ def validate_node(_node): try: if not isinstance(_node["cmd"], str) or not isinstance(_node["name"], str): return False, "Bad src_tree _node {}. Item 'cmd' or 'name' has wrong type.".format(_node) - if _node["cmd"] != "ignore": - if (not all([isinstance(_node[x], int) for x in ["st_mode", "st_uid", "st_gid", "st_size"]]) or - not isinstance(_node["st_mtime"], float)): - return (False, "Bad src_tree _node {}. Item 'st_mode', 'st_uid', 'st_gid', 'st_mtime' or " - "'st_size' has wrong type.".format(_node)) + if _node["cmd"] != "ignore" and ( + not all( + isinstance(_node[x], int) + for x in ["st_mode", "st_uid", "st_gid", "st_size"] + ) + or not isinstance(_node["st_mtime"], float) + ): + return (False, "Bad src_tree _node {}. Item 'st_mode', 'st_uid', 'st_gid', 'st_mtime' or " + "'st_size' has wrong type.".format(_node)) except KeyError as exc: return False, "Bad src_tree _node {}. KeyError: '{}'".format(tree, exc) return True, "" @@ -770,9 +776,7 @@ def run(self): try: os.remove(name) except OSError as exc: - if exc.errno == 2: - pass - else: + if exc.errno != 2: raise reply["result"] = "done" elif cmd == "mv": @@ -801,9 +805,8 @@ def post_file_write_action(self, request): request["st_size"], request["st_blocks"], tmp_st.st_size, tmp_st.st_blocks) if hash_file(request.name) == request["sha256"]: return {"result": "done"} - else: - self.logger.error("Checksum mismatch after patching %r.", request.name) - return {"status": 2, "reason": "Checksum mismatch for '{}'.".format(request.name)} + self.logger.error("Checksum mismatch after patching %r.", request.name) + return {"status": 2, "reason": "Checksum mismatch for '{}'.".format(request.name)} def setup_logging(log_level, pid):