Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sourcery refactored master branch #7

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Sourcery refactored master branch #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 16 additions & 71 deletions zrsync/zrsync.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@ def __init__(self, name, source_dir, server_addr, server_port, target_dir, ssh,
self.ssh = ssh
self.no_delete = no_delete
self.initial_only = initial_only
self._error_tmp = dict()
self._move_tmp = dict()
self._previous_stat = dict()
self._error_tmp = {}
self._move_tmp = {}
self._previous_stat = {}
Comment on lines -64 to +66
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function SyncClient.__init__ refactored with the following changes:


def do_connect(self):
self._socket = self._context.socket(zmq.REQ)
Expand Down Expand Up @@ -162,17 +162,17 @@ def watch_dir(self, base_dir):
self.logger.debug("HANDLE_EVENTS: %r", HANDLE_EVENTS)
mask = reduce(lambda x, y: x | y, [getattr(inotify.constants, ev) for ev in HANDLE_EVENTS], 0)
for handler in self.logger.handlers:
if not handler in inotify.adapters._LOGGER.handlers:
if handler not in inotify.adapters._LOGGER.handlers:
inotify.adapters._LOGGER.addHandler(handler)
inotify.adapters._LOGGER.setLevel(self.logger.level)
notifyier = inotify.adapters.InotifyTree(base_dir, mask=mask)
for event in notifyier.event_gen():
if event is not None:
header, type_names, watch_path, filename = event
if not any([tn in HANDLE_EVENTS for tn in type_names]):
if all(tn not in HANDLE_EVENTS for tn in type_names):
self.logger.debug("ignoring event type_names=%r", type_names)
continue
if any([re.search(regex, filename) for regex in FILE_IGNORE_REGEX]):
if any(re.search(regex, filename) for regex in FILE_IGNORE_REGEX):
Comment on lines -165 to +175
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function SyncClient.watch_dir refactored with the following changes:

self.logger.debug("ignoring filename=%r", filename)
continue
self.logger.debug("WD=(%d) MASK=(%d) COOKIE=(%d) LEN=(%d) MASK->NAMES=%s WATCH-PATH=[%s] FILENAME=[%s]",
Expand Down Expand Up @@ -208,8 +208,8 @@ def handle_fs_event(self, action, path, filename, cookie):
self.logger.debug("File %r vanished, ignoring.", local_filename)
return True

if "IN_ISDIR" in action:
if "IN_CREATE" in action:
if "IN_CREATE" in action:
if "IN_ISDIR" in action:
Comment on lines -211 to +212
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function SyncClient.handle_fs_event refactored with the following changes:

This removes the following comments ( why? ):

# prevent unnecessary request by detecting IN_ATTRIB following
# file was moved away, noticed by request.add_stat() above
# IN_CREATE and IN_CLOSE_WRITE without stat change
# moved in from outside of inotify-observed tree
# prevent unnecessary request in case of an empty file

# This should be just a newly created, _empty_ directory, but
# inotify fails to add watches for newly created directories
# fast enough to catch for example 'mkdir -p'.
Expand All @@ -220,73 +220,18 @@ def handle_fs_event(self, action, path, filename, cookie):
return False
else:
request["cmd"] = "mkdir"
elif "IN_DELETE" in action:
if self.no_delete:
self.logger.debug("Not deleting %r: no_delete is set.", local_filename)
return True
else:
request["cmd"] = "rmdir"
else:
if "IN_CREATE" in action:
else:
self._previous_stat = request.copy()
if request["st_size"] == 0:
request["cmd"] = "truncate"
else:
request.add_full(local_filename).add_parent_time(local_filename).add_hash(local_filename)
elif "IN_ATTRIB" in action:
# prevent unnecessary request by detecting IN_ATTRIB following
# IN_CREATE and IN_CLOSE_WRITE without stat change
if request.has_same_stat(self._previous_stat):
return True
else:
self.logger.debug("***** IN_ATTRIB in action: request.has_same_stat(self._previous_stat) = False")
self.logger.debug("***** request.to_dict()=%r", request.to_dict())
self.logger.debug("***** self._previous_stat=%r", self._previous_stat)
self._previous_stat = request.copy()
request["cmd"] = "touch"
elif "IN_CLOSE_WRITE" in action:
# prevent unnecessary request in case of an empty file
if request.has_same_stat(self._previous_stat):
self.logger.debug("***** IN_CLOSE_WRITE in action: request.has_same_stat(self._previous_stat) = True")
return True
if request["st_size"] == 0:
if request.has_same_stat(self._previous_stat):
self.logger.debug("***** IN_CLOSE_WRITE in action: [st_size] = 0 AND request.has_same_stat(self._previous_stat) = True")
return True
else:
self.logger.debug("***** IN_CLOSE_WRITE in action: [st_size] = 0 AND request.has_same_stat(self._previous_stat) = False")
self.logger.debug("***** request.to_dict()=%r", request.to_dict())
self.logger.debug("***** self._previous_stat=%r", self._previous_stat)
self._previous_stat = request.copy()
request["cmd"] = "truncate"
elif request["st_blocks"] < MIN_BLOCK_DIFF_SIZE:
request.add_full(local_filename).request.add_parent_time(local_filename).add_hash(local_filename)
else:
request["cmd"] = "sig"
elif "IN_DELETE" in action:
if self.no_delete:
self.logger.debug("Not deleting %r: no_delete is set.", local_filename)
return True
else:
request["cmd"] = "rm"
elif "IN_MOVED_FROM" in action:
if request.cmd == "rm":
# file was moved away, noticed by request.add_stat() above
pass
else:
self._move_tmp[cookie] = remote_filename
return True
elif "IN_MOVED_TO" in action:
mv_src = self._move_tmp.pop("cookie", None)
if mv_src:
request.update({
"cmd": "mv",
"from": mv_src
})
else:
# moved in from outside of inotify-observed tree
request.add_full(local_filename).add_parent_time(local_filename).add_hash(local_filename)

elif "IN_DELETE" in action:
if self.no_delete:
self.logger.debug("Not deleting %r: no_delete is set.", local_filename)
return True
else:
request["cmd"] = "rmdir"
if request.cmd == "none":
raise RuntimeError("request.cmd was not set with action='{}' path='{}' filename='{}' cookie='{}'.".format(action, path, filename, cookie))

Expand Down Expand Up @@ -454,7 +399,7 @@ def main(args):
log_level=args["log_level"]
)
jobs = zip(args["<source-dir>"], args["<target-dir>"])
processes = list()
processes = []
Comment on lines -457 to +402
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function main refactored with the following changes:

for src, trg in jobs:
p_name = "zrsync client {}".format(src)
kwargs.update(dict(
Expand Down
103 changes: 53 additions & 50 deletions zrsync/zrsyncd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,7 @@ def __repr__(self):
pprint.pformat(self.shorten_data(self)))

def __setitem__(self, key, value):
if key == "data":
self._request[key] = self.encode_binary(value)
else:
self._request[key] = value
self._request[key] = self.encode_binary(value) if key == "data" else value
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function ZRequest.__setitem__ refactored with the following changes:


def add_diff(self, signature, filename=None):
if not filename:
Expand Down Expand Up @@ -198,17 +195,31 @@ def encode_binary(data):
def has_same_stat(self, other):
if not other:
return False
for key in ["st_mode", "st_uid", "st_gid", "st_size", "st_atime", "st_mtime", "st_blocks"]:
if self[key] != other[key]:
return False
return True
return all(
self[key] == other[key]
for key in [
"st_mode",
"st_uid",
"st_gid",
"st_size",
"st_atime",
"st_mtime",
"st_blocks",
]
)
Comment on lines -201 to +209
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function ZRequest.has_same_stat refactored with the following changes:


def is_valid(self):
if not all([isinstance(self.status, int) and (self.status == 0 or self.get("reason")),
isinstance(self.cmd, str), isinstance(self.name, str)]):
return False
if self.cmd == "patch":
if self.status == 0 and not (self.get("result") == "done" or all([self.get(x) for x in ["data", "parent_atime", "parent_mtime"]])):
if (
self.status == 0
and self.get("result") != "done"
and not all(
self.get(x) for x in ["data", "parent_atime", "parent_mtime"]
)
):
Comment on lines -211 to +222
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function ZRequest.is_valid refactored with the following changes:

return False
elif self.cmd == "sync_dir":
if not (isinstance(self.get("tree_diff"), list) or
Expand Down Expand Up @@ -344,10 +355,7 @@ def receive(self):
request.get("result") or request.get("from", ""), *request.size)
self.logger.debug("Received %s", request)
except TypeError as exc:
if isinstance(message, dict):
msg = ZRequest.shorten_data(message)
else:
msg = message
msg = ZRequest.shorten_data(message) if isinstance(message, dict) else message
Comment on lines -347 to +358
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function ZRsyncBase.receive refactored with the following changes:

raise MessageFormatError("Request has bad format: {}".format(exc), exception=exc, ori_message=msg)
return request

Expand Down Expand Up @@ -396,9 +404,8 @@ def stat(cls, path, hash_it=False, parent_times=False):
)
if stat.S_ISLNK(res["st_mode"]):
res["lnk_target"] = os.readlink(path)
else:
if hash_it:
res["sha256"] = hash_file(path)
elif hash_it:
res["sha256"] = hash_file(path)
Comment on lines -399 to +408
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function ZRsyncBase.stat refactored with the following changes:

if parent_times:
p_stat = os.lstat(os.path.dirname(os.path.normpath(path)))
res["parent_atime"] = p_stat.st_atime
Expand Down Expand Up @@ -445,7 +452,7 @@ def sync_tree(self, src_tree, no_delete):
:return: list: of dicts (results of cmp_node())
"""
self.logger.debug("sync_tree %r", src_tree["me"]["name"])
self.diff_tree = dict()
self.diff_tree = {}
Comment on lines -448 to +455
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function SyncServer.sync_tree refactored with the following changes:


# purge files/dirs not in source
self.logger.debug("deleting files (%r)...", src_tree["me"]["name"])
Expand All @@ -465,21 +472,20 @@ def sync_tree(self, src_tree, no_delete):

# walk src_tree, collect requests
self.logger.debug("creating / modifying files (%r)...", src_tree["me"]["name"])
requests = list()
requests = []
my_nodes = [src_tree["me"]]
my_nodes.extend(src_tree["files"])
for node in my_nodes:
if node["cmd"] == "ignore":
self.logger.debug("Ignoring %r (cmd=ignore).", node["name"])
continue
res = self.cmp_node(node)
if res:
if res := self.cmp_node(node):
requests.append(res)
for adir in src_tree["dirs"]:
requests.extend(self.sync_tree(adir, no_delete))
if not adir["me"]["cmd"] == "ignore":
if adir["me"]["cmd"] != "ignore":
os.utime(adir["me"]["name"], (adir["me"]["st_atime"], adir["me"]["st_mtime"]))
if not src_tree["me"]["cmd"] == "ignore":
if src_tree["me"]["cmd"] != "ignore":
os.utime(src_tree["me"]["name"], (src_tree["me"]["st_atime"], src_tree["me"]["st_mtime"]))

self.logger.debug("done (%r).", src_tree["me"]["name"])
Expand Down Expand Up @@ -582,26 +588,22 @@ def cmp_node(self, node):
self.logger.warning("mtime difference is %d microseconds", (src_d - dst_d).microseconds)
if stat.S_ISDIR(node["st_mode"]):
warn = self.utime(file_path, node["st_atime"], node["st_mtime"])
elif node["st_size"] == 0:
self.logger.info("truncate %r", file_path)
with open(file_path, "w") as fp:
fp.truncate(0)
warn = self.utime(file_path, node["st_atime"], node["st_mtime"])
return dict(cmd="truncate", result="done", name=file_path, warn=warn)
elif node["st_blocks"] < MIN_BLOCK_DIFF_SIZE:
# not worth diffing on both ends and patching
self.logger.info("req full %r", file_path)
return dict(cmd="full", name=file_path)
else:
if node["st_size"] == 0:
self.logger.info("truncate %r", file_path)
with open(file_path, "w") as fp:
fp.truncate(0)
warn = self.utime(file_path, node["st_atime"], node["st_mtime"])
return dict(cmd="truncate", result="done", name=file_path, warn=warn)
elif node["st_blocks"] < MIN_BLOCK_DIFF_SIZE:
# not worth diffing on both ends and patching
self.logger.info("req full %r", file_path)
return dict(cmd="full", name=file_path)
else:
self.logger.info("req diff %r", file_path)
sig = self.get_signature(file_path)
return dict(cmd="diff", name=file_path, data=ZRequest.encode_binary(sig.read()))
self.logger.info("req diff %r", file_path)
sig = self.get_signature(file_path)
return dict(cmd="diff", name=file_path, data=ZRequest.encode_binary(sig.read()))
else:
raise RuntimeError("We should not be here.")
else:
# src_st == dst_st
pass
Comment on lines +591 to -604
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function SyncServer.cmp_node refactored with the following changes:

This removes the following comments ( why? ):

# src_st == dst_st

except OSError as exc:
self.logger.exception("Uncaught OSError. node=%r st=%r args=%r errno=%r filename=%r message=%r "
"strerror=%r -> %r", node, st, exc.args, exc.errno, exc.filename, exc.message,
Expand Down Expand Up @@ -690,11 +692,15 @@ def validate_node(_node):
try:
if not isinstance(_node["cmd"], str) or not isinstance(_node["name"], str):
return False, "Bad src_tree _node {}. Item 'cmd' or 'name' has wrong type.".format(_node)
if _node["cmd"] != "ignore":
if (not all([isinstance(_node[x], int) for x in ["st_mode", "st_uid", "st_gid", "st_size"]]) or
not isinstance(_node["st_mtime"], float)):
return (False, "Bad src_tree _node {}. Item 'st_mode', 'st_uid', 'st_gid', 'st_mtime' or "
"'st_size' has wrong type.".format(_node))
if _node["cmd"] != "ignore" and (
not all(
isinstance(_node[x], int)
for x in ["st_mode", "st_uid", "st_gid", "st_size"]
)
or not isinstance(_node["st_mtime"], float)
):
return (False, "Bad src_tree _node {}. Item 'st_mode', 'st_uid', 'st_gid', 'st_mtime' or "
"'st_size' has wrong type.".format(_node))
Comment on lines -693 to +703
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function SyncServer.validate_tree_structure.validate_node refactored with the following changes:

except KeyError as exc:
return False, "Bad src_tree _node {}. KeyError: '{}'".format(tree, exc)
return True, ""
Expand Down Expand Up @@ -770,9 +776,7 @@ def run(self):
try:
os.remove(name)
except OSError as exc:
if exc.errno == 2:
pass
else:
if exc.errno != 2:
Comment on lines -773 to +779
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function SyncServer.run refactored with the following changes:

raise
reply["result"] = "done"
elif cmd == "mv":
Expand Down Expand Up @@ -801,9 +805,8 @@ def post_file_write_action(self, request):
request["st_size"], request["st_blocks"], tmp_st.st_size, tmp_st.st_blocks)
if hash_file(request.name) == request["sha256"]:
return {"result": "done"}
else:
self.logger.error("Checksum mismatch after patching %r.", request.name)
return {"status": 2, "reason": "Checksum mismatch for '{}'.".format(request.name)}
self.logger.error("Checksum mismatch after patching %r.", request.name)
return {"status": 2, "reason": "Checksum mismatch for '{}'.".format(request.name)}
Comment on lines -804 to +809
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function SyncServer.post_file_write_action refactored with the following changes:



def setup_logging(log_level, pid):
Expand Down