scim

#!/usr/bin/env python

import base64, struct, pathlib as pl, contextlib as cl
import os, sys, io, stat, logging, enum, re, tempfile


err_fmt = lambda err: f'[{err.__class__.__name__}] {err}'

log = logging.getLogger('scim')
log_meta = logging.getLogger('scim.meta')
log_links = logging.getLogger('scim.links')

class adict(dict):
	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.__dict__ = self

def copy_fd_meta(fd_src, fd_dst, mode=None, uidgid=None, xattrs=None):
	'''Metadata values: True = copy from src,
		None = best-effort copy, False = ignore, other value = apply that.'''
	uidgid_be, xattrs_be = uidgid is None, xattrs is None
	val_auto = lambda v: fd_src is not None and (v is True or v is None)
	if val_auto(mode) or val_auto(uidgid):
		st = os.stat(fd_src)
		if val_auto(mode): mode = stat.S_IMODE(st.st_mode)
		if val_auto(uidgid): uidgid = st.st_uid, st.st_gid
	if val_auto(xattrs) and getattr(os, 'getxattr', None):
		xattrs = dict()
		for k in os.listxattr(fd_src):
			try: xattrs[k] = os.getxattr(fd_src, k)
			except OSError:
				if not xattrs_be: raise
	val_restore = lambda v: v is not False and v is not None
	if val_restore(mode): os.fchmod(fd_dst, mode)
	if val_restore(uidgid):
		try: os.fchown(fd_dst, *uidgid)
		except OSError:
			if not uidgid_be: raise
	if val_restore(xattrs):
		for k, v in xattrs.items():
			try: os.setxattr(fd_dst, k, v)
			except OSError:
				if not xattrs_be: raise

@cl.contextmanager
def safe_replacement( path, *open_args,
		mode=None, uidgid=None, xattrs=None, **open_kws ):
	'''Context to atomically create/replace file-path in-place unless errors are raised.
		Metadata options: True = copy from path,
			None = best-effort copy, False = ignore, other value = apply that.'''
	open_kws.update( delete=False,
		dir=os.path.dirname(path := str(path)), prefix=os.path.basename(path)+'.' )
	if not open_args: open_kws.setdefault('mode', 'w')
	with tempfile.NamedTemporaryFile(*open_args, **open_kws) as tmp:
		try:
			fd_src = None
			try:
				try: fd_src = os.open(path, os.O_RDONLY)
				except FileNotFoundError: pass
				copy_fd_meta(fd_src, tmp.fileno(), mode, uidgid, xattrs)
			finally:
				if fd_src is not None: os.close(fd_src)
			yield tmp
			if not tmp.closed: tmp.flush()
			try: os.fdatasync(tmp)
			except AttributeError: pass
			os.rename(tmp.name, path)
		finally:
			try: os.unlink(tmp.name)
			except FileNotFoundError: pass

def open_read_fd(p, root=None, _cache=[]):
	'''Returns read-only fd context for specified path under root-dir,
		opened safely wrt symlinks pointing outside of that root, if specified.'''
	# Simple way to avoid symlink-related race-conditions w/o disallowing them:
	#  p = realpath(path); validate_is_acceptable(p); file = openat2(p, no_symlinks)
	# As of 2022-10-13 glibc has no openat2() wrapper yet, hence syscall here.
	if root:
		p = (root := root.resolve(True)) / p
		if not (p := p.resolve(True)).is_relative_to(root):
			raise OSError(f'Specified path/root mismatch [ {p} ]: {root}')
	else: p = p.resolve(True) # no-validation passthrough
	if not _cache:
		import ctypes as ct
		openat2_how = ct.create_string_buffer(struct.pack(
			'@QQQ', os.O_RDONLY | os.O_NOFOLLOW, 0, res_no_symlinks := 0x04 ))
		c_sys_openat2, c_at_fdcwd, openat2_how_len = 437, -100, len(openat2_how) - 1
		c_syscall = ct.CDLL(None, use_errno=True).syscall
		@cl.contextmanager
		def _openat2(p):
			fd = c_syscall( c_sys_openat2,
				c_at_fdcwd, bytes(p), openat2_how, openat2_how_len )
			if fd < 0: raise OSError(err := ct.get_errno(), os.strerror(err))
			try: yield fd
			finally:
				try: os.close(fd)
				except OSError as err:
					if err.errno != 9: raise # EBADFD = already closed
		_cache.append(_openat2)
	return _cache[0](p)

class ListFile:
	cache = None
	def __init__(self, root, p):
		self.root, self.path, self.node = root, p, str(p.relative_to(root))
	def exists(self): return self.path.exists()
	def lines(self):
		if self.cache is None:
			self.cache = list( ln.strip()
				for ln in self.path.read_text().rstrip().split('\n') )
			if any(( ln.startswith('>>>>>>>') or
					ln.startswith('<<<<<<<') ) for ln in self.cache):
				log.error(f'git-merge block detected in [ {self.path.name} ]:{n}')
		return self.cache
	def update(self, lines):
		with safe_replacement(self.path) as dst:
			for line in lines: dst.write(line + '\n')
		self.cache = None

class ChangeCount:
	errs = meta = errs_fix = 0
	errs_max = meta_max = errs_fix_max = 2**30
	def __init__(self, **limits):
		for k, v in limits.items():
			setattr(self, f'{k}_max', {None: 2**30}.get(v, v))
	def inc(self, k):
		setattr(self, k, n := getattr(self, k) + 1)
		return n >= getattr(self, f'{k}_max')
	inc_errs = lambda s: s.inc('errs')
	inc_errs_fix = lambda s: [s.inc('errs_fix'), s.inc('errs')][1]
	inc_meta = lambda s: s.inc('meta') # metadata updates
	inc_meta_errs = lambda s: [s.inc('meta'), s.inc('errs')][1]
	errs_limit = property(lambda s: s.errs >= s.errs_max)



### Metadata handling

def meta_uidgid_resolver(numeric=False):
	'Return uidgid func with baked-in processing parameters'
	if numeric:
		uid_func = gid_func = lambda n: n if isinstance(n, int) else int(n)
	else:
		import pwd, grp
		uid_func = lambda n: ( pwd.getpwuid(n).pw_name
			if isinstance(n, int) else (
				int(n) if n.isdigit() else pwd.getpwnam(n).pw_uid ) )
		gid_func = lambda n: ( grp.getgrgid(n).gr_name
			if isinstance(n, int) else (
				int(n) if n.isdigit() else grp.getgrnam(n).gr_gid ) )
	def uidgid(uid=None, gid=None):
		'Returns specified id, resolved according to baked-in parameters'
		res = list()
		for n0, resolve in (uid, uid_func), (gid, gid_func):
			if n0 is None: continue
			try: n = resolve(n0)
			except KeyError: pass
			res.append(int(n) if not isinstance(n0, int) else str(n))
		return res if len(res) != 1 else res[0]
	return uidgid

class ACLTag(enum.IntEnum):
	uo = 0x01; u = 0x02; go = 0x04; g = 0x08
	mask = 0x10; other = 0x20
	str = property(lambda s: s._name_)

class ACLPerm(enum.IntFlag):
	r = 4; w = 2; x = 1
	str = property(lambda s: ''.join(
		(v._name_ if v in s else '-') for v in s.__class__ ))

Caps = enum.IntEnum('Caps', dict((cap, 1 << n) for n, cap in enumerate((
	' chown dac_override dac_read_search fowner fsetid kill setgid setuid setpcap'
	' linux_immutable net_bind_service net_broadcast net_admin net_raw ipc_lock ipc_owner'
	' sys_module sys_rawio sys_chroot sys_ptrace sys_pacct sys_admin sys_boot'
	' sys_nice sys_resource sys_time sys_tty_config mknod lease audit_write audit_control'
	' setfcap mac_override mac_admin syslog wake_alarm block_suspend audit_read'
	' perfmon bpf checkpoint_restore' ).split())))

def meta_acl_parse(acl, uidgid, default=False):
	acl, lines = io.BytesIO(acl), list()
	if (v := acl.read(4)) != b'\2\0\0\0':
		raise ValueError(f'ACL version mismatch [ {v} ]')
	while True:
		if not (entry := acl.read(8)): break
		elif len(entry) != 8: raise ValueError('ACL length mismatch')
		tag, perm, n = struct.unpack('HHI', entry)
		tag, perm = ACLTag(tag), ACLPerm(perm)
		match tag:
			case ACLTag.uo | ACLTag.go:
				lines.append(f'{tag.str[0]}::{perm.str}')
			case ACLTag.u | ACLTag.g:
				name = uidgid(uid=n) if tag is tag.u else uidgid(gid=n)
				if set(name).intersection(':,'): raise ValueError(n, name)
				lines.append(f'{tag.str}:{name}:{perm.str}')
			case ACLTag.other: lines.append(f'o::{perm.str}')
			case ACLTag.mask: lines.append(f'm::{perm.str}')
			case _: raise ValueError(tag)
	prefix = '' if not default else 'd:'
	return ','.join(f'{prefix}{s}' for s in lines)

def meta_caps_parse(cap):
	cap = io.BytesIO(cap)
	ver, eff = ((v := struct.unpack('I', cap.read(4))[0]) >> 3*8) & 0xff, v & 1
	if ver not in [2, 3]: raise ValueError(f'Unsupported capability v{ver}')
	perm1, inh1, perm2, inh2 = struct.unpack('IIII', cap.read(16))
	if (n := len(cap.read())) != (n_tail := {2:0, 3:4}[ver]):
		raise ValueError(f'Cap-string length mismatch [ {n} != {n_tail} ]')
	perm_bits, inh_bits = perm2 << 32 | perm1, inh2 << 32 | inh1
	perm, inh = set(), set()
	for c in Caps:
		if perm_bits & c.value: perm.add(c.name); perm_bits -= c.value
		if inh_bits & c.value: inh.add(c.name); inh_bits -= c.value
	if perm_bits or inh_bits:
		raise ValueError(f'Unrecognized cap-bits: P={perm_bits:x} I={inh_bits:x}')
	if not (eff or perm or inh): return ''
	cap_str = lambda caps: ','.join(sorted(caps, key=Caps.__getitem__))
	caps = [cap_str(perm) + ':' + cap_str(inh)]
	if perm and perm.issubset(inh):
		caps.extend([
			'' + cap_str(perm) + ':+' + cap_str(inh-perm),
			'-' + cap_str(inh-perm) + ':' + cap_str(inh) ])
	elif inh and inh.issubset(perm):
		caps.extend([
			'+' + cap_str(perm-inh) + ':' + cap_str(inh),
			'' + cap_str(perm) + ':-' + cap_str(perm-inh) ])
	caps = sorted(caps, key=len)[0]
	pre = ''.join(v for v,e in zip('EPI', [eff, perm, inh]) if e)
	return (pre + ':' + caps).rstrip(':')

def meta_acls_encode(acls, uidgid):
	acl_acc, acl_def = list(), list()
	for acl in acls:
		entries, (k, acl) = acl_acc, acl.split(':', 1)
		if k == 'd': entries, (k, acl) = acl_def, acl.split(':', 1)
		if k in 'ug':
			name, acl = acl.split(':', 1)
			n = uidgid(**{f'{k}id': name})
		else: n = 2**32 - 1
		perm = sum(2**n for n, k in enumerate('xwr') if k in acl)
		entries.append(struct.pack('HHI', ACLTag[k], perm, n))
	return ((b'\2\0\0\0' + b''.join(entries)) for entries in [acl_acc, acl_def])

def meta_caps_encode(caps):
	if not caps: return
	pre, caps = caps.split(':', 1)
	cp, sep, ci = caps.partition(':')
	(cpx, cp), (cix, ci) = ((cap[:1], set(
		Caps[c] for c in cap.lstrip('+-').split(',') if c )) for cap in [cp, ci])
	if cpx and cpx in '+-': cp = (ci - cp) if cpx == '-' else (cp | ci)
	elif cix and cix in '+-': ci = (cp - ci) if cix == '-' else (cp | ci)
	perm, inh = sum(cp), sum(ci)
	perm1, inh1 = perm & (2**32 - 1), inh & (2**32 - 1)
	perm2, inh2 = perm >> 32, inh >> 32
	return struct.pack( 'IIIII',
		(3 << 3*8) + ('E' in pre), perm1, inh1, perm2, inh2 )


def meta_get(p, uidgid, root=None):
	with open_read_fd(p, root) as fd: return _meta_get(fd, uidgid)

def meta_set(p, uidgid, meta, root=None):
	with open_read_fd(p, root) as fd: return _meta_set(fd, uidgid, meta)

def _meta_get(fd, uidgid):
	fstat = os.stat(fd)
	try: xattrs = dict((k, os.getxattr(fd, k)) for k in os.listxattr(fd))
	except: xattrs = dict()
	meta = ':'.join(uidgid(fstat.st_uid, fstat.st_gid))
	meta += f':{stat.S_IMODE(fstat.st_mode):o}'
	caps, acls = list(), list()
	if cap := xattrs.pop('security.capability', ''):
		caps.append(meta_caps_parse(cap))
	if acl := xattrs.pop('system.posix_acl_access', ''):
		acls.append(meta_acl_parse(acl, uidgid))
	if acl := xattrs.pop('system.posix_acl_default', ''):
		acls.append(meta_acl_parse(acl, uidgid, default=True))
	if caps or acls or xattrs:
		meta += f'/{",".join(caps)}'
		if acls or xattrs:
			meta += f'/{",".join(acls)}'
			if xattrs:
				ext = list()
				for k, v in xattrs.items():
					try:
						v = v.decode('ascii')
						if ',' in v or ':' in v: raise ValueError
					except ValueError:
						v = ':' + base64.urlsafe_b64encode(v).decode()
					ext.append(f'{k}={v}')
				meta += f'/{",".join(ext)}'
	return meta

def _meta_set(fd, uidgid, meta):
	caps = acls = xattrs = None
	try:
		meta, xattrs = meta.split('/', 1)
		caps, xattrs = meta.split('/', 1)
		acls, xattrs = caps.split('/', 1)
	except ValueError: pass
	uid, gid, mode = meta.split(':')
	(uid, gid), mode = uidgid(uid, gid), int(mode, 8)
	caps = meta_caps_encode(caps)
	if xattrs:
		xattrs, xattr_str = dict(), xattrs
		for xattr in xattr_str.split(','):
			k, v = xattr.split(':', 1)
			if v.startswith(':'): v = base64.urlsafe_b64decode(v[1:])
			xattrs[k] = v
	# Strip mode/xattrs first to prevent racy access in-between changes
	os.chmod(fd, 0)
	for k in os.listxattr(fd): os.removexattr(fd, k)
	os.chown(fd, uid, gid)
	os.chmod(fd, mode)
	if acls:
		acl_acc, acl_def = meta_acls_encode((acls or '').split(','), uidgid)
		os.setxattr(fd, 'system.posix_acl_access', acl_acc, os.XATTR_CREATE)
		os.setxattr(fd, 'system.posix_acl_default', acl_def, os.XATTR_CREATE)
	if caps: os.setxattr(fd, 'security.capability', caps, os.XATTR_CREATE)
	if xattrs:
		for k, v in xattrs.items(): os.setxattr(fd, k, v, os.XATTR_CREATE)


def meta_map_make(root, lines, exc, nnn, uidgid):
	'Returns {repo_path: metadata} mapping with some sanity-checks'
	res = dict()
	for line in lines:
		if not line: continue
		node, sep, meta = line.rpartition(' ')
		node = pl.Path(node)
		if node.is_absolute() and node.is_relative_to(root):
			node = node.relative_to(root)
		if (ns := str(node)) and any(chk.search(ns) for chk in exc):
			log_meta.debug('Excluded path in metadata-list [ %s ]: %s', node, meta)
		if ns in res:
			if res[ns] != (meta_diff := meta): meta_diff += f' => {res[ns]}'
			log_meta.warning( 'Duplicate metadata for path,'
				' last one will be used [ %s ]: %s', node, meta_diff )
			if nnn.inc_meta(): break
		res[ns] = meta
	for p_dir, dirs, files in os.walk(root): # reverse check
		if nnn.errs_limit: break
		p_dir = pl.Path(p_dir)
		for fn in files:
			if nnn.errs_limit: break
			node = pl.Path('')
			for s in (src := p_dir / fn).relative_to(root).parts:
				if (ns := str(node := node / s)) in res: continue
				if any(chk.search(ns) for chk in exc): continue
				meta = res[ns] = meta_get(node, uidgid, root)
				log_meta.warning('Not in metadata-list [ %s ]: %s', node, meta)
				if nnn.inc_meta_errs(): break
	return res

def meta_map_update(root, meta_map, path_res, nnn, uidgid, fix, update_list):
	for ns, meta in list(meta_map.items()):
		if nnn.errs_limit: return
		if not any(pre.search(ns) for pre in path_res): continue

		try: meta_new = meta_get(ns, uidgid, root)
		except OSError as err:
			log_meta.error('Failed to stat path, removing it [ %s ]: %s', ns, err)
			del meta_map[ns]
			if nnn.inc_meta_errs(): break
			meta_new = None
		if not meta_new or meta_new == meta: continue
		if nnn.inc_errs(): break

		log_meta.error( 'Metadata %s [ %s ]: < %s > should be < %s >',
			*( ('mismatch', ns, meta_new, meta)
				if not update_list else ('list update', ns, meta, meta_new) ) )
		if update_list:
			meta = meta_map[ns] = meta_new
			if nnn.inc_meta(): break
		elif fix:
			try: meta_set(ns, uidgid, meta, root)
			except OSError as err:
				log_meta.warning('Unable to fix metadata [ %s ]: %s : %s', ns, meta, err)
				if nnn.inc_errs_fix(): break
			else: log_meta.info('Fixed metadata [ %s ]: %s', ns, meta)



### Links handling

LinkType = enum.IntEnum('LT', dict(sym=1, rsym=3, cp=4))
LinkTypeSep = {LinkType.sym: '->', LinkType.rsym: '+>', LinkType.cp: '>'}

def link_sym_with_uidgid(p, p_link, meta):
	if meta: uid, gid = (st := p_link.stat()).st_uid, st.st_gid
	p.symlink_to(p_link)
	if meta: os.chown(p, uid, gid, follow_symlinks=False)

def link_relative_path(src, dst):
	'''Returns relative path from dst to src, including .. where necessary,
		i.e. what os.readlink(dst) should return for this src path.'''
	src, dst = src.absolute().parts[1:], dst.absolute().parent.parts[1:]
	for n in range(min(len(dst), len(src))):
		if dst[n] != src[n]: break
		else: n +=1
	return pl.Path(os.path.join(*([os.pardir]*(len(dst)-n)), *src[n:]))

def link_file_type( p,
		_ts='reg lnk dir chr blk fifo sock door port wht'.split(),
		_ts_name=dict(blk='blk_dev', reg='file', lnk='symlink', wht='whiteout') ):
	try: mode = p.stat().st_mode
	except FileNotFoundError: return '-nx-'
	types = list()
	for c in _ts:
		if getattr(stat, f'S_IS{c.upper()}')(mode): types.append(_ts_name.get(c, c))
	return '/'.join(types) or '???'

def link_copy(src, dst, meta=False, root=None, **repl_kws):
	'Copy src to dst with all metadata (if requested), in a reasonably atomic way'
	with open_read_fd(src, root) as a_fd, open(a_fd, 'rb') as a:
		kws = dict() if not meta else dict(
			mode=stat.S_IMODE((st := os.stat(a_fd)).st_mode),
			uidgid=(st.st_uid, st.st_gid),
			xattrs=dict((k, os.getxattr(a_fd, k)) for k in os.listxattr(a_fd)) )
		kws.update(repl_kws)
		a.seek(0, os.SEEK_END)
		with safe_replacement(dst, 'wb', **kws) as b:
			os.sendfile(b.fileno(), a_fd, 0, a.tell())

def link_match_data(src, dst, bs=64*2**10):
	try:
		if src.samefile(dst): return True
		sa, sb = src.stat(), dst.stat()
		if sa.st_size != sb.st_size: return False
		if stat.S_IFMT(sa.st_mode) != stat.S_IFMT(sb.st_mode): return
		with src.open('rb') as a, dst.open('rb') as b:
			while True:
				ca, cb = a.read(bs), b.read(bs)
				if ca == cb == b'': return True
				if ca != cb: return False
	except FileNotFoundError: return False

def link_diff_cmd(diff_cmd_list, src, dst):
	if not diff_cmd_list: return
	import subprocess
	log_links.warning(' <<<<<<<<< Source/Replacement diff:')
	for diff_cmd in diff_cmd_list:
		try: proc = subprocess.run(diff_cmd + [src, dst])
		except Exception as err:
			log_links.fatal('Failed to run diff-command: %s', err_fmt(err))
			raise
		if proc.returncode: break
	log_links.warning(' >>>>>>>>>')

def link_line_split(line):
	for lt, sep in LinkTypeSep.items():
		try: node, dst = line.split(f' {sep} ')
		except ValueError: continue
		return pl.Path(node), lt, pl.Path(dst)
	else: return None, None, None

def link_list_hash(lines):
	list_hash = dict()
	for line in lines:
		if not line: continue
		node, lt, dst = link_line_split(line)
		if not node: continue
		list_hash[str(node)] = lt, dst
		list_hash[str(dst)], list_hash[line] = str(node), str(dst)
	return list_hash

def link_list_enforce( root, lines, exc, diff_cmd_list, path_res, nnn,
		uidgid, fix, meta, content_from_repo, meta_to_repo, no_content ):
	linked_nodes, err_fail_check = set(), None
	link_sym = lambda p, p_link: link_sym_with_uidgid(p, p_link, meta=meta)

	for n, line in enumerate(lines, 1):
		if err_fail_check is not None and nnn.errs > err_fail_check:
			log_links.critical('Critical error, stopping checks here')
			raise RuntimeError(f'Critical error in links-file on line {n-1}')
		else: err_fail_check = None
		if nnn.errs_limit: break

		if not line: continue
		node, lt, dst = link_line_split(line)
		if not node:
			log_links.error('Invalid line format [uncorrectable]:%d: %r', n, line)
			continue
		if (ns := str(node)) and any(chk.search(ns) for chk in exc):
			log_meta.debug('Excluded path in links-list:%d: %s', n, line)

		src, dst = root / node, root / dst.expanduser()
		src_rsym = src if not lt == lt.rsym else link_relative_path(src, dst)
		dst_res = dst.resolve()
		if src.resolve() == root: root, err_fail_check = dst, nnn.errs
		else:
			if src.is_dir():
				log_links.debug('Linked whole path [bad-practice]:%d: %s', n, node)
			linked_nodes.add(ns) # skipped in reverse-check
		if not any(pre.search(ns) for pre in path_res): continue

		if not src.exists():
			if nnn.inc_errs(): break
			log_links.error('Link src missing [uncorrectable]:%d: %s', n, node)
			continue

		## Missing dst
		if not os.path.lexists(dst):
			log_links.error('No link destination [ %s ]:%d: %s', node, n, dst)
			if fix:
				try:
					if lt & lt.sym: link_sym(dst, src_rsym)
					elif lt & lt.cp:
						if dst.is_dir():
							log_links.error( 'Cannot copy to link dst - is'
								' an existing dir [ %s ]:%d: %s', node, n, dst )
							if nnn.inc_errs_fix(): break
							continue
						link_copy(src, dst, meta, root)
				except OSError as err:
					log_links.warning( 'Failed to create link dst [ %s'
						' %s %s ]:%d: %s', node, LinkTypeSep[lt], dst, n, err )
					if nnn.inc_errs_fix(): break
				else: log_links.info('Corrected:%d: %s %s %s', n, node, LinkTypeSep[lt], dst)
			if nnn.inc_errs(): break
			continue # no dst prevents other checks, and fix here should correct everything

		## Mismatched dst type - link instead of copy or vice-versa
		# Missing dst is already handled above
		if (lt & lt.sym and not dst.is_symlink()) or (lt & lt.cp and dst.is_symlink()):
			match, dst_is_broken_link = ..., not dst.exists()
			# no_content skips replacing diff-content dst that needs to copy data from it first
			if ( no_content and not dst_is_broken_link and
				(match := link_match_data(src, dst_res)) is False ): continue

			node_type_err = copy_dst_contents_to_src = False
			if lt & lt.sym:
				log_links.error('Not a symlink [ %s ]:%d: %s', node, n, dst)
				node_type_err = True
			elif lt & lt.cp:
				log_links.error('Symlink in place of a copy [ %s ]:%d: %s', node, n, dst)
				node_type_err = True
			if not (dst_is_broken_link or content_from_repo):
				# if dst is replaced by symlink: can be a file, to copy contents from into repo
				# if dst replaced by copy: can be symlink to file with diff contents, also diff/copy
				if match is ...: match = link_match_data(src, dst_res)
				copy_dst_contents_to_src = match is False

			if not fix:
				dst_repl = 'symlink' if lt & lt.sym else 'copy'
				if match is None:
					log_links.error(
						'Diff dst-type to replace with %s:%d: %s [dst=%s]',
						dst_repl, n, node, link_file_type(dst_res) )
				elif copy_dst_contents_to_src:
					log_links.error( 'Diff dst-content to replace'
						' with %s:%d: %s [fix: <dst]', dst_repl, n, node )
					link_diff_cmd(diff_cmd_list, src, dst)
			else:
				if copy_dst_contents_to_src: link_copy(dst_res, src)
				dst.unlink()
				if lt & lt.sym: link_sym(dst, src_rsym)
				elif lt & lt.cp: link_copy(src, dst, meta, root)
				log_links.info( 'Corrected: %s %s %s %s',
					node, '<>'[content_from_repo], LinkTypeSep[lt], dst )

			if node_type_err and nnn.inc_errs(): break
			continue # rest of the checks irrelevant here, or copy/link fixed everything

		## Symlinks - check readlink() target and uid/gid
		if lt & lt.sym:
			uidgid_update, dst_targets = False, [dst.readlink(), dst_res]
			if src_rsym not in dst_targets:
				log_links.error( 'Wrong symlink:%d: %s ->'
					' [ %s -should-be-> %s ]', n, dst, dst_targets[0], src_rsym )
				if fix:
					dst.unlink()
					link_sym(dst, src_rsym)
					log_links.info( 'Corrected: %s -link-to-> %s%s',
						dst, src_rsym, f' [ {src} ]' if src_rsym != src else '' )
					uidgid_update = True
				if nnn.inc_errs(): break
			if meta:
				uid, gid = (st := src.stat(follow_symlinks=False)).st_uid, st.st_gid
				luid, lgid = (st := dst.stat(follow_symlinks=False)).st_uid, st.st_gid
				if (luid, lgid) != (uid, gid):
					if not uidgid_update:
						log_links.error( 'Symlink uid/gid mismatch [ %s ]:%d:'
							' %s:%s should be %s:%s', dst, n, luid, lgid, uid, gid )
					if fix:
						os.chown(dst, uid, gid, follow_symlinks=False)
						if not uidgid_update:
							log_links.info( 'Corrected symlink uid/gid'
								' [ %s ]: %s:%s -> %s:%s', dst, luid, lgid, uid, gid )
					if not uidgid_update and nnn.inc_errs(): break

		## Copies - check permissions and content
		elif lt & lt.cp:
			if meta: # copy metadata, from dir/repo to fs by default
				cp_from, cp_to = src, dst
				meta_src, meta_dst = meta_get(src, uidgid, root), meta_get(dst, uidgid)
				if meta_to_repo:
					cp_from, cp_to, meta_src, meta_dst = cp_to, cp_from, meta_dst, meta_src
				if meta_src != meta_dst:
					log_links.error( 'Metadata mismatch for %s path'
						' [ %s ]:%d: < %s > should be < %s >', *( ('live-fs', dst)
							if not meta_to_repo else ('src', node) ), n, meta_dst, meta_src )
					if fix:
						try: meta_set(cp_to, uidgid, meta_src)
						except OSError as err:
							log_links.error( 'Failed to fix metadata'
								' [ %s ]:%d: < %s > %s', cp_to, n, meta_src, err )
							if nnn.inc_errs_fix(): break
						else: log_links.info('Fixed metadata [ %s ]: < %s >', cp_to, meta_src)
					if nnn.inc_errs(): break

			if not no_content and not (match := link_match_data(src, dst)):
				cp_from, cp_to = (src, dst) if content_from_repo else (dst, src)
				cp_dir = '<>'[content_from_repo]
				log_links.error('Diff content:%d: %s [fix: %sdst]', n, node, cp_dir)
				if not fix:
					if match is None:
						log_links.error( 'Diff types:%d: %s [%s] %s %s [%s]',
							n, node, link_file_type(src), cp_dir, dst, link_file_type(dst) )
					else: link_diff_cmd(diff_cmd_list, cp_to, cp_from)
				else:
					link_copy(cp_from, cp_to)
					log_links.info('Copy: %s %s %s', node, cp_dir, dst)
				if nnn.inc_errs(): break

	## Reverse check that all files are listed in links, incorrectable
	for p_dir, dirs, files in os.walk(root):
		if nnn.errs_limit: break
		p_dir = pl.Path(p_dir)
		for fn in files:
			p = (src := p_dir / fn).relative_to(root)
			if (ns := str(p)) in linked_nodes: continue
			if any(chk.search(ns) for chk in exc): continue
			node = pl.Path('')
			for s in p.parts: # path dir(s) also not linked
				if str(node := node / s) in linked_nodes: break
			else:
				log_links.warning('Not in links-list: %s', src.relative_to(root))
				if nnn.inc_errs(): break

def link_copy_new(root, src, dst_list, links_hash, lt, meta, dst_home):
	if root_new := links_hash.get('.'):
		if not isinstance(root_new, str) and (lt := root_new[0]) & lt.sym:
			if root.resolve().samefile(root_new := root_new[1].expanduser()):
				root = root_new
	lines, links_replace = list(), list()

	src_rel = src.resolve().relative_to(root.resolve())
	if src_rel.is_dir(): node_func = lambda dst: src_rel / dst.name
	elif len(dst_list) > 1:
		return log_links.critical('Destination must be a dir w/ multiple new links')
	else: node_func = lambda dst: src_rel

	for dst in dst_list:
		dst_chk = [dst, dst := (dst_res := dst.resolve())]
		if dst_home:
			try: dst_chk.append(dst := pl.Path('~') / dst.relative_to(dst_home))
			except ValueError: pass
		dst_chk.append(node := node_func(dst))
		dst_chk.append(src := root / node)
		dst_chk.append(line := f'{node} {LinkTypeSep[lt]} {dst}')
		for p in dst_chk:
			if str(p) not in links_hash: continue
			return log_links.critical('Path/line already in links-list: %s', p)
		if src.exists():
			return log_links.critical('Destination path already exists: %s', src)
		link_copy(dst_res, src, meta)
		if lt & lt.sym:
			link = src if lt != lt.rsym else link_relative_path(src, dst_res)
			links_replace.append((src, dst_res, link))
		print(line)
		lines.append(line)

	for src, dst, link in links_replace:
		for n in range(50):
			ext = base64.urlsafe_b64encode(os.urandom(3)).decode()
			if not (tmp := dst.with_name(dst.name + f'.tmp.{ext}')).exists(): break
		else: raise RuntimeError(f'Failed to find tmp-path for: {dst}')
		link_sym_with_uidgid(tmp, link, meta)
		if not tmp.samefile(src):
			tmp.unlink()
			raise RuntimeError(f'Symlink samefile-check failed: {tmp} [ {link} ] != {src}')
		tmp.rename(dst)
	return lines



def main(args=None):
	import argparse, textwrap
	dd = lambda text: re.sub( r' \t+', ' ',
		textwrap.dedent(text).strip('\n') + '\n' ).replace('\t', '  ')
	parser = argparse.ArgumentParser(
		formatter_class=argparse.RawTextHelpFormatter,
		usage='%(prog)s [options]', description=dd('''
			Tool to check and fix file symlink and permission lists for files
				in a current dir, e.g. maintaining links into a git configuration repository.
			It maintains and applies two separate manifest-files -
				one for symlinks and file-copies, and another one for metadata on files.
			"dir" and "scim-path" in descriptions and messages refers to "source" files
					under current dir, or an upper one where manifest files are found (similar to git).
				Manifest-files can refer to paths relative to this directory.
			"fs" or "live-fs" refers to files linking into it or deployed/copied from it.'''))

	group = parser.add_argument_group('Main output/operation modes')
	group.add_argument('-c', '--check', action='store_true', help=dd('''
		Quietly check if there is any inconsistency between
			manifest-files and filesystem, indicating it by returning error code 39.'''))
	group.add_argument('-s', '--summary', action='store_true',
		help='Do not print individual diffs, just the summary and number of them.')
	group.add_argument('-f', '--fix', action='store_true', help=dd('''
		Fix detected errors.
		In case of links, its the update of a source from a destination or dst-link creation.
		In case of file-copies (if there are diffs), live-fs (dst) file will overwrite source.
		Metadata is restored for known files, then list is updated with new/deleted ones.'''))
	group.add_argument('-n', '--limit', type=int, metavar='n',
		help='Process only specified number of diffs from the top of the manifest file(s).')

	group = parser.add_argument_group('Creating new link(s)')
	group.add_argument('-L', '--ln', metavar='src/dst', nargs='*', help=dd('''
		Create new link entry(-ies) in link-list file and copy/symlink file(s) and exit.
		Should be followed by "source... [destination]" argument(s), with
			destination being either new path to create or a directory, similar to how "ln" works.
		Can be used without follow-up arguments to sort the links in-place.
		Most other options are not used with this one, except for ones
			in this group, -m/--meta, links/metadata list file paths and logging tweaks.
		Using -m/--meta (default with uid=root) will copy/preserve all metadata for file(s).
		Default destination is current directory, if omitted.'''))
	group.add_argument('--symlink', action='store_true',
		help='Create symlink instead of a copy. Alias: -l (lowercase L)')
	group.add_argument('--relative', action='store_true',
		help='Create relative symlink (implies -l/--symlink). Alias: -r')
	group.add_argument('--no-tilde', action='store_true',
		help='Do not replace current user\'s homedir with ~ in dst path.')

	group = parser.add_argument_group('Opts to apply diffs in reverse')
	group.add_argument('-r', '--reverse', action='store_true',
		help='Reverse direction of a fixes, both for links and metadata.')
	group.add_argument('-w', '--reverse-meta', action='store_true', help=dd('''
		Reverse direction of a metadata fixes, so live-fs uid/gid/mode and
			xattrs will be applied to repository files and fixed in meta-manifest file.
		This is implied in -r/--reverse, and redundant with that option.'''))

	group = parser.add_argument_group('Limit operation to only links or metadata')
	group.add_argument('-l', '--links', dest='links_on', action='store_true', help=dd('''
		Change stuff related to links.
		Enabled/implied by default, if neither -l/--links or -m/--meta are specified.'''))
	group.add_argument('-b', '--links-broken', action='store_true', help=dd('''
		Only process links that are "broken" in some way
			without any diff involved - e.g. "Not a symlink" cases. Implies -l/--links.'''))
	group.add_argument('--links-meta', action='store_true', help=dd('''
		Sync metadata for linked files (dir/repo -> live-fs, unless reversed).
		Implies -l/--links, but does not involve -m/--meta and --meta-list in any way.'''))
	group.add_argument('-m', '--meta', dest='meta_on', action='store_true', help=dd('''
		Run metadata-related manipulations - enforce/update --meta-list.
		Implied if none of -l/--links or -m/--meta are specified explicitly,
			except when user has no ability to manipulate file
			ownership (e.g. non-root) and meta-list file does not exists.'''))

	group = parser.add_argument_group('Path filtering')
	group.add_argument('-p', '--path-match', metavar='path', action='append', help=dd('''
		Only process entries that match specified local path.
		Path spec can include python/shell glob/fnmatch wildcards, with ** matching slashes.
		Option can be used multiple times to include all paths matched by any pattern.'''))

	group = parser.add_argument_group('Manifest file names/paths')
	group.add_argument('--links-list', dest='links',
		metavar='name/path', default='.scim_links', help=dd('''
			Either basename (which will be searched upwards from cwd)
				or full path to links-list manifest file. Default: %(default)s'''))
	group.add_argument('--links-exclude', dest='links_exc',
		metavar='name/path', default='.scim_links_exclude', help=dd('''
			Either basename or full path of links-exclude file.
			This is a file with regexp patters (python re) to ignore when
				checking for files not yet in the links-list manifest, kinda like gitignore.
			Should be in the same directory as links-list file. Default: %(default)s'''))
	group.add_argument('--meta-list', dest='meta',
		metavar='name/path', default='.scim_meta', help=dd('''
			Either basename or full path of metadata manifest file.
			Should be in the same path as links-list. Default: %(default)s'''))
	group.add_argument('--meta-exclude', dest='meta_exc',
		metavar='name/path', default='.scim_meta_exclude', help=dd('''
			Either basename or full path of metadata-exclude file.
			Should be in the same path as links-list. Default: %(default)s'''))

	group = parser.add_argument_group('Misc other tweaks')
	group.add_argument('--diff-cmd',
		default='git --no-pager diff -aw --minimal --color --no-index --',
		metavar='command', help=dd('''
			Command to use for generating diffs in case of a link mismatch.
			Arguments are split on spaces, old/new file paths are appended at the end.
			Default: %(default)r'''))
	group.add_argument('--diff-cmd2',
		default='git --no-pager diff -a --minimal --color --no-index --',
		metavar='command', help=dd('''
			Same as --diff-cmd, but run if former exits with 0, indicating no diff.
			Example use is to have first cmd skip spaces, and second one to show them,
				so that whitespace diffs are only displayed if there's nothing else different.
			Can be set to empty string or "-" to disable it.
			Default: %(default)r'''))
	group.add_argument('--numeric-ids', action='store_true',
		help='Use numeric ids instead of user/group names everywhere.')
	group.add_argument('--debug', action='store_true', help='Log info on what is going on.')

	opts = parser.parse_args(sys.argv[1:] if args is None else args)

	if opts.check: log_level = logging.CRITICAL
	elif opts.debug: log_level = logging.DEBUG
	else: log_level = logging.INFO
	for old, new in dict( DEBUG='debug', INFO='info',
			WARNING='warn', ERROR='err', CRITICAL='fatal' ).items():
		logging.addLevelName(getattr(logging, old), new)
	logging.basicConfig( level=log_level,
		format='[{name} {levelname:5}] {message}', style='{' )
	global log, log_meta, log_links
	log_meta = logging.getLogger('meta')
	log_links = logging.getLogger('links')

	conf = adict( fix=opts.fix and not opts.check,
		nnn=ChangeCount(errs=opts.limit if not opts.check else 1) )

	mf_links = pl.Path(opts.links)
	if mf_links.is_absolute():
		root, mf_links = mf_links.parent, pl.Path(mf_links.name)
	else:
		root = pl.Path('.').resolve()
		while not (root / mf_links).exists():
			if str(root) == root.root:
				parser.error('Unable to determine scim-path location')
			root = root.parent
	conf.root = root.resolve(True)

	conf.lists = list()
	for k in 'links', 'links_exc', 'meta', 'meta_exc':
		if conf.get(k.split('_', 1)[0]) is False: continue
		p = getattr(opts, k)
		if (p_abs := (root / p).resolve()).parent != root:
			parser.error(
				'All manifest/exclude-files should be in the'
				f' same scim-path: [ {p} ] is not in [ {root} ]' )
		lsf = conf[k] = ListFile(root, p_abs)
		conf.lists.append(lsf.node)

	if opts.ln is not None:
		opts.symlink, opts.relative, opts.links_on = (
			opts.symlink or opts.links_on, opts.relative or opts.reverse, None )
	if not opts.links_on and not opts.meta_on:
		if not conf.meta.exists():
			if os.geteuid() != 0: conf.meta = False
	else:
		if not ( opts.links_on or opts.ln or
			opts.links_meta or opts.links_broken ): conf.links = False
		if not opts.meta_on: conf.meta = False
	conf.meta_uidgid = meta_uidgid_resolver(opts.numeric_ids)

	if opts.ln is not None:
		lines_new, lines = list(), conf.links.lines()
		if opts.ln:
			if len(dst_list := list(opts.ln)) == 1: dst_list.append('.')
			dst_list = list(map(pl.Path, dst_list))
			dst_list, src = dst_list[:-1], dst_list[-1]
			for dst in dst_list:
				if dst.is_symlink(): err = 'Specified path is already a symlink'
				elif dst.is_dir(): err = 'Directory links not supported'
				else: continue
				log_links.critical('%s: %s', err, dst); return 1
			links_hash = link_list_hash(lines)
			lt = ( LinkType.rsym if opts.relative else
				(LinkType.sym if opts.symlink else LinkType.cp) )
			try:
				if opts.no_tilde: raise ValueError
				import pwd
				dst_home = pl.Path(pwd.getpwuid(os.getuid()).pw_dir).resolve()
			except: dst_home = None
			lines_new = link_copy_new(
				conf.root, src, dst_list, links_hash,
				lt=lt, meta=conf.meta, dst_home=dst_home )
			if not lines_new: return 1
		conf.links.update(sorted([*lines, *lines_new]))
		return log_links.info('Links-list updated: %d change(s)', len(lines_new))

	if pat := opts.path_match:
		import fnmatch
		conf.path_filter = list(re.compile( fnmatch
				.translate(pat.replace('**', '\ue017').replace('**', '\ue018'))
				.replace('\ue017', '.*').replace('\ue018', '[^/]*') ) for pat in pat)
	else: conf.path_filter = [re.compile('')]

	if conf.meta:
		conf.meta_exc = list(re.compile(ln) for ln in conf.meta_exc.lines() if ln)
		conf.meta_update_list = opts.reverse or opts.reverse_meta
		conf.meta_update_n = 0
		conf.meta_map = meta_map_make(
			conf.root, conf.meta.lines(),
			conf.meta_exc, conf.nnn, conf.meta_uidgid )

		def _meta_update():
			meta_map_update(
				conf.root, conf.meta_map,
				conf.path_filter, conf.nnn, conf.meta_uidgid,
				fix=conf.fix, update_list=conf.meta_update_list )
			if conf.fix and conf.nnn.meta:
				if not (meta_node_exists := conf.meta_map.get(conf.meta.node)):
					user, group = conf.meta_uidgid(os.getuid(), os.getgid())
					meta = conf.meta_map[conf.meta.node] = f'{user}:{group}:600'
					conf.nnn.inc_meta()
				conf.meta.update( f'{node} {meta}'
					for node, meta in sorted(conf.meta_map.items()) )
				if not meta_node_exists:
					meta_set(conf.meta.node, conf.meta_uidgid, meta, conf.root)
				log_meta.info('Metadata list updated: %d change(s)', conf.nnn.meta)

		# Meta-list update has to be delayed if it's replaced from
		#  file permissions, as these will have to be copied into repo first
		if not conf.meta_update_list: _meta_update()

	if conf.links:
		conf.links_exc = list(re.compile(ln) for ln in conf.links_exc.lines() if ln)
		conf.links_exc.extend(re.compile(
			f'^{re.escape(str(node))}$' ) for node in conf.lists)
		if opts.summary or opts.check: conf.diff_cmd = None
		else:
			conf.diff_cmd = [opts.diff_cmd.split()]
			if opts.diff_cmd2 and opts.diff_cmd2 not in [opts.diff_cmd, '-']:
				conf.diff_cmd.append(opts.diff_cmd2.split())
		link_list_enforce(
			conf.root, conf.links.lines(), conf.links_exc,
			conf.diff_cmd, conf.path_filter, conf.nnn, conf.meta_uidgid,
			fix=conf.fix, meta=conf.meta or opts.links_meta,
			no_content=opts.links_broken,
			content_from_repo=opts.reverse,
			meta_to_repo=opts.reverse or opts.reverse_meta )

	if conf.meta and conf.meta_update_list: _meta_update() # after copying files/perms

	if conf.nnn.errs and opts.summary:
		log.info( f'Total {"things to fix" if not conf.fix else "fixes"}'
			f' = {conf.nnn.errs:,d} (metadata quirks = {conf.nnn.meta:,d})' )

	if opts.check and conf.nnn.errs: return 39
	return bool(conf.nnn.errs) if not conf.fix else bool(conf.nnn.errs_fix)

if __name__ == '__main__': sys.exit(main())