From 713a73967baa8de60d7a1c002829b77c2168a91f Mon Sep 17 00:00:00 2001 From: melis_m Date: Thu, 20 Jun 2019 16:58:14 +0200 Subject: [PATCH 01/37] Start revamp, atm only take npf(s) and unwraps them --- core/__init__.py | 0 core/args.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ core/package.py | 31 ++++++++++++++++++++++++++++ npf-checker.py | 16 +++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 core/__init__.py create mode 100644 core/args.py create mode 100644 core/package.py create mode 100755 npf-checker.py diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/args.py b/core/args.py new file mode 100644 index 0000000..93d9fb2 --- /dev/null +++ b/core/args.py @@ -0,0 +1,53 @@ +import argparse +import os +import sys + +global _args +global _parser + + +def parse_args(): + global _args + global _parser + + _parser = argparse.ArgumentParser( + description="A checker that ensure the validity and conformance of an NPF", + ) + _parser.add_argument( + 'npf', + nargs='+', + ) + _parser.add_argument( + '-c', + '--cache-dir', + default=os.path.join( + os.getcwd(), + os.path.dirname(sys.argv[0]), + 'cache', + ), + help="Cache directory used to unpack and check the NPF. Default: cache/", + ) + _parser.add_argument( + '-o', + '--output', + default=None, + help="Output NPF file, by default it replaces the given one", + ) + _parser.add_argument( + '--backup', + default=None, + help="If no -o/--output is given, this creates a backup, default is {npf}.bak", + ) + _args = _parser.parse_args() + _args.output = _args.output or _args.npf + _args.backup = _args.backup or f"{_args.npf}.bak" + + +def get_args(): + global _args + return _args + + +def get_parser(): + global _parser + return _parser diff --git a/core/package.py b/core/package.py new file mode 100644 index 0000000..5375f61 --- /dev/null +++ b/core/package.py @@ -0,0 +1,31 @@ +import shutil +import os +import tarfile +import core +import enum +import toml + + +class Kind(enum.Enum): + EFFECTIVE = 'effective' + VIRTUAL = 'virtual' + + +class Package: + def __init__(self, npf_path): + self.cache = core.args.get_args().cache_dir + self.npf_path = npf_path + + def unwrap(self): + if os.path.exists(self.cache): + shutil.rmtree(self.cache) + os.makedirs(self.cache) + with tarfile.open(self.npf_path, 'r') as tar: + tar.extractall(self.cache) + self.manifest_path = os.path.join(self.cache, 'manifest.toml') + self.manifest = toml.load(self.manifest_path) + data = os.path.join(self.cache, 'data.tar.gz') + if os.path.exists(data): + with tarfile.open(data, 'r:gz') as tar: + tar.extractall(self.cache) + os.remove(data) diff --git a/npf-checker.py b/npf-checker.py new file mode 100755 index 0000000..d8f8ff7 --- /dev/null +++ b/npf-checker.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import core.args +import core.package + + +def main(): + core.args.parse_args() + args = core.args.get_args() + for f in args.npf: + pkg = core.package.Package(f) + pkg.unwrap() + + +if __name__ == '__main__': + main() From ea2baec6c1e3701f1d4548f4793e63ee1284b082 Mon Sep 17 00:00:00 2001 From: melis_m Date: Thu, 20 Jun 2019 17:50:09 +0200 Subject: [PATCH 02/37] Add log functions (from nbuild + qlog) --- core/log.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 core/log.py diff --git a/core/log.py b/core/log.py new file mode 100644 index 0000000..7098181 --- /dev/null +++ b/core/log.py @@ -0,0 +1,90 @@ +""" +Functions to write and manipulate logs. +""" + +import termcolor +from contextlib import contextmanager + +log_tab_level = 0 + + +@contextmanager +def push(): + """Increase the log indentation level by one, making every new line indented by one extra tabulation.""" + global log_tab_level + log_tab_level += 1 + + try: + yield + finally: + log_tab_level -= 1 + + +def d(*logs: str): + """Print a debug log, prefixed by a magenta ``[d]``. + :param logs: The content of the log. + """ + global log_tab_level + + indent = ' ' * log_tab_level + print(f"{termcolor.colored('[d]', 'magenta', attrs=['bold'])} {indent}", *logs) + + +def i(*logs: str): + """Print an informative log, prefixed by a blue ``[*]``. + :param logs: The content of the log. + """ + global log_tab_level + + indent = ' ' * log_tab_level + print(f"{termcolor.colored('[*]', 'blue', attrs=['bold'])} {indent}", *logs) + + +def s(*logs: str): + """Print a success log, prefixed by a green ``[+]``. + :param logs: The content of the log. + """ + global log_tab_level + + indent = ' ' * log_tab_level + print(f"{termcolor.colored('[+]', 'green', attrs=['bold'])} {indent}", *logs) + + +def w(*logs: str): + """Print a warning log, prefixed by a yellow ``[!]``. + :param logs: The content of the log. + """ + global log_tab_level + + indent = ' ' * log_tab_level + print(f"{termcolor.colored('[!]', 'yellow', attrs=['bold'])} {indent}", *logs) + + +def e(*logs: str): + """Print a non-fatal error log, prefixed by a red ``[-]``. + :param logs: The content of the log. + """ + global log_tab_level + + indent = ' ' * log_tab_level + print(f"{termcolor.colored('[-]', 'red', attrs=['bold'])} {indent}", *logs) + + +def f(*logs: str): + """Print a fatal error log, all in red, prefixed by ``[-]``. + :info: This function does NOT abort the current process's execution. + :param logs: The content of the log. + """ + termcolor.cprint(f"[-] {' '.join(logs)}", 'red', attrs=['bold']) + + +def q(*logs: str): + """Print a question, prefixed by a yellow ``[?]`` and waits for user input. + :param logs: The content of the log. + :returns: The string returned by ``input()`` + """ + global log_tab_level + + indent = '\t' * log_tab_level + print(f"{termcolor.colored('[?]', 'yellow')} {indent}", *logs, end='') + return input() From 22be1a1c44aba60158dbad0d91f38255c8c64f50 Mon Sep 17 00:00:00 2001 From: melis_m Date: Thu, 20 Jun 2019 17:52:38 +0200 Subject: [PATCH 03/37] Remove Kind enum --- core/package.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/package.py b/core/package.py index 5375f61..0ed918a 100644 --- a/core/package.py +++ b/core/package.py @@ -2,15 +2,9 @@ import os import tarfile import core -import enum import toml -class Kind(enum.Enum): - EFFECTIVE = 'effective' - VIRTUAL = 'virtual' - - class Package: def __init__(self, npf_path): self.cache = core.args.get_args().cache_dir From 388406f6cedd79cd1e936a957f96b5b5b4a8235b Mon Sep 17 00:00:00 2001 From: melis_m Date: Thu, 20 Jun 2019 17:54:09 +0200 Subject: [PATCH 04/37] Add tarball/npf wrap function, no updates of the manifest.toml yet --- core/__init__.py | 1 + core/package.py | 36 ++++++++++++++++++++++++++++++++++++ npf-checker.py | 2 ++ 3 files changed, 39 insertions(+) diff --git a/core/__init__.py b/core/__init__.py index e69de29..6b43a5e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -0,0 +1 @@ +from core.pushd import pushd diff --git a/core/package.py b/core/package.py index 0ed918a..9660598 100644 --- a/core/package.py +++ b/core/package.py @@ -9,6 +9,7 @@ class Package: def __init__(self, npf_path): self.cache = core.args.get_args().cache_dir self.npf_path = npf_path + self.is_effective = False def unwrap(self): if os.path.exists(self.cache): @@ -18,8 +19,43 @@ def unwrap(self): tar.extractall(self.cache) self.manifest_path = os.path.join(self.cache, 'manifest.toml') self.manifest = toml.load(self.manifest_path) + self.is_effective = self.manifest['kind'] == 'effective' data = os.path.join(self.cache, 'data.tar.gz') if os.path.exists(data): with tarfile.open(data, 'r:gz') as tar: tar.extractall(self.cache) os.remove(data) + + def check(self): + print('TODO: Package.check()') + + def wrap(self): + self.create_manifest_toml() + if self.is_effective: + self.create_data_tar() + self.create_nest_file() + + def create_manifest_toml(self): + print('TODO: Package.create_manifest_toml()') + pass + + def create_data_tar(self): + with core.pushd(self.cache): + for root, _, filenames in os.walk('.'): + for name in filenames: + print(os.path.join(root, name)) + with tarfile.open('data.tar.gz', 'w:gz') as archive: + archive.add('./') + + def create_nest_file(self): + with core.pushd(self.cache): + new_nest_file_path = os.path.basename(self.npf_path) + '.new' + with tarfile.open(new_nest_file_path, 'w') as nest_file: + nest_file.add('manifest.toml') + if self.is_effective: + nest_file.add('data.tar.gz') + os.remove('manifest.toml') + if self.is_effective: + os.remove('data.tar.gz') + new_path = f'{self.npf_path}.new' + os.rename(os.path.join(self.cache, new_nest_file_path), new_path) diff --git a/npf-checker.py b/npf-checker.py index d8f8ff7..025d774 100755 --- a/npf-checker.py +++ b/npf-checker.py @@ -10,6 +10,8 @@ def main(): for f in args.npf: pkg = core.package.Package(f) pkg.unwrap() + pkg.check() + pkg.wrap() if __name__ == '__main__': From fdb3be8cfad4a5711b930f11105a65f49e993962 Mon Sep 17 00:00:00 2001 From: melis_m Date: Mon, 24 Jun 2019 16:02:13 +0200 Subject: [PATCH 05/37] Remove args.{output,backup}, as they're not used yet --- core/args.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/core/args.py b/core/args.py index 93d9fb2..6473e4d 100644 --- a/core/args.py +++ b/core/args.py @@ -27,20 +27,7 @@ def parse_args(): ), help="Cache directory used to unpack and check the NPF. Default: cache/", ) - _parser.add_argument( - '-o', - '--output', - default=None, - help="Output NPF file, by default it replaces the given one", - ) - _parser.add_argument( - '--backup', - default=None, - help="If no -o/--output is given, this creates a backup, default is {npf}.bak", - ) _args = _parser.parse_args() - _args.output = _args.output or _args.npf - _args.backup = _args.backup or f"{_args.npf}.bak" def get_args(): From 8c5ff82a4c849593d7b1732819b7822df5eb1771 Mon Sep 17 00:00:00 2001 From: melis_m Date: Mon, 24 Jun 2019 16:03:12 +0200 Subject: [PATCH 06/37] Add function to refresh date of manifest wrap_date --- core/package.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/package.py b/core/package.py index 9660598..025fdc6 100644 --- a/core/package.py +++ b/core/package.py @@ -1,8 +1,9 @@ import shutil import os import tarfile -import core import toml +import datetime +import core class Package: @@ -30,14 +31,15 @@ def check(self): print('TODO: Package.check()') def wrap(self): - self.create_manifest_toml() + self.update_manifest_toml_wrap_date() if self.is_effective: self.create_data_tar() self.create_nest_file() - def create_manifest_toml(self): - print('TODO: Package.create_manifest_toml()') - pass + def update_manifest_toml_wrap_date(self): + self.manifest['wrap_date'] = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + with open(self.manifest_path, 'w') as filename: + toml.dump(self.manifest, filename) def create_data_tar(self): with core.pushd(self.cache): From 4e39c7180f86f3d7c2105b40c951bb1bb55fc32f Mon Sep 17 00:00:00 2001 From: melis_m Date: Mon, 24 Jun 2019 18:02:34 +0200 Subject: [PATCH 07/37] Add log to data.tar.gz creation and manifest.toml --- core/package.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++--- npf-checker.py | 4 ++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/core/package.py b/core/package.py index 025fdc6..09e82c4 100644 --- a/core/package.py +++ b/core/package.py @@ -3,7 +3,9 @@ import tarfile import toml import datetime +import termcolor import core +import core.log as log class Package: @@ -32,6 +34,7 @@ def check(self): def wrap(self): self.update_manifest_toml_wrap_date() + self.show_manifest() if self.is_effective: self.create_data_tar() self.create_nest_file() @@ -43,11 +46,21 @@ def update_manifest_toml_wrap_date(self): def create_data_tar(self): with core.pushd(self.cache): - for root, _, filenames in os.walk('.'): - for name in filenames: - print(os.path.join(root, name)) + self.manifest = toml.load('manifest.toml') + os.remove('manifest.toml') + files_count = 0 + log.s("Files added:") + with log.push(): + for root, _, filenames in os.walk('.'): + for name in filenames: + log.s(os.path.join(root, name)) + files_count += 1 + log.s(f"(That's {files_count} files.)") + log.s(f"Creating data.tar.gz") with tarfile.open('data.tar.gz', 'w:gz') as archive: archive.add('./') + with open('manifest.toml', 'w') as filename: + toml.dump(self.manifest, filename) def create_nest_file(self): with core.pushd(self.cache): @@ -61,3 +74,44 @@ def create_nest_file(self): os.remove('data.tar.gz') new_path = f'{self.npf_path}.new' os.rename(os.path.join(self.cache, new_nest_file_path), new_path) + + def show_manifest(self): + m = self.manifest + metadata = m['metadata'] + log.s(f"Manifest:") + with log.push(): + log.s(f"name: {m['name']}") + log.s(f"category: {m['category']}") + log.s(f"version: {'version'}") + log.s(f"description: {metadata['description']}") + log.s(f"tags: {', '.join(metadata['tags'])}") + log.s(f"maintainer: {metadata['maintainer']}") + log.s(f"licenses: {', '.join(metadata['licenses'])}") + log.s(f"upstream_url: {metadata['upstream_url']}") + log.s(f"kind: {m['kind']}") + log.s(f"wrap_date: {datetime.datetime.utcnow().replace(microsecond=0).isoformat() + 'Z'}") + log.s(f"dependencies:") + with log.push(): + for (full_name, version_req) in m['dependencies'].items(): + log.s(f"{full_name}#{version_req}") + + +def _colored_path(path, pretty_path=None): + if pretty_path is None: + pretty_path = path + + if os.path.islink(path): + target_path = os.path.join( + os.path.dirname(path), + os.readlink(path), + ) + if os.path.exists(target_path): + return f"{termcolor.colored(path, 'cyan', attrs=['bold'])} -> {_colored_path(target_path, os.readlink(path))}" + else: + return f"{termcolor.colored(path, on_color='on_red', attrs=['bold'])} -> {termcolor.colored(os.readlink(path), on_color='on_red', attrs=['bold'])}" + elif os.path.isdir(path): + return termcolor.colored(pretty_path, 'blue', attrs=['bold']) + elif os.access(path, os.X_OK): + return termcolor.colored(pretty_path, 'green', attrs=['bold']) + else: + return pretty_path diff --git a/npf-checker.py b/npf-checker.py index 025d774..58ef148 100755 --- a/npf-checker.py +++ b/npf-checker.py @@ -2,6 +2,7 @@ import core.args import core.package +import core.log as log def main(): @@ -9,8 +10,11 @@ def main(): args = core.args.get_args() for f in args.npf: pkg = core.package.Package(f) + log.s(f"Unwrapping {f}") pkg.unwrap() + log.s(f"Checking {f}") pkg.check() + log.s(f"Wrapping {f}") pkg.wrap() From 230cd2f99e2c0042ab2b2735b042211ac1156a88 Mon Sep 17 00:00:00 2001 From: melis_m Date: Fri, 28 Jun 2019 14:13:48 +0200 Subject: [PATCH 08/37] Add options to args to choose from fix/edit/diff --- core/args.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/args.py b/core/args.py index 6473e4d..ca31aac 100644 --- a/core/args.py +++ b/core/args.py @@ -27,6 +27,11 @@ def parse_args(): ), help="Cache directory used to unpack and check the NPF. Default: cache/", ) + _parser.add_argument( + '--action', + choices=['edit', 'fix', 'diff'], + default='edit', + ) _args = _parser.parse_args() From a90ad70c71aa4627a7fb6e1ced683d120e34ae83 Mon Sep 17 00:00:00 2001 From: melis_m Date: Fri, 28 Jun 2019 14:25:27 +0200 Subject: [PATCH 09/37] Add .gitignore --- .gitignore | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b9cac1d --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +*.nest +*.nest.new +*.toml +cache/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ From f2adf6ee2af380540c4e0640fddb6d13439ba75e Mon Sep 17 00:00:00 2001 From: melis_m Date: Fri, 28 Jun 2019 17:43:06 +0200 Subject: [PATCH 10/37] Add .gitignore and definition of core.pushd --- .gitignore | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++ core/pushd.py | 12 +++++ 2 files changed, 141 insertions(+) create mode 100644 .gitignore create mode 100644 core/pushd.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b9cac1d --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +*.nest +*.nest.new +*.toml +cache/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/core/pushd.py b/core/pushd.py new file mode 100644 index 0000000..65a6c87 --- /dev/null +++ b/core/pushd.py @@ -0,0 +1,12 @@ +import os +import contextlib + + +@contextlib.contextmanager +def pushd(path: str = '.'): + old_path = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(old_path) From 840963341dd3c0e6f4727e41acc567fc0e376afa Mon Sep 17 00:00:00 2001 From: melis_m Date: Fri, 28 Jun 2019 14:43:43 +0200 Subject: [PATCH 11/37] Add basis for checks --- core/args.py | 5 +++ core/check.py | 2 + core/checks/base.py | 63 ++++++++++++++++++++++++++++++ core/checks/utils.py | 91 ++++++++++++++++++++++++++++++++++++++++++++ core/package.py | 3 +- core/pushd.py | 12 ++++++ npf-checker.py | 2 + 7 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 core/check.py create mode 100644 core/checks/base.py create mode 100644 core/checks/utils.py create mode 100644 core/pushd.py diff --git a/core/args.py b/core/args.py index ca31aac..e6b3144 100644 --- a/core/args.py +++ b/core/args.py @@ -32,6 +32,11 @@ def parse_args(): choices=['edit', 'fix', 'diff'], default='edit', ) + _parser.add_argument( + '--visual', + action='store_true', + help="Try to use more visual tools (only makes sense with --action=edit, which is the default)" + ) _args = _parser.parse_args() diff --git a/core/check.py b/core/check.py new file mode 100644 index 0000000..16834ec --- /dev/null +++ b/core/check.py @@ -0,0 +1,2 @@ +def check_package(package): + print('TODO: call checks') diff --git a/core/checks/base.py b/core/checks/base.py new file mode 100644 index 0000000..7d43ac7 --- /dev/null +++ b/core/checks/base.py @@ -0,0 +1,63 @@ +import enum +import core +import core.args +import core.log as log +import core.checks.utils as utils + + +class Type(enum.Enum): + FIX = enum.auto() + DIFF = enum.auto() + EDIT = enum.auto() + + @staticmethod + def from_string(s: str): + if s == 'fix': + return Type.FIX + elif s == 'edit': + return Type.EDIT + elif s == 'diff': + return Type.DIFF + + +class Check(): + global_state = None + + def __init__(self, items): + self.items = items + + def run(self): + with log.push(), core.pushd(core.args.get_args().cache_dir): + for item in self.items: + if not self.validate(item): + self.show(item) + if Check.global_state is Type.FIX: + self.fix(item) + elif Check.global_state is Type.DIFF: + self.diff(item) + elif Check.global_state is Type.EDIT: + log.i("The automatic changes would be as follows") + if self.diff(item) is not False: + answer = utils.ask_yne("Accept those changes? ") + if answer is utils.Answer.YES: + self.fix(item) + elif answer == utils.Answer.EDIT: + self.edit(item) + else: + if utils.ask_yn("Edit manually? "): + self.edit(item) + + def validate(self, item): + raise NotImplementedError + + def show(self, item): + raise NotImplementedError + + def fix(self, item): + raise NotImplementedError + + def diff(self, item): + raise NotImplementedError + + def edit(self, item): + raise NotImplementedError diff --git a/core/checks/utils.py b/core/checks/utils.py new file mode 100644 index 0000000..56bb8d2 --- /dev/null +++ b/core/checks/utils.py @@ -0,0 +1,91 @@ +import enum +import os +import braceexpand +import glob +import core.log as log +import core.args + + +class Answer(enum.Enum): + YES = enum.auto() + NO = enum.auto() + EDIT = enum.auto() + NONE = enum.auto() + + +def ask_yne(question, default=True): + while True: + answer = log.q(question + '[Y/n/e] ').lower() + if answer == '': + return default + elif answer in ['y', 'yes', 'ye']: + return Answer.YES + elif answer in ['n', 'no']: + return Answer.NO + elif answer in ['e', 'edit']: + return Answer.EDIT + else: + log.w("Unrecognized answer") + continue + + +def ask_yn(question, default=True): + while True: + answer = log.q(question + '[Y/n] ').lower() + if answer == '': + return default + elif answer in ['y', 'yes', 'ye']: + return True + elif answer in ['n', 'no']: + return False + else: + log.w("Unrecognized answer") + continue + + +def open_shell(path): + args = core.args.get_args() + if args.visual: + ret = os.system(f'xdg-open {path}') + if ret == 0: + return + else: + log.w("A problem occured while using xdg-open, falling back on opening a shell") + shell = os.environ.get('SHELL') + if shell is None: + log.w("No $SHELL environment variable found") + shell = log.q("Please provide a valid shell: ") + log.i(f"Opening {shell} in {path}, press CTRL-D to exit and resume checks") + os.system(f'cd {path} && {shell}') + + +def open_editor(filepath): + args = core.args.get_args() + if args.visual: + editor = os.environ.get('VISUAL') + if editor is None: + log.w("No $VISUAL environment variable found, trying $EDITOR") + editor = os.environ.get('EDITOR') + if editor is None: + log.w("No $EDITOR environment variable found") + editor = log.q("Please provide a valid editor: ") + else: + editor = os.environ.get('EDITOR') + if editor is None: + log.w("No $EDITOR environment variable found") + editor = log.q("Please provide a valid editor: ") + log.i(f"Opening {filepath} with {editor}") + os.system(f'{editor} {filepath}') + + +def find_files(*paths): + files = [] + cache = core.args.get_args().cache_dir + with core.pushd(cache): + for path in paths: + for rglob in braceexpand.braceexpand(path): + if os.path.isabs(rglob): + raise RuntimeError(f"Cannot receive absolute path '{rglob}'") + for rpath in glob.glob(rglob, recursive=True): + files.append(rpath) + return files diff --git a/core/package.py b/core/package.py index 09e82c4..539e1a2 100644 --- a/core/package.py +++ b/core/package.py @@ -6,6 +6,7 @@ import termcolor import core import core.log as log +import core.check class Package: @@ -30,7 +31,7 @@ def unwrap(self): os.remove(data) def check(self): - print('TODO: Package.check()') + core.check.check_package(self) def wrap(self): self.update_manifest_toml_wrap_date() diff --git a/core/pushd.py b/core/pushd.py new file mode 100644 index 0000000..65a6c87 --- /dev/null +++ b/core/pushd.py @@ -0,0 +1,12 @@ +import os +import contextlib + + +@contextlib.contextmanager +def pushd(path: str = '.'): + old_path = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(old_path) diff --git a/npf-checker.py b/npf-checker.py index 58ef148..a0d3100 100755 --- a/npf-checker.py +++ b/npf-checker.py @@ -3,11 +3,13 @@ import core.args import core.package import core.log as log +import core.checks.base as base def main(): core.args.parse_args() args = core.args.get_args() + base.Check.global_state = base.Type.from_string(core.args.get_args().action) for f in args.npf: pkg = core.package.Package(f) log.s(f"Unwrapping {f}") From 0c0740cf236ccd6cb6389e793040588060d83abb Mon Sep 17 00:00:00 2001 From: melis_m Date: Fri, 28 Jun 2019 15:58:50 +0200 Subject: [PATCH 12/37] Add executable checks on shlibs/binaries --- core/check.py | 5 ++++- core/checks/executable.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 core/checks/executable.py diff --git a/core/check.py b/core/check.py index 16834ec..38783b3 100644 --- a/core/check.py +++ b/core/check.py @@ -1,2 +1,5 @@ +import core.checks.executable as exe + + def check_package(package): - print('TODO: call checks') + exe.ExecCheck(package).run() diff --git a/core/checks/executable.py b/core/checks/executable.py new file mode 100644 index 0000000..7ee1493 --- /dev/null +++ b/core/checks/executable.py @@ -0,0 +1,39 @@ +import os +import stat +import core.log as log +import core.checks.base as base +import core.checks.utils as utils + + +class FilesExecCheck(base.Check): + def __init__(self, pkg, files): + super().__init__(files) + self.pkg = pkg + + def validate(self, item): + log.i(f"Checking {item}") + return os.access(item, os.X_OK) + + def show(self, item): + log.e(f"'{item}' is not executable, but should be") + + def fix(self, item): + perms = os.stat(item).st_mode + log.i(f"'{item}' has been given execute permissions") + os.chmod(item, perms | stat.S_IXOTH | stat.S_IXGRP | stat.S_IXUSR) + + def diff(self, item): + log.i("X permissions would be added") + + def edit(self, item): + utils.open_shell(os.path.dirname(item)) + + +class ExecCheck(): + def __init__(self, pkg): + self.pkg = pkg + + def run(self): + log.s(f"Checking files execute permission") + FilesExecCheck(self.pkg, utils.find_files('./usr/{,s}bin/**/*')).run() + FilesExecCheck(self.pkg, utils.find_files('./usr/lib{32,64}/**/*.so')).run() From 02c7e30b582d60bf04df088858ee082a59d6edd0 Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 17 Jul 2019 22:17:47 +0200 Subject: [PATCH 13/37] npf-checker now only takes one .npf --- core/args.py | 1 - core/check.py | 4 ++-- core/checks/executable.py | 10 +++------- core/package.py | 2 +- npf-checker.py | 16 ++++++++-------- 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/core/args.py b/core/args.py index e6b3144..282492c 100644 --- a/core/args.py +++ b/core/args.py @@ -15,7 +15,6 @@ def parse_args(): ) _parser.add_argument( 'npf', - nargs='+', ) _parser.add_argument( '-c', diff --git a/core/check.py b/core/check.py index 38783b3..6e01cb4 100644 --- a/core/check.py +++ b/core/check.py @@ -1,5 +1,5 @@ import core.checks.executable as exe -def check_package(package): - exe.ExecCheck(package).run() +def check_package(): + exe.ExecCheck().run() diff --git a/core/checks/executable.py b/core/checks/executable.py index 7ee1493..de397f9 100644 --- a/core/checks/executable.py +++ b/core/checks/executable.py @@ -6,9 +6,8 @@ class FilesExecCheck(base.Check): - def __init__(self, pkg, files): + def __init__(self, files): super().__init__(files) - self.pkg = pkg def validate(self, item): log.i(f"Checking {item}") @@ -30,10 +29,7 @@ def edit(self, item): class ExecCheck(): - def __init__(self, pkg): - self.pkg = pkg - def run(self): log.s(f"Checking files execute permission") - FilesExecCheck(self.pkg, utils.find_files('./usr/{,s}bin/**/*')).run() - FilesExecCheck(self.pkg, utils.find_files('./usr/lib{32,64}/**/*.so')).run() + FilesExecCheck(utils.find_files('./usr/{,s}bin/**/*')).run() + FilesExecCheck(utils.find_files('./usr/lib{32,64}/**/*.so')).run() diff --git a/core/package.py b/core/package.py index 539e1a2..46cc8c2 100644 --- a/core/package.py +++ b/core/package.py @@ -31,7 +31,7 @@ def unwrap(self): os.remove(data) def check(self): - core.check.check_package(self) + core.check.check_package() def wrap(self): self.update_manifest_toml_wrap_date() diff --git a/npf-checker.py b/npf-checker.py index a0d3100..ebe28ee 100755 --- a/npf-checker.py +++ b/npf-checker.py @@ -10,14 +10,14 @@ def main(): core.args.parse_args() args = core.args.get_args() base.Check.global_state = base.Type.from_string(core.args.get_args().action) - for f in args.npf: - pkg = core.package.Package(f) - log.s(f"Unwrapping {f}") - pkg.unwrap() - log.s(f"Checking {f}") - pkg.check() - log.s(f"Wrapping {f}") - pkg.wrap() + f = args.npf + pkg = core.package.Package(f) + log.s(f"Unwrapping {f}") + pkg.unwrap() + log.s(f"Checking {f}") + pkg.check() + log.s(f"Wrapping {f}") + pkg.wrap() if __name__ == '__main__': From d9d7e422962c250d60b8edbc4a1e546d4dd578ed Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 17 Jul 2019 22:32:52 +0200 Subject: [PATCH 14/37] Add base class for checks modifying manifest.toml --- core/checks/base.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/checks/base.py b/core/checks/base.py index 7d43ac7..8b2d909 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -1,4 +1,7 @@ +import os +import toml import enum +import datetime import core import core.args import core.log as log @@ -61,3 +64,18 @@ def diff(self, item): def edit(self, item): raise NotImplementedError + + +class CheckWithManifest(Check): + def __init__(self, items): + super.__init__(items) + self.manifest_path = os.path.join(core.args.get_args().cache_dir, 'manifest.toml') + self.manifest = toml.load(self.manifest_path) + + def edit(self): + utils.open_editor(self.manifest_path) + self.manifest = toml.load(self.manifest_path) + + def write_manifest(self): + with open(self.manifest_path, 'w') as filename: + toml.dump(self.manifest, filename) From 0c4c80fab008c629e356bc22afb42039463952fe Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 17 Jul 2019 22:42:29 +0200 Subject: [PATCH 15/37] Fix utils.ask_yne default (when pressing enter directly) --- core/checks/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/checks/utils.py b/core/checks/utils.py index 56bb8d2..9c111f4 100644 --- a/core/checks/utils.py +++ b/core/checks/utils.py @@ -13,7 +13,7 @@ class Answer(enum.Enum): NONE = enum.auto() -def ask_yne(question, default=True): +def ask_yne(question, default=Answer.YES): while True: answer = log.q(question + '[Y/n/e] ').lower() if answer == '': From 3fdc2d5b58b991b9dc2fe97426846c71c84dcd8e Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 17 Jul 2019 22:49:22 +0200 Subject: [PATCH 16/37] Add some log.push and use of colored_path in tar creation logs --- core/checks/base.py | 34 ++++++++++++++++++---------------- core/package.py | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/core/checks/base.py b/core/checks/base.py index 8b2d909..0ed309c 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -33,22 +33,24 @@ def run(self): with log.push(), core.pushd(core.args.get_args().cache_dir): for item in self.items: if not self.validate(item): - self.show(item) - if Check.global_state is Type.FIX: - self.fix(item) - elif Check.global_state is Type.DIFF: - self.diff(item) - elif Check.global_state is Type.EDIT: - log.i("The automatic changes would be as follows") - if self.diff(item) is not False: - answer = utils.ask_yne("Accept those changes? ") - if answer is utils.Answer.YES: - self.fix(item) - elif answer == utils.Answer.EDIT: - self.edit(item) - else: - if utils.ask_yn("Edit manually? "): - self.edit(item) + with log.push(): + self.show(item) + if Check.global_state is Type.FIX: + self.fix(item) + elif Check.global_state is Type.DIFF: + self.diff(item) + elif Check.global_state is Type.EDIT: + log.i("The automatic changes would be as follows") + with log.push(): + if self.diff(item) is not False: + answer = utils.ask_yne("Accept those changes? ") + if answer is utils.Answer.YES: + self.fix(item) + elif answer == utils.Answer.EDIT: + self.edit(item) + else: + if utils.ask_yn("Edit manually? "): + self.edit(item) def validate(self, item): raise NotImplementedError diff --git a/core/package.py b/core/package.py index 46cc8c2..681f8ea 100644 --- a/core/package.py +++ b/core/package.py @@ -54,7 +54,7 @@ def create_data_tar(self): with log.push(): for root, _, filenames in os.walk('.'): for name in filenames: - log.s(os.path.join(root, name)) + log.s(_colored_path(os.path.join(root, name))) files_count += 1 log.s(f"(That's {files_count} files.)") log.s(f"Creating data.tar.gz") From 0599448315794f5bf200c4c7d442d171b884b88f Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 17 Jul 2019 22:58:01 +0200 Subject: [PATCH 17/37] Reintroduce pkg to check_package, and to checks with manifest.toml --- core/check.py | 3 ++- core/checks/base.py | 11 +++++++---- core/package.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core/check.py b/core/check.py index 6e01cb4..5d00d33 100644 --- a/core/check.py +++ b/core/check.py @@ -1,5 +1,6 @@ import core.checks.executable as exe +import core.checks.syntax_check as stx -def check_package(): +def check_package(pkg): exe.ExecCheck().run() diff --git a/core/checks/base.py b/core/checks/base.py index 0ed309c..bc9d7de 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -69,10 +69,11 @@ def edit(self, item): class CheckWithManifest(Check): - def __init__(self, items): - super.__init__(items) - self.manifest_path = os.path.join(core.args.get_args().cache_dir, 'manifest.toml') - self.manifest = toml.load(self.manifest_path) + def __init__(self, pkg, items): + super().__init__(items) + self.pkg = pkg + self.manifest_path = pkg.manifest_path + self.manifest = pkg.manifest def edit(self): utils.open_editor(self.manifest_path) @@ -81,3 +82,5 @@ def edit(self): def write_manifest(self): with open(self.manifest_path, 'w') as filename: toml.dump(self.manifest, filename) + self.pkg.manifest = self.manifest + diff --git a/core/package.py b/core/package.py index 681f8ea..d635661 100644 --- a/core/package.py +++ b/core/package.py @@ -31,7 +31,7 @@ def unwrap(self): os.remove(data) def check(self): - core.check.check_package() + core.check.check_package(self) def wrap(self): self.update_manifest_toml_wrap_date() From b8fe5f7fef6b086a700f6866e7169f8b14f4cb66 Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 17 Jul 2019 22:59:05 +0200 Subject: [PATCH 18/37] Add checks on manifest.toml's description --- core/check.py | 1 + core/checks/syntax_check.py | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 core/checks/syntax_check.py diff --git a/core/check.py b/core/check.py index 5d00d33..def6612 100644 --- a/core/check.py +++ b/core/check.py @@ -4,3 +4,4 @@ def check_package(pkg): exe.ExecCheck().run() + stx.DescriptionCheck(pkg).run() diff --git a/core/checks/syntax_check.py b/core/checks/syntax_check.py new file mode 100644 index 0000000..dc807cd --- /dev/null +++ b/core/checks/syntax_check.py @@ -0,0 +1,39 @@ +import core.log as log +import core.checks.base as base + + +class DescriptionCheck(base.CheckWithManifest): + def __init__(self, pkg): + super().__init__(pkg, None) # items is provided on the next line + self.items = [self.manifest['metadata']['description']] + self.capital = False + self.full_stop = False + + def validate(self, description): + self.capital = description[0].isupper() + self.full_stop = description[-1] == '.' + return len(description) >= 2 and self.capital and self.full_stop + + def show(self, description): + log.e( + f"The description of the package doesn't respect the required syntax" + ) + + def fix(self, description): + if len(description) < 2: + log.w("Nothing can be done automatically, the description is too short") + else: + if not self.full_stop: + description += '.' + log.i("Full stop has been added at the end") + if not self.capital: + description = description[0].upper() + description[1:] + log.i("First letter has been converted to uppercase") + self.manifest['metadata']['description'] = description + self.write_manifest() + + def diff(self, description): + if not self.capital: + log.i("The first letter would be converted to uppercase") + if not self.full_stop: + log.i("A full stop would be added at the end") From d82214b3d06ac38ececfc869a50df0ad346ef6e6 Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 17 Jul 2019 23:14:25 +0200 Subject: [PATCH 19/37] Remove self.manifest{,path} from CheckWithManifest, now use self.pkg --- core/checks/base.py | 16 ++++++---------- core/checks/syntax_check.py | 6 +++--- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/core/checks/base.py b/core/checks/base.py index bc9d7de..0dad91b 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -72,15 +72,11 @@ class CheckWithManifest(Check): def __init__(self, pkg, items): super().__init__(items) self.pkg = pkg - self.manifest_path = pkg.manifest_path - self.manifest = pkg.manifest - def edit(self): - utils.open_editor(self.manifest_path) - self.manifest = toml.load(self.manifest_path) - - def write_manifest(self): - with open(self.manifest_path, 'w') as filename: - toml.dump(self.manifest, filename) - self.pkg.manifest = self.manifest + def edit(self, item): + utils.open_editor(self.pkg.manifest_path) + self.pkg.manifest = toml.load(self.pkg.manifest_path) + def write_pkg_manifest(self): + with open(self.pkg.manifest_path, 'w') as filename: + toml.dump(self.pkg.manifest, filename) diff --git a/core/checks/syntax_check.py b/core/checks/syntax_check.py index dc807cd..ff0a4f3 100644 --- a/core/checks/syntax_check.py +++ b/core/checks/syntax_check.py @@ -5,7 +5,7 @@ class DescriptionCheck(base.CheckWithManifest): def __init__(self, pkg): super().__init__(pkg, None) # items is provided on the next line - self.items = [self.manifest['metadata']['description']] + self.items = [pkg.manifest['metadata']['description']] self.capital = False self.full_stop = False @@ -29,8 +29,8 @@ def fix(self, description): if not self.capital: description = description[0].upper() + description[1:] log.i("First letter has been converted to uppercase") - self.manifest['metadata']['description'] = description - self.write_manifest() + self.pkg.manifest['metadata']['description'] = description + self.write_pkg_manifest() def diff(self, description): if not self.capital: From fb7bff4c48c91acf1b069cf5d06316c7fc37302f Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 17 Jul 2019 23:20:49 +0200 Subject: [PATCH 20/37] add correct caps formatting to default option to utils.ask_yne --- core/checks/base.py | 2 +- core/checks/utils.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core/checks/base.py b/core/checks/base.py index 0dad91b..f52f907 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -43,7 +43,7 @@ def run(self): log.i("The automatic changes would be as follows") with log.push(): if self.diff(item) is not False: - answer = utils.ask_yne("Accept those changes? ") + answer = utils.ask_yne("Accept those changes?") if answer is utils.Answer.YES: self.fix(item) elif answer == utils.Answer.EDIT: diff --git a/core/checks/utils.py b/core/checks/utils.py index 9c111f4..df98c98 100644 --- a/core/checks/utils.py +++ b/core/checks/utils.py @@ -15,7 +15,14 @@ class Answer(enum.Enum): def ask_yne(question, default=Answer.YES): while True: - answer = log.q(question + '[Y/n/e] ').lower() + s = "" + if default == Answer.YES: + s = 'Y/n/e' + elif default == Answer.NO: + s = 'y/N/e' + elif default == Answer.EDIT: + s = 'y/n/E' + answer = log.q(f"{question}[{s}] ").lower() if answer == '': return default elif answer in ['y', 'yes', 'ye']: From 2f4964aca2718bd8af20e5e21113f63a2338b3dc Mon Sep 17 00:00:00 2001 From: melis_m Date: Sun, 21 Jul 2019 15:39:45 +0200 Subject: [PATCH 21/37] Add check on pkg kind, and a bit more logs to other things --- core/check.py | 2 ++ core/checks/base.py | 2 +- core/checks/nature.py | 51 +++++++++++++++++++++++++++++++++++++ core/checks/syntax_check.py | 1 + core/package.py | 3 +++ 5 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 core/checks/nature.py diff --git a/core/check.py b/core/check.py index def6612..f0bc231 100644 --- a/core/check.py +++ b/core/check.py @@ -1,7 +1,9 @@ import core.checks.executable as exe import core.checks.syntax_check as stx +import core.checks.nature as nature def check_package(pkg): exe.ExecCheck().run() stx.DescriptionCheck(pkg).run() + nature.NatureCheck(pkg).run() diff --git a/core/checks/base.py b/core/checks/base.py index f52f907..0dad91b 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -43,7 +43,7 @@ def run(self): log.i("The automatic changes would be as follows") with log.push(): if self.diff(item) is not False: - answer = utils.ask_yne("Accept those changes?") + answer = utils.ask_yne("Accept those changes? ") if answer is utils.Answer.YES: self.fix(item) elif answer == utils.Answer.EDIT: diff --git a/core/checks/nature.py b/core/checks/nature.py new file mode 100644 index 0000000..1d67ee8 --- /dev/null +++ b/core/checks/nature.py @@ -0,0 +1,51 @@ +import os +import core +import core.log as log +import core.checks.base as base +import core.checks.utils as utils + + +class NatureCheck(base.CheckWithManifest): + def __init__(self, pkg): + super().__init__(pkg, [None]) + log.s("Checking package kind") + + def validate(self, _): + length = len(os.listdir(self.pkg.cache)) + return (self.pkg.is_effective and length != 1) \ + or (not self.pkg.is_effective and length == 1) + + def show(self, _): + if self.pkg.is_effective: + log.e("Package is effective but is missing a data.tar.gz") + else: + log.e("Package is virtual but has a data.tar.gz") + + def diff(self, _): + target = 'virtual' if self.pkg.is_effective else 'effective' + log.i(f"Package kind would be changed to {target}") + + def fix(self, _): + target = 'virtual' if self.pkg.is_effective else 'effective' + self.pkg.manifest['kind'] = target + self.pkg.is_effective = not self.pkg.is_effective + + def edit(self, _): + if self.pkg.is_effective: + ans = log.q("Would you like to edit the manifest.toml (1) " + "or to add files to the package (2)?") + if ans == "1": + super().edit(_) + elif ans == "2": + utils.open_shell(self.pkg.cache) + else: + log.w("Only recognized answers are 1 and 2") + else: + ans = log.q("Would you like to edit the manifest.toml (1) " + "or to remove the files from the package? (2)") + if ans == "1": + super().edit(_) + elif ans == "2": + pass # The rewrap at the end will ignore the files if the kind is virtual + else: + log.w("Only recognized answers are 1 and 2") diff --git a/core/checks/syntax_check.py b/core/checks/syntax_check.py index ff0a4f3..97f67d0 100644 --- a/core/checks/syntax_check.py +++ b/core/checks/syntax_check.py @@ -5,6 +5,7 @@ class DescriptionCheck(base.CheckWithManifest): def __init__(self, pkg): super().__init__(pkg, None) # items is provided on the next line + log.s("Checking package description") self.items = [pkg.manifest['metadata']['description']] self.capital = False self.full_stop = False diff --git a/core/package.py b/core/package.py index d635661..24838b0 100644 --- a/core/package.py +++ b/core/package.py @@ -38,6 +38,8 @@ def wrap(self): self.show_manifest() if self.is_effective: self.create_data_tar() + else: + log.i("Ignoring data.tar.gz creation phase because package is virtual") self.create_nest_file() def update_manifest_toml_wrap_date(self): @@ -75,6 +77,7 @@ def create_nest_file(self): os.remove('data.tar.gz') new_path = f'{self.npf_path}.new' os.rename(os.path.join(self.cache, new_nest_file_path), new_path) + log.s(f"New nest file is located at {new_path}") def show_manifest(self): m = self.manifest From 962407188734891e1bdcd631816bf95aa082ba31 Mon Sep 17 00:00:00 2001 From: melis_m Date: Tue, 23 Jul 2019 17:01:17 +0200 Subject: [PATCH 22/37] Add log when running DescCheck's run function --- core/checks/syntax_check.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/checks/syntax_check.py b/core/checks/syntax_check.py index ff0a4f3..db5975a 100644 --- a/core/checks/syntax_check.py +++ b/core/checks/syntax_check.py @@ -37,3 +37,7 @@ def diff(self, description): log.i("The first letter would be converted to uppercase") if not self.full_stop: log.i("A full stop would be added at the end") + + def run(self): + log.s("Checking the description of the package") + super().run() From 6444c748d1dab166b986ef719867826a07c66a83 Mon Sep 17 00:00:00 2001 From: melis_m Date: Tue, 23 Jul 2019 17:03:48 +0200 Subject: [PATCH 23/37] Add spacing in the sentences of utils.ask_yn{,e} --- core/checks/base.py | 2 +- core/checks/utils.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/checks/base.py b/core/checks/base.py index f52f907..2b3b3e1 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -49,7 +49,7 @@ def run(self): elif answer == utils.Answer.EDIT: self.edit(item) else: - if utils.ask_yn("Edit manually? "): + if utils.ask_yn("Edit manually?"): self.edit(item) def validate(self, item): diff --git a/core/checks/utils.py b/core/checks/utils.py index df98c98..9901843 100644 --- a/core/checks/utils.py +++ b/core/checks/utils.py @@ -22,7 +22,7 @@ def ask_yne(question, default=Answer.YES): s = 'y/N/e' elif default == Answer.EDIT: s = 'y/n/E' - answer = log.q(f"{question}[{s}] ").lower() + answer = log.q(f"{question} [{s}] ").lower() if answer == '': return default elif answer in ['y', 'yes', 'ye']: @@ -38,7 +38,8 @@ def ask_yne(question, default=Answer.YES): def ask_yn(question, default=True): while True: - answer = log.q(question + '[Y/n] ').lower() + s = '[Y/n]' if default else ['y/N'] + answer = log.q(f"{question} {s} ").lower() if answer == '': return default elif answer in ['y', 'yes', 'ye']: From 67d7d8abc100d3cc8a24abf0b15e8b8a7336280c Mon Sep 17 00:00:00 2001 From: melis_m Date: Tue, 23 Jul 2019 17:25:31 +0200 Subject: [PATCH 24/37] Rename nature to kind --- core/check.py | 4 ++-- core/checks/{nature.py => kind.py} | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) rename core/checks/{nature.py => kind.py} (96%) diff --git a/core/check.py b/core/check.py index f0bc231..ec03e2d 100644 --- a/core/check.py +++ b/core/check.py @@ -1,9 +1,9 @@ import core.checks.executable as exe import core.checks.syntax_check as stx -import core.checks.nature as nature +import core.checks.kind as kind def check_package(pkg): exe.ExecCheck().run() stx.DescriptionCheck(pkg).run() - nature.NatureCheck(pkg).run() + kind.KindCheck(pkg).run() diff --git a/core/checks/nature.py b/core/checks/kind.py similarity index 96% rename from core/checks/nature.py rename to core/checks/kind.py index 1d67ee8..bcab435 100644 --- a/core/checks/nature.py +++ b/core/checks/kind.py @@ -1,11 +1,10 @@ import os -import core import core.log as log import core.checks.base as base import core.checks.utils as utils -class NatureCheck(base.CheckWithManifest): +class KindCheck(base.CheckWithManifest): def __init__(self, pkg): super().__init__(pkg, [None]) log.s("Checking package kind") From f3cad7d7326b9361928bfb7cc1340f880ae4b1b8 Mon Sep 17 00:00:00 2001 From: melis_m Date: Tue, 23 Jul 2019 18:11:51 +0200 Subject: [PATCH 25/37] Fix pkg.show_manifest, version is now displayed properly --- core/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/package.py b/core/package.py index d635661..2a91e9a 100644 --- a/core/package.py +++ b/core/package.py @@ -83,7 +83,7 @@ def show_manifest(self): with log.push(): log.s(f"name: {m['name']}") log.s(f"category: {m['category']}") - log.s(f"version: {'version'}") + log.s(f"version: {m['version']}") log.s(f"description: {metadata['description']}") log.s(f"tags: {', '.join(metadata['tags'])}") log.s(f"maintainer: {metadata['maintainer']}") From 653277585ec6536e9f57995c6abc3a3ba8553ec0 Mon Sep 17 00:00:00 2001 From: melis_m Date: Fri, 26 Jul 2019 15:00:24 +0200 Subject: [PATCH 26/37] Refresh self.pkg.is_effective in CheckWithManifest after manual edit --- core/checks/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/checks/base.py b/core/checks/base.py index 0dad91b..3c35e08 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -76,6 +76,7 @@ def __init__(self, pkg, items): def edit(self, item): utils.open_editor(self.pkg.manifest_path) self.pkg.manifest = toml.load(self.pkg.manifest_path) + self.pkg.is_effective = self.pkg.manifest['kind'] == 'effective' def write_pkg_manifest(self): with open(self.pkg.manifest_path, 'w') as filename: From 6d6341939b3457173b217aa3575efd89cb75b055 Mon Sep 17 00:00:00 2001 From: melis_m Date: Fri, 26 Jul 2019 15:03:35 +0200 Subject: [PATCH 27/37] Add space after edit questions in pkg kind check --- core/checks/kind.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/checks/kind.py b/core/checks/kind.py index bcab435..2fe81d1 100644 --- a/core/checks/kind.py +++ b/core/checks/kind.py @@ -32,7 +32,7 @@ def fix(self, _): def edit(self, _): if self.pkg.is_effective: ans = log.q("Would you like to edit the manifest.toml (1) " - "or to add files to the package (2)?") + "or to add files to the package (2)? ") if ans == "1": super().edit(_) elif ans == "2": @@ -41,7 +41,7 @@ def edit(self, _): log.w("Only recognized answers are 1 and 2") else: ans = log.q("Would you like to edit the manifest.toml (1) " - "or to remove the files from the package? (2)") + "or to remove the files from the package? (2) ") if ans == "1": super().edit(_) elif ans == "2": From aebbcc2ff8827c41f1ed9bd82a6583304e00656b Mon Sep 17 00:00:00 2001 From: melis_m Date: Fri, 26 Jul 2019 15:14:56 +0200 Subject: [PATCH 28/37] Fix edit question in pkg kind check, when answer isn't 1 or 2 --- core/checks/kind.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/core/checks/kind.py b/core/checks/kind.py index 2fe81d1..16925af 100644 --- a/core/checks/kind.py +++ b/core/checks/kind.py @@ -31,20 +31,28 @@ def fix(self, _): def edit(self, _): if self.pkg.is_effective: - ans = log.q("Would you like to edit the manifest.toml (1) " - "or to add files to the package (2)? ") - if ans == "1": + ans = self._ask_1or2("Would you like to edit the manifest.toml (1) " + "or to add files to the package (2)? ") + if ans == 1: super().edit(_) - elif ans == "2": + elif ans == 2: utils.open_shell(self.pkg.cache) - else: - log.w("Only recognized answers are 1 and 2") else: - ans = log.q("Would you like to edit the manifest.toml (1) " - "or to remove the files from the package? (2) ") - if ans == "1": + + ans = self._ask_1or2("Would you like to edit the manifest.toml (1) " + "or to remove the files from the package? (2) ") + if ans == 1: super().edit(_) - elif ans == "2": - pass # The rewrap at the end will ignore the files if the kind is virtual + # elif ans == 2: + # pass # The rewrap at the end will ignore the files if the kind is virtual + + @staticmethod + def _ask_1or2(question): + while True: + ans = log.q(question) + if ans == '1': + return 1 + elif ans == '2': + return 2 else: log.w("Only recognized answers are 1 and 2") From a87a8325c50ca2479daa8fc6da7b669fd66eae9c Mon Sep 17 00:00:00 2001 From: melis_m Date: Mon, 12 Aug 2019 16:02:58 +0200 Subject: [PATCH 29/37] Add /{,s}bin/ and /lib{,32,64}/ and /usr/lib/ to Exec Checks --- core/checks/executable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/checks/executable.py b/core/checks/executable.py index de397f9..c4303d8 100644 --- a/core/checks/executable.py +++ b/core/checks/executable.py @@ -31,5 +31,5 @@ def edit(self, item): class ExecCheck(): def run(self): log.s(f"Checking files execute permission") - FilesExecCheck(utils.find_files('./usr/{,s}bin/**/*')).run() - FilesExecCheck(utils.find_files('./usr/lib{32,64}/**/*.so')).run() + FilesExecCheck(utils.find_files('./{,usr}/{,s}bin/**/*')).run() + FilesExecCheck(utils.find_files('./{,usr}/lib{,32,64}/**/*.so')).run() From 75791b3900c08fb047c4c8bf42cd686420e621ed Mon Sep 17 00:00:00 2001 From: melis_m Date: Mon, 12 Aug 2019 16:07:43 +0200 Subject: [PATCH 30/37] Add help section to --action argument --- core/args.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/args.py b/core/args.py index 282492c..7ab8224 100644 --- a/core/args.py +++ b/core/args.py @@ -30,6 +30,7 @@ def parse_args(): '--action', choices=['edit', 'fix', 'diff'], default='edit', + help="Action to be used, 'edit' prompts the user the possible changes, 'fix' automatically applies all of them, and 'diff' only shows what would be automatically done" ) _parser.add_argument( '--visual', From a5c79e9cf515e5aa69ac0c3a3e78e13816c69324 Mon Sep 17 00:00:00 2001 From: melis_m Date: Tue, 3 Sep 2019 12:44:46 +0200 Subject: [PATCH 31/37] Move run message to run function, and add write_manifest after fix --- core/checks/kind.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/checks/kind.py b/core/checks/kind.py index 16925af..ecad327 100644 --- a/core/checks/kind.py +++ b/core/checks/kind.py @@ -7,7 +7,10 @@ class KindCheck(base.CheckWithManifest): def __init__(self, pkg): super().__init__(pkg, [None]) + + def run(self): log.s("Checking package kind") + super().run() def validate(self, _): length = len(os.listdir(self.pkg.cache)) @@ -28,6 +31,7 @@ def fix(self, _): target = 'virtual' if self.pkg.is_effective else 'effective' self.pkg.manifest['kind'] = target self.pkg.is_effective = not self.pkg.is_effective + self.write_pkg_manifest() def edit(self, _): if self.pkg.is_effective: From cddd15032b9ecb8db08408b0c32257a1b0070dfd Mon Sep 17 00:00:00 2001 From: melis_m Date: Tue, 3 Sep 2019 13:05:05 +0200 Subject: [PATCH 32/37] Remove duplicate log --- core/checks/syntax_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/checks/syntax_check.py b/core/checks/syntax_check.py index 9b5f601..db5975a 100644 --- a/core/checks/syntax_check.py +++ b/core/checks/syntax_check.py @@ -5,7 +5,6 @@ class DescriptionCheck(base.CheckWithManifest): def __init__(self, pkg): super().__init__(pkg, None) # items is provided on the next line - log.s("Checking package description") self.items = [pkg.manifest['metadata']['description']] self.capital = False self.full_stop = False From e5670dc90e6479f7bd99e552db2ec1c1726af4be Mon Sep 17 00:00:00 2001 From: melis_m Date: Tue, 3 Sep 2019 13:07:28 +0200 Subject: [PATCH 33/37] Remove extra space in question during run --- core/checks/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/checks/base.py b/core/checks/base.py index 8795c13..115a1b4 100644 --- a/core/checks/base.py +++ b/core/checks/base.py @@ -43,7 +43,7 @@ def run(self): log.i("The automatic changes would be as follows") with log.push(): if self.diff(item) is not False: - answer = utils.ask_yne("Accept those changes? ") + answer = utils.ask_yne("Accept those changes?") if answer is utils.Answer.YES: self.fix(item) elif answer == utils.Answer.EDIT: From d931a1d32f533b5271c18592bce675817dc98d4d Mon Sep 17 00:00:00 2001 From: melis_m Date: Sun, 22 Sep 2019 21:49:10 +0200 Subject: [PATCH 34/37] Rename 'nest file' to 'NPF' --- core/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/package.py b/core/package.py index 83c396d..bdc642b 100644 --- a/core/package.py +++ b/core/package.py @@ -77,7 +77,7 @@ def create_nest_file(self): os.remove('data.tar.gz') new_path = f'{self.npf_path}.new' os.rename(os.path.join(self.cache, new_nest_file_path), new_path) - log.s(f"New nest file is located at {new_path}") + log.s(f"New NPF is located at {new_path}") def show_manifest(self): m = self.manifest From 982b7b81670d5f6c3d9692b6454a818a4ced99a2 Mon Sep 17 00:00:00 2001 From: melis_m Date: Sun, 22 Sep 2019 22:01:08 +0200 Subject: [PATCH 35/37] Remove commented code --- core/checks/kind.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/checks/kind.py b/core/checks/kind.py index ecad327..a5e2c15 100644 --- a/core/checks/kind.py +++ b/core/checks/kind.py @@ -47,8 +47,6 @@ def edit(self, _): "or to remove the files from the package? (2) ") if ans == 1: super().edit(_) - # elif ans == 2: - # pass # The rewrap at the end will ignore the files if the kind is virtual @staticmethod def _ask_1or2(question): From caca88d8b1a68459fb59231b06d47c425189c961 Mon Sep 17 00:00:00 2001 From: melis_m Date: Sun, 22 Sep 2019 22:02:07 +0200 Subject: [PATCH 36/37] Reformat indentation on one line --- core/checks/kind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/checks/kind.py b/core/checks/kind.py index a5e2c15..42b3f3c 100644 --- a/core/checks/kind.py +++ b/core/checks/kind.py @@ -36,7 +36,7 @@ def fix(self, _): def edit(self, _): if self.pkg.is_effective: ans = self._ask_1or2("Would you like to edit the manifest.toml (1) " - "or to add files to the package (2)? ") + "or to add files to the package (2)? ") if ans == 1: super().edit(_) elif ans == 2: From 67eb5a110e65a2b805a368e7bda63be0626c24ba Mon Sep 17 00:00:00 2001 From: melis_m Date: Wed, 25 Sep 2019 17:44:19 +0200 Subject: [PATCH 37/37] Fix a typo in a variable definition --- core/args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/args.py b/core/args.py index 4ee4a43..7ab8224 100644 --- a/core/args.py +++ b/core/args.py @@ -2,7 +2,7 @@ import os import sys -global _argst +global _args global _parser