From b0df3fc8257b3919703101c1d597486d035a3108 Mon Sep 17 00:00:00 2001 From: Johannes Paul Date: Mon, 20 Nov 2023 15:11:16 +0100 Subject: [PATCH 1/8] Feature: Reload script on retrying commands Can be enabled/disabled via config file. Default: true --- README.md | 3 ++ automatix/automatix.py | 83 ++++++++++++++++++++++++-------------- automatix/command.py | 11 +++++ automatix/config.py | 2 + example.automatix.cfg.yaml | 13 ++---- 5 files changed, 72 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 574f282..0d19464 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,9 @@ Default location is "~/.automatix.cfg.yaml". # Path to scripts directory script_dir: ~/automatix_script_files + + # Reload policy for retries, 'true' reloads the automatix script from the file system + reload_on_retry: true # Global constants for use in pipeline scripts constants: diff --git a/automatix/automatix.py b/automatix/automatix.py index eb33c06..bc4c55a 100644 --- a/automatix/automatix.py +++ b/automatix/automatix.py @@ -1,9 +1,11 @@ import logging from argparse import Namespace from collections import OrderedDict +from functools import cached_property from typing import List -from .command import Command, AbortException, SkipBatchItemException, PERSISTENT_VARS +from .command import Command, AbortException, SkipBatchItemException, PERSISTENT_VARS, ReloadFromFile +from .config import get_script from .environment import PipelineEnvironment @@ -31,9 +33,17 @@ def __init__( logger=logging.getLogger(config['logger']), ) + @cached_property + def command_lists(self) -> dict: + return { + 'always': self.build_command_list(pipeline='always'), + 'main': self.build_command_list(pipeline='pipeline'), + 'cleanup': self.build_command_list(pipeline='cleanup'), + } + def build_command_list(self, pipeline: str) -> List[Command]: command_list = [] - for index, cmd in enumerate(self.script[pipeline]): + for index, cmd in enumerate(self.script.get(pipeline, [])): new_cmd = self.cmd_class( cmd=cmd, index=index, @@ -45,6 +55,10 @@ def build_command_list(self, pipeline: str) -> List[Command]: self.env.vars[new_cmd.assignment_var] = f'{{{new_cmd.assignment_var}}}' return command_list + def reload_script(self): + self.script = get_script(args=self.env.cmd_args) + del self.command_lists # Clear cache + def print_main_data(self): self.env.LOG.info('\n\n') self.env.LOG.info(f' ------ Overview ------') @@ -58,32 +72,43 @@ def print_command_line_steps(self, command_list: List[Command]): for cmd in command_list: self.env.LOG.info(f"({cmd.index}) [{cmd.orig_key}]: {cmd.get_resolved_value()}") - def execute_main_pipeline(self, command_list: List[Command]): - self.env.LOG.info('\n------------------------------') - self.env.LOG.info(' --- Start MAIN pipeline ---') + def _execute_command_list(self, name: str, start_index: int, treat_as_main: bool): + try: + steps = self.script.get('steps') + for cmd in self.command_lists[name][start_index:]: + if treat_as_main: + if steps and (self.script['exclude'] == (cmd.index in steps)): + # Case 1: exclude is True and index is in steps => skip + # Case 2: exclude is False and index is in steps => execute + self.env.LOG.notice(f'\n({cmd.index}) Not selected for execution: skip') + continue + cmd.execute(interactive=self.env.cmd_args.interactive, force=self.env.cmd_args.force) + else: + cmd.execute() + except ReloadFromFile as exc: + self.env.LOG.notice(f'\n({exc.index}) Reload script from file and retry.') + self.reload_script() + self._execute_command_list(name=name, start_index=exc.index, treat_as_main=treat_as_main) + + def execute_pipeline(self, name: str): + if not self.command_lists[name]: + return + + if name == 'main': + treat_as_main = True + start_index = self.env.cmd_args.jump_to + else: + treat_as_main = False + start_index = 0 - steps = self.script.get('steps') + self.env.LOG.info('\n------------------------------') + self.env.LOG.info(f' --- Start {name.upper()} pipeline ---') - for cmd in command_list[self.env.cmd_args.jump_to:]: - if steps and (self.script['exclude'] == (cmd.index in steps)): - # Case 1: exclude is True and index is in steps => skip - # Case 2: exclude is False and index is in steps => execute - self.env.LOG.notice(f'\n({cmd.index}) Not selected for execution: skip') - continue - cmd.execute(interactive=self.env.cmd_args.interactive, force=self.env.cmd_args.force) + self._execute_command_list(name=name, start_index=start_index, treat_as_main=treat_as_main) - self.env.LOG.info('\n --- End MAIN pipeline ---') + self.env.LOG.info(f'\n --- End {name.upper()} pipeline ---') self.env.LOG.info('------------------------------\n') - def execute_extra_pipeline(self, pipeline: str): - if self.script.get(pipeline): - self.env.LOG.info('\n------------------------------') - self.env.LOG.info(f' --- Start {pipeline.upper()} pipeline ---') - for cmd in self.build_command_list(pipeline=pipeline): - cmd.execute() - self.env.LOG.info(f'\n --- End {pipeline.upper()} pipeline ---') - self.env.LOG.info('------------------------------\n') - def run(self): self.env.LOG.info('\n\n') self.env.LOG.info('//////////////////////////////////////////////////////////////////////') @@ -92,24 +117,22 @@ def run(self): PERSISTENT_VARS.clear() - command_list = self.build_command_list(pipeline='pipeline') - - self.execute_extra_pipeline(pipeline='always') + self.execute_pipeline(name='always') self.print_main_data() - self.print_command_line_steps(command_list) + self.print_command_line_steps(command_list=self.command_lists['main']) if self.env.cmd_args.print_overview: exit() try: - self.execute_main_pipeline(command_list=command_list) + self.execute_pipeline(name='main') except (AbortException, SkipBatchItemException): self.env.LOG.debug('Abort requested. Cleaning up.') - self.execute_extra_pipeline(pipeline='cleanup') + self.execute_pipeline(name='cleanup') self.env.LOG.debug('Clean up done. Exiting.') raise - self.execute_extra_pipeline(pipeline='cleanup') + self.execute_pipeline(name='cleanup') self.env.LOG.info('---------------------------------------------------------------') self.env.LOG.info('Automatix finished: Congratulations and have a N.I.C.E. day :-)') diff --git a/automatix/command.py b/automatix/command.py index cd39389..c150a8f 100644 --- a/automatix/command.py +++ b/automatix/command.py @@ -48,6 +48,7 @@ def __init__(self, cmd: dict, index: int, pipeline: str, env: PipelineEnvironmen self.value = value break # There should be only one entry in pipeline_cmd + def get_type(self): if self.key == 'local': return 'local' @@ -125,6 +126,8 @@ def execute(self, interactive: bool = False, force: bool = False): err_answer = self._ask_user(question='[CF] What should I do?', allowed_options=['p', 'r', 'a']) # answers 'a' and 'c' are handled by _ask_user, 'p' means just pass if err_answer == 'r': + if self.env.config.get('reload_on_retry'): + raise ReloadFromFile(index=self.index) return self.execute(interactive) def _ask_user(self, question: str, allowed_options: list) -> str: @@ -344,3 +347,11 @@ class SkipBatchItemException(Exception): class UnknownCommandException(Exception): pass + + +class ReloadFromFile(Exception): + def __init__(self, index: int): + self.index = index + + def __int__(self): + return self.index diff --git a/automatix/config.py b/automatix/config.py index 35c8a34..b532a07 100644 --- a/automatix/config.py +++ b/automatix/config.py @@ -47,6 +47,7 @@ def read_yaml(yamlfile: str) -> dict: CONFIG = { 'script_dir': '~/automatix-config', + 'reload_on_retry': True, 'constants': {}, 'encoding': os.getenv('ENCODING', 'utf-8'), 'import_path': '.', @@ -156,6 +157,7 @@ def get_script(args: argparse.Namespace) -> dict: script = read_yaml(s_file) validate_script(script) + script['script_file_path'] = s_file for field in SCRIPT_FIELDS.keys(): if vars(args).get(field): diff --git a/example.automatix.cfg.yaml b/example.automatix.cfg.yaml index 57cb666..0effbab 100644 --- a/example.automatix.cfg.yaml +++ b/example.automatix.cfg.yaml @@ -1,42 +1,35 @@ # Path to scripts directory - script_dir: ~/automatix_script_files -# Global constants for use in pipeline scripts +# Reload policy for retries, 'true' reloads the automatix script from the file system +reload_on_retry: true +# Global constants for use in pipeline scripts constants: apt_update: 'apt-get -qy update' apt_upgrade: 'DEBIAN_FRONTEND=noninteractive apt-get -qy -o Dpkg::Options::=--force-confold --no-install-recommends upgrade' apt_full_upgrade: 'DEBIAN_FRONTEND=noninteractive apt-get -qy -o Dpkg::Options::=--force-confold --no-install-recommends full-upgrade' # Encoding - encoding: utf-8 # Path to shell imports - import_path: '.' # SSH Command used for remote connections - ssh_cmd: 'ssh {hostname} sudo ' # Temporary directory on remote machines for shell imports - remote_tmp_dir: 'automatix_tmp' # Logger - logger: mylogger # Logging library - logging_lib: mylib.logging # Bundlewrap support, bundlewrap has to be installed (default: false) - bundlewrap: true # Teamvault / Secret support, bundlewrap-teamvault has to be installed (default: false) - teamvault: true From 10ea2d644c277632e1ffc309d97529cefb1d9d61 Mon Sep 17 00:00:00 2001 From: Johannes Paul Date: Mon, 20 Nov 2023 16:59:25 +0100 Subject: [PATCH 2/8] Update requirements to python 3.8 and fix local tests --- README.md | 2 +- automatix/command_test.py | 10 +++++----- automatix/config.py | 7 +------ pytest.ini | 2 +- setup.py | 4 ++-- tests/README.md | 1 + tests/test_environment.py | 9 ++++++--- 7 files changed, 17 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0d19464..b97b08d 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Please use the interactive mode and doublecheck commands before # INSTALLATION -Automatix requires Python ≥ 3.6. +Automatix requires Python ≥ 3.8. ``` pip install automatix diff --git a/automatix/command_test.py b/automatix/command_test.py index fc2ce93..7a7a227 100644 --- a/automatix/command_test.py +++ b/automatix/command_test.py @@ -9,7 +9,7 @@ def test__execute_remote_cmd(ssh_up): - cmd = Command(cmd={'remote@testsystem': 'touch /test_remote_cmd'}, index=2, env=environment) + cmd = Command(cmd={'remote@testsystem': 'touch /test_remote_cmd'}, index=2, pipeline='pipeline', env=environment) cmd.execute() try: run_command_and_check('ssh docker-test ls /test_remote_cmd >/dev/null') @@ -23,7 +23,7 @@ def test__execute_local_cmd(capfd): # empty captured stdin and stderr _ = capfd.readouterr() - cmd = Command(cmd={'local': f'echo {test_string}'}, index=2, env=environment) + cmd = Command(cmd={'local': f'echo {test_string}'}, index=2, pipeline='pipeline', env=environment) cmd.execute() out, err = capfd.readouterr() @@ -44,7 +44,7 @@ def test__execute_local_with_condition(capfd): # empty captured stdin and stderr _ = capfd.readouterr() - cmd = Command(cmd={f'{condition_var}?local': 'pwd'}, index=2, env=environment) + cmd = Command(cmd={f'{condition_var}?local': 'pwd'}, pipeline='pipeline', index=2, env=environment) cmd.execute() out, err = capfd.readouterr() @@ -59,10 +59,10 @@ def test__execute_python_cmd(): PERSISTENT_VARS.update(locals()) """ - cmd = Command(cmd={'python': test_cmd}, index=2, env=environment) + cmd = Command(cmd={'python': test_cmd}, index=2, pipeline='pipeline', env=environment) cmd.execute() - cmd = Command(cmd={'python': 'print(uuid4())'}, index=2, env=environment) + cmd = Command(cmd={'python': 'print(uuid4())'}, index=2, pipeline='pipeline', env=environment) cmd.execute() diff --git a/automatix/config.py b/automatix/config.py index b532a07..647c46a 100644 --- a/automatix/config.py +++ b/automatix/config.py @@ -2,8 +2,8 @@ import logging import os import re -import sys from collections import OrderedDict +from importlib import metadata from time import sleep import yaml @@ -16,11 +16,6 @@ def read_yaml(yamlfile: str) -> dict: return yaml.load(file.read(), Loader=yaml.SafeLoader) -if sys.version_info >= (3, 8): - from importlib import metadata -else: - import importlib_metadata as metadata - try: from argcomplete import autocomplete from .bash_completion import ScriptFileCompleter diff --git a/pytest.ini b/pytest.ini index 3bdab05..7073109 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ [pytest] addopts = --durations=0 --docker-compose=tests/docker-compose.yml -testpaths = automatix \ No newline at end of file +testpaths = automatix diff --git a/setup.py b/setup.py index 0662a3d..26ab7e1 100644 --- a/setup.py +++ b/setup.py @@ -14,10 +14,10 @@ author='Johannes Paul, //SEIBERT/MEDIA GmbH', author_email='jpaul@seibert-media.net', license='MIT', - python_requires='>=3.6', + python_requires='>=3.8', install_requires=[ + 'cython<3.0.0', 'pyyaml>=5.1', - 'importlib-metadata >= 1.0 ; python_version < "3.8"', ], extras_require={ 'bash completion': 'argcomplete', diff --git a/tests/README.md b/tests/README.md index e17b623..a931cd7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,6 +4,7 @@ * Install docker and docker-compose * Install pytest and pytest-docker-compose via pip + * If install fails, have a look at https://github.com/yaml/pyyaml/issues/601#issuecomment-1813963845 * Generate a ssh keypair and place it in tests with `ssh-keygen -t rsa -f tests/id_rsa_tests` * In tests: Copy docker-compose.example.yml to docker-compose.yml and replace the public key * Put something like the following in your ~.ssh/config diff --git a/tests/test_environment.py b/tests/test_environment.py index cfd9380..8a6ebf9 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -5,8 +5,9 @@ import pytest -from automatix import get_script, collect_vars, CONFIG, cmdClass, SCRIPT_FIELDS +from automatix import get_script, collect_vars, CONFIG, Command, SCRIPT_FIELDS from automatix.automatix import Automatix +from automatix.logger import init_logger SELFDIR = dirname(abspath(__file__)) @@ -32,7 +33,7 @@ script=script, variables=variables, config=CONFIG, - cmd_class=cmdClass, + cmd_class=Command, script_fields=SCRIPT_FIELDS, cmd_args=default_args, ) @@ -47,6 +48,8 @@ environment = testauto.env +init_logger(name=CONFIG['logger'], debug=True) + def run_command_and_check(cmd): subprocess.run(cmd, shell=True).check_returncode() @@ -55,7 +58,7 @@ def run_command_and_check(cmd): @pytest.fixture(scope='function') def ssh_up(function_scoped_container_getter): max_retries = 20 - for i in range(max_retries): + for _ in range(max_retries): sleep(1) try: run_command_and_check( From a9129a213c5386512b34512962a1d7e7f10e4580 Mon Sep 17 00:00:00 2001 From: Johannes Paul Date: Mon, 20 Nov 2023 17:44:24 +0100 Subject: [PATCH 3/8] automatix: Suppress ReloadFromFile exception as context --- automatix/automatix.py | 2 +- automatix/command.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/automatix/automatix.py b/automatix/automatix.py index bc4c55a..f034f85 100644 --- a/automatix/automatix.py +++ b/automatix/automatix.py @@ -86,7 +86,7 @@ def _execute_command_list(self, name: str, start_index: int, treat_as_main: bool else: cmd.execute() except ReloadFromFile as exc: - self.env.LOG.notice(f'\n({exc.index}) Reload script from file and retry.') + self.env.LOG.info(f'\n({exc.index}) Reload script from file and retry.') self.reload_script() self._execute_command_list(name=name, start_index=exc.index, treat_as_main=treat_as_main) diff --git a/automatix/command.py b/automatix/command.py index c150a8f..150778f 100644 --- a/automatix/command.py +++ b/automatix/command.py @@ -48,7 +48,6 @@ def __init__(self, cmd: dict, index: int, pipeline: str, env: PipelineEnvironmen self.value = value break # There should be only one entry in pipeline_cmd - def get_type(self): if self.key == 'local': return 'local' @@ -207,6 +206,8 @@ def _python_action(self) -> int: 'Seems you are trying to use bundlewrap functions without having bundlewrap support enabled.' ' Please check your configuration.') return 1 + if isinstance(exc.__context__, ReloadFromFile): + exc.__suppress_context__ = True self.env.LOG.exception('Unknown error occured:') return 1 From 8d177f8ce05ad0e40be7be4e686eca40bef620ed Mon Sep 17 00:00:00 2001 From: Johannes Paul Date: Mon, 20 Nov 2023 17:59:57 +0100 Subject: [PATCH 4/8] tests/README.md: Add note for broken remote tests --- tests/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index a931cd7..5a45af0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -17,4 +17,7 @@ StrictHostKeyChecking no -* Run `make test` in the automatix root directory. \ No newline at end of file +* Run `make test` in the automatix root directory. + +Note: Testing remote commands on MacOs with podman seems broken (for me). +Maybe this needs some adjustment or further investigation. From 9d362257f669e50207219ec19ea39a609b640e7c Mon Sep 17 00:00:00 2001 From: Johannes Paul Date: Mon, 20 Nov 2023 18:17:12 +0100 Subject: [PATCH 5/8] automatix/config.py: Remove unneeded line --- automatix/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/automatix/config.py b/automatix/config.py index 647c46a..667526b 100644 --- a/automatix/config.py +++ b/automatix/config.py @@ -152,7 +152,6 @@ def get_script(args: argparse.Namespace) -> dict: script = read_yaml(s_file) validate_script(script) - script['script_file_path'] = s_file for field in SCRIPT_FIELDS.keys(): if vars(args).get(field): From 2221121bce524575a002c167303b76b33f65ebbc Mon Sep 17 00:00:00 2001 From: Johannes Paul Date: Mon, 20 Nov 2023 18:55:37 +0100 Subject: [PATCH 6/8] Refactor reloading from file and add it as extra option --- README.md | 3 --- automatix/command.py | 13 +++++++------ example.automatix.cfg.yaml | 3 --- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b97b08d..e0a3ee3 100644 --- a/README.md +++ b/README.md @@ -55,9 +55,6 @@ Default location is "~/.automatix.cfg.yaml". # Path to scripts directory script_dir: ~/automatix_script_files - - # Reload policy for retries, 'true' reloads the automatix script from the file system - reload_on_retry: true # Global constants for use in pipeline scripts constants: diff --git a/automatix/command.py b/automatix/command.py index 150778f..b93a57c 100644 --- a/automatix/command.py +++ b/automatix/command.py @@ -24,6 +24,7 @@ def __setattr__(self, key: str, value): POSSIBLE_ANSWERS = { 'p': 'proceed (default)', 'r': 'retry', + 'R': 'reload from file and retry command (same index)', 's': 'skip', 'a': 'abort', 'c': 'abort & continue to next (CSV processing)', @@ -122,11 +123,9 @@ def execute(self, interactive: bool = False, force: bool = False): if force: return - err_answer = self._ask_user(question='[CF] What should I do?', allowed_options=['p', 'r', 'a']) - # answers 'a' and 'c' are handled by _ask_user, 'p' means just pass + err_answer = self._ask_user(question='[CF] What should I do?', allowed_options=['p', 'r', 'R', 'a']) + # answers 'a', 'c' and 'R' are handled by _ask_user, 'p' means just pass if err_answer == 'r': - if self.env.config.get('reload_on_retry'): - raise ReloadFromFile(index=self.index) return self.execute(interactive) def _ask_user(self, question: str, allowed_options: list) -> str: @@ -144,19 +143,21 @@ def _ask_user(self, question: str, allowed_options: list) -> str: if self.env.batch_mode: allowed_options.append('c') - options = ', '.join([f'{k}: {POSSIBLE_ANSWERS[k]}' for k in allowed_options]) + options = '\n'.join([f' {k}: {POSSIBLE_ANSWERS[k]}' for k in allowed_options]) answer = None while answer not in allowed_options: if answer is not None: self.env.LOG.info('Invalid input. Try again.') - answer = input(f'{question} ({options})\a') + answer = input(f'{question}\n{options}\nYour answer: \a') if answer == '': # default answer = 'p' if answer == 'a': raise AbortException(1) + if answer == 'R': + raise ReloadFromFile(index=self.index) if self.env.batch_mode and answer == 'c': raise SkipBatchItemException() diff --git a/example.automatix.cfg.yaml b/example.automatix.cfg.yaml index 0effbab..836b216 100644 --- a/example.automatix.cfg.yaml +++ b/example.automatix.cfg.yaml @@ -1,9 +1,6 @@ # Path to scripts directory script_dir: ~/automatix_script_files -# Reload policy for retries, 'true' reloads the automatix script from the file system -reload_on_retry: true - # Global constants for use in pipeline scripts constants: apt_update: 'apt-get -qy update' From 61582b4ca331b6fdb9d635a08b2d4050bd36c1a1 Mon Sep 17 00:00:00 2001 From: Johannes Paul Date: Mon, 20 Nov 2023 18:59:33 +0100 Subject: [PATCH 7/8] automatix/command.py: Add file reload option for interactive mode --- automatix/command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automatix/command.py b/automatix/command.py index b93a57c..1655bc5 100644 --- a/automatix/command.py +++ b/automatix/command.py @@ -100,8 +100,8 @@ def execute(self, interactive: bool = False, force: bool = False): if self.get_type() == 'manual' or interactive: self.env.LOG.debug('Ask for user interaction.') - answer = self._ask_user(question='[MS] Proceed?', allowed_options=['p', 's', 'a']) - # answers 'a' and 'c' are handled by _ask_user, 'p' means just pass + answer = self._ask_user(question='[MS] Proceed?', allowed_options=['p', 's', 'R', 'a']) + # answers 'a', 'c' and 'R' are handled by _ask_user, 'p' means just pass if answer == 's': return From 9f28092690f5164e83bbdee218d2d8bc9d446bde Mon Sep 17 00:00:00 2001 From: Johannes Paul Date: Mon, 20 Nov 2023 19:11:12 +0100 Subject: [PATCH 8/8] Remove unneeded line --- automatix/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/automatix/config.py b/automatix/config.py index 667526b..d30b93e 100644 --- a/automatix/config.py +++ b/automatix/config.py @@ -42,7 +42,6 @@ def read_yaml(yamlfile: str) -> dict: CONFIG = { 'script_dir': '~/automatix-config', - 'reload_on_retry': True, 'constants': {}, 'encoding': os.getenv('ENCODING', 'utf-8'), 'import_path': '.',