Skip to content

Commit

Permalink
add continue sub steps on failure option
Browse files Browse the repository at this point in the history
  • Loading branch information
@itewk
itewk committed Aug 4, 2021
1 parent e0c8bac commit 8bae426
Show file tree
Hide file tree
Showing 8 changed files with 455 additions and 44 deletions.
50 changes: 38 additions & 12 deletions src/ploigos_step_runner/config/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@
import copy
import glob
import os.path
from distutils import util

from ploigos_step_runner.decryption_utils import DecryptionUtils
from ploigos_step_runner.config.step_config import StepConfig
from ploigos_step_runner.config.config_value import ConfigValue
from ploigos_step_runner.utils.file import parse_yaml_or_json_file
from ploigos_step_runner.config.step_config import StepConfig
from ploigos_step_runner.decryption_utils import DecryptionUtils
from ploigos_step_runner.utils.dict import deep_merge
from ploigos_step_runner.utils.file import parse_yaml_or_json_file


class Config:
"""Representation of configuration for Ploigos workflow.
Expand Down Expand Up @@ -42,6 +44,7 @@ class Config:
CONFIG_KEY_GLOBAL_DEFAULTS = 'global-defaults'
CONFIG_KEY_GLOBAL_ENVIRONMENT_DEFAULTS = 'global-environment-defaults'
CONFIG_KEY_ENVIRONMENT_NAME = 'environment-name'
CONFIG_KEY_CONTINUE_SUB_STEPS_ON_FAILURE = 'continue-sub-steps-on-failure'
CONFIG_KEY_STEP_IMPLEMENTER = 'implementer'
CONFIG_KEY_SUB_STEP_NAME = 'name'
CONFIG_KEY_SUB_STEP_CONFIG = 'config'
Expand Down Expand Up @@ -263,7 +266,7 @@ def __add_config_file(self, config_file):
f"Failed to add parsed configuration file ({config_file}): {error}"
) from error

def __add_config_dict(self, config_dict, source_file_path=None): # pylint: disable=too-many-locals, too-many-branches
def __add_config_dict(self, config_dict, source_file_path=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Add a configuration dictionary to the list of configuration dictionaries.
Parameters
Expand Down Expand Up @@ -374,24 +377,41 @@ def __add_config_dict(self, config_dict, source_file_path=None): # pylint: disab
else:
sub_step_name = sub_step_implementer_name

# determine sub step config
if Config.CONFIG_KEY_SUB_STEP_CONFIG in sub_step:
sub_step_config_dict = copy.deepcopy(
sub_step[Config.CONFIG_KEY_SUB_STEP_CONFIG])
else:
sub_step_config_dict = {}

# determine sub step environment config
if Config.CONFIG_KEY_SUB_STEP_ENVIRONMENT_CONFIG in sub_step:
sub_step_env_config = copy.deepcopy(
sub_step[Config.CONFIG_KEY_SUB_STEP_ENVIRONMENT_CONFIG])
else:
sub_step_env_config = {}

# determine if continue sub steps on this sub step failure
sub_step_contine_sub_steps_on_failure = False
if Config.CONFIG_KEY_CONTINUE_SUB_STEPS_ON_FAILURE in sub_step:
sub_step_contine_sub_steps_on_failure = sub_step[
Config.CONFIG_KEY_CONTINUE_SUB_STEPS_ON_FAILURE
]
if isinstance(sub_step_contine_sub_steps_on_failure.value, bool):
sub_step_contine_sub_steps_on_failure = \
sub_step_contine_sub_steps_on_failure.value
else:
sub_step_contine_sub_steps_on_failure = bool(
util.strtobool(sub_step_contine_sub_steps_on_failure.value)
)

self.add_or_update_step_config(
step_name=step_name,
sub_step_name=sub_step_name,
sub_step_implementer_name=sub_step_implementer_name,
sub_step_config_dict=sub_step_config_dict,
sub_step_env_config=sub_step_env_config
sub_step_env_config=sub_step_env_config,
sub_step_contine_sub_steps_on_failure=sub_step_contine_sub_steps_on_failure
)

@staticmethod
Expand Down Expand Up @@ -434,12 +454,14 @@ def parse_and_register_decryptors_definitions(decryptors_definitions):
)

def add_or_update_step_config( # pylint: disable=too-many-arguments
self,
step_name,
sub_step_name,
sub_step_implementer_name,
sub_step_config_dict,
sub_step_env_config):
self,
step_name,
sub_step_name,
sub_step_implementer_name,
sub_step_config_dict,
sub_step_env_config,
sub_step_contine_sub_steps_on_failure=False
):
"""Adds a new step configuration with a single new sub step or
updates an existing step with new or updated sub step.
Expand All @@ -461,6 +483,9 @@ def add_or_update_step_config( # pylint: disable=too-many-arguments
new or updated step.
If updating this can not have any duplicative leaf keys to the existing
sub step environment configuration.
sub_step_contine_sub_steps_on_failure : bool
True to continue executing other sub steps in current step if this sub step fails.
False to fail all step execution if this sub step fails.
Raises
------
Expand All @@ -482,5 +507,6 @@ def add_or_update_step_config( # pylint: disable=too-many-arguments
sub_step_name=sub_step_name,
sub_step_implementer_name=sub_step_implementer_name,
sub_step_config_dict=sub_step_config_dict,
sub_step_env_config=sub_step_env_config
sub_step_env_config=sub_step_env_config,
sub_step_contine_sub_steps_on_failure=sub_step_contine_sub_steps_on_failure
)
34 changes: 24 additions & 10 deletions src/ploigos_step_runner/config/step_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,13 @@ def step_config_overrides(self, step_config_overrides):
self.__step_config_overrides = step_config_overrides if step_config_overrides else {}

def add_or_update_sub_step_config(
self,
sub_step_name,
sub_step_implementer_name,
sub_step_config_dict=None,
sub_step_env_config=None):
self,
sub_step_name,
sub_step_implementer_name,
sub_step_config_dict=None,
sub_step_env_config=None,
sub_step_contine_sub_steps_on_failure=False
): # pylint: disable=too-many-arguments
"""Add a new or update an existing sub step configuration for this step.
Parameters
Expand All @@ -124,6 +126,9 @@ def add_or_update_sub_step_config(
Sub step environment configuration to add or update for named sub step.
If updating this can not have any duplicative leaf keys to the existing
sub step environment configuration.
sub_step_contine_sub_steps_on_failure : bool
True to continue executing other sub steps in current step if this sub step fails.
False to fail all step execution if this sub step fails.
Raises
------
Expand All @@ -149,15 +154,24 @@ def add_or_update_sub_step_config(
sub_step_name=sub_step_name,
sub_step_implementer_name=sub_step_implementer_name,
sub_step_config_dict=sub_step_config_dict,
sub_step_env_config=sub_step_env_config
sub_step_env_config=sub_step_env_config,
sub_step_contine_sub_steps_on_failure=sub_step_contine_sub_steps_on_failure
)
self.sub_steps.append(sub_step_config)
else:
assert sub_step_implementer_name == sub_step_config.sub_step_implementer_name, \
assert sub_step_implementer_name == existing_sub_step_config.sub_step_implementer_name,\
f"Step ({self.step_name}) failed to update sub step ({sub_step_name})" + \
" with new config due to new sub step implementer" + \
f" ({sub_step_implementer_name}) not matching existing sub step implementer" + \
f" ({sub_step_config.sub_step_implementer_name})."
f" ({existing_sub_step_config.sub_step_implementer_name})."

sub_step_config.merge_sub_step_config(sub_step_config_dict)
sub_step_config.merge_sub_step_env_config(sub_step_env_config)
assert sub_step_contine_sub_steps_on_failure == \
existing_sub_step_config.sub_step_contine_sub_steps_on_failure, \
f"Step ({self.step_name}) failed to update sub step ({sub_step_name})" + \
" with new config due to new continue sub steps on failure" + \
f" ({sub_step_contine_sub_steps_on_failure}) not matching existing" + \
" continue sub steps on failure " + \
f" ({existing_sub_step_config.sub_step_implementer_name})."

existing_sub_step_config.merge_sub_step_config(sub_step_config_dict)
existing_sub_step_config.merge_sub_step_env_config(sub_step_env_config)
31 changes: 25 additions & 6 deletions src/ploigos_step_runner/config/sub_step_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ class SubStepConfig:
Configuration specific to this sub step.
sub_step_env_config : dict, optional
Environment specific configuration specific to this sub step.
sub_step_contine_sub_steps_on_failure : bool
True to continue executing other sub steps in current step if this sub step fails.
False to fail all step execution if this sub step fails.
Attributes
----------
Expand All @@ -33,16 +36,19 @@ class SubStepConfig:
"""

def __init__( # pylint: disable=too-many-arguments
self,
parent_step_config,
sub_step_name,
sub_step_implementer_name,
sub_step_config_dict=None,
sub_step_env_config=None):
self,
parent_step_config,
sub_step_name,
sub_step_implementer_name,
sub_step_config_dict=None,
sub_step_env_config=None,
sub_step_contine_sub_steps_on_failure=False
):

self.__parent_step_config = parent_step_config
self.__sub_step_name = sub_step_name
self.__sub_step_implementer_name = sub_step_implementer_name
self.__sub_step_contine_sub_steps_on_failure = sub_step_contine_sub_steps_on_failure

if sub_step_config_dict is None:
sub_step_config_dict = {}
Expand Down Expand Up @@ -147,6 +153,19 @@ def sub_step_env_config(self):
"""
return copy.deepcopy(self.__sub_step_env_config)

@property
def sub_step_contine_sub_steps_on_failure(self):
"""Gets whether to continue executing other sub steps that belong to the step that
this sub step belongs to or not if this sub step fails.
Returns
-------
bool
True to continue executing other sub steps in current step if this sub step fails.
False to fail all step execution if this sub step fails.
"""
return self.__sub_step_contine_sub_steps_on_failure

def get_global_environment_defaults(self, env):
"""Convince function for getting the global environment defaults from the parent config.
Expand Down
2 changes: 1 addition & 1 deletion src/ploigos_step_runner/step_implementer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ def run_step(self):

# print information about the configuration
StepImplementer.__print_section_title(
f"Configuration - {self.step_name}",
f"Configuration - {self.step_name} - {self.sub_step_name}",
div_char="-",
indent=1
)
Expand Down
14 changes: 10 additions & 4 deletions src/ploigos_step_runner/step_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ def run_step(self, step_name, environment=None):
f"Can not run step ({step_name}) because no step configuration provided."

# for each sub step in the step config get the step implementer and run it
aggregate_success = True
for sub_step_config in sub_step_configs:
sub_step_implementer_name = sub_step_config.sub_step_implementer_name

Expand Down Expand Up @@ -174,11 +175,16 @@ def run_step(self, step_name, environment=None):
yml_filename=self.results_file_path
)

# bail out if one of the sub steps fails
if not step_result.success:
return False
# aggregate success
aggregate_success = (aggregate_success and step_result.success)

return True
# if this sub step failed and not configured to continue on failure, bail
# else execute next sub step and continue aggregating success
if (not step_result.success) and \
(not sub_step_config.sub_step_contine_sub_steps_on_failure):
break

return aggregate_success

@staticmethod
def __get_step_implementer_class(step_name, step_implementer_name):
Expand Down
103 changes: 99 additions & 4 deletions tests/config/test_config.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import os.path

from ploigos_step_runner.config import Config, ConfigValue
from ploigos_step_runner.config.decryptors.sops import SOPS
from ploigos_step_runner.decryption_utils import DecryptionUtils
from testfixtures import TempDirectory

from tests.helpers.base_test_case import BaseTestCase

from ploigos_step_runner.decryption_utils import DecryptionUtils
from ploigos_step_runner.config import Config, ConfigValue
from ploigos_step_runner.config.decryptors.sops import SOPS

class TestConfig(BaseTestCase):
def test_add_config_invalid_type(self):
Expand Down Expand Up @@ -922,6 +921,102 @@ def test_multiple_sub_steps(self):
}
)

def test_sub_step_with_continue_sub_steps_on_failure_bool(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'continue-sub-steps-on-failure': True,
'config': {
'test1': 'foo'
}
},
{
'implementer': 'foo2',
'config': {
'test2': 'foo'
}
}
]

}
})

step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 2)

self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config,
),
{
'test1': 'foo'
}
)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo2').sub_step_config
),
{
'test2': 'foo'
}
)
self.assertTrue(
step_config.get_sub_step('foo1').sub_step_contine_sub_steps_on_failure
)
self.assertFalse(
step_config.get_sub_step('foo2').sub_step_contine_sub_steps_on_failure
)

def test_sub_step_with_continue_sub_steps_on_failure_str(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'continue-sub-steps-on-failure': 'true',
'config': {
'test1': 'foo'
}
},
{
'implementer': 'foo2',
'config': {
'test2': 'foo'
}
}
]

}
})

step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 2)

self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config,
),
{
'test1': 'foo'
}
)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo2').sub_step_config
),
{
'test2': 'foo'
}
)
self.assertTrue(
step_config.get_sub_step('foo1').sub_step_contine_sub_steps_on_failure
)
self.assertFalse(
step_config.get_sub_step('foo2').sub_step_contine_sub_steps_on_failure
)

def test_sub_step_with_name(self):
config = Config({
Config.CONFIG_KEY: {
Expand Down
Loading

0 comments on commit 8bae426

Please sign in to comment.