-
-
Notifications
You must be signed in to change notification settings - Fork 10
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: add matrix strategy #118
Merged
Merged
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
f61fe03
add matrix functionality
abhijeetSaroha 678e808
re-order the imports
abhijeetSaroha 8536dd5
add matrix_vars to TEMPALTE
abhijeetSaroha eff1d49
update test list (tests.smoke)
abhijeetSaroha 7dbd1cd
Trigger Checks
abhijeetSaroha 3ffebe8
update the code
abhijeetSaroha cc288ba
Merge branch 'osl-incubator:main' into matrix-strategy
abhijeetSaroha File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -17,6 +17,7 @@ | |||||
import warnings | ||||||
|
||||||
from copy import deepcopy | ||||||
from itertools import product | ||||||
from pathlib import Path | ||||||
from typing import Any, Dict, List, Optional, Union, cast | ||||||
|
||||||
|
@@ -363,7 +364,7 @@ def _load_dotenv(self, data_scope: dict[str, Any]) -> dict[str, str]: | |||||
|
||||||
def _load_scoped_data( | ||||||
self, scope: str | ||||||
) -> tuple[dict[str, str], dict[str, str]]: | ||||||
) -> tuple[dict[str, str], dict[str, Any]]: | ||||||
scope_options = ('global', 'group', 'task') | ||||||
if scope not in scope_options: | ||||||
raise Exception(f'The given scope `{scope}` is not valid.') | ||||||
|
@@ -451,6 +452,66 @@ def _load_task_args(self) -> None: | |||||
else (False if is_bool else None) | ||||||
) | ||||||
|
||||||
def _log_command_execution( | ||||||
self, execution_context: dict[str, Any], width: int | ||||||
) -> None: | ||||||
"""Log the command execution details. | ||||||
|
||||||
execution_context should contain: args_input, current_vars, | ||||||
env, and optionally matrix_vars | ||||||
""" | ||||||
MakimLogs.print_info('=' * width) | ||||||
MakimLogs.print_info( | ||||||
'TARGET: ' + f'{self.group_name}.{self.task_name}' | ||||||
) | ||||||
MakimLogs.print_info('ARGS:') | ||||||
MakimLogs.print_info(pprint.pformat(execution_context['args_input'])) | ||||||
MakimLogs.print_info('VARS:') | ||||||
MakimLogs.print_info(pprint.pformat(execution_context['current_vars'])) | ||||||
MakimLogs.print_info('ENV:') | ||||||
MakimLogs.print_info(str(execution_context['env'])) | ||||||
|
||||||
if execution_context.get('matrix_vars'): | ||||||
MakimLogs.print_info('MATRIX:') | ||||||
MakimLogs.print_info( | ||||||
pprint.pformat(execution_context['matrix_vars']) | ||||||
) | ||||||
|
||||||
MakimLogs.print_info('-' * width) | ||||||
|
||||||
def _generate_matrix_combinations( | ||||||
self, matrix_config: dict[str, Any] | ||||||
) -> list[dict[str, Any]]: | ||||||
"""Generate all possible combinations from matrix configuration. | ||||||
|
||||||
Parameters | ||||||
---------- | ||||||
matrix_config : dict | ||||||
Dictionary containing matrix variables and their possible values | ||||||
|
||||||
Returns | ||||||
------- | ||||||
list[dict] | ||||||
List of dictionaries, each containing one possible combination | ||||||
""" | ||||||
if not matrix_config: | ||||||
return [{}] | ||||||
|
||||||
# Convert matrix config into format suitable for product | ||||||
keys = list(matrix_config.keys()) | ||||||
values = [ | ||||||
matrix_config[k] | ||||||
if isinstance(matrix_config[k], list) | ||||||
else [matrix_config[k]] | ||||||
for k in keys | ||||||
] | ||||||
|
||||||
combinations = [] | ||||||
for combo in product(*values): | ||||||
combinations.append(dict(zip(keys, combo))) | ||||||
|
||||||
return combinations | ||||||
|
||||||
# run commands | ||||||
def _run_hooks(self, args: dict[str, Any], hook_type: str) -> None: | ||||||
if not self.task_data.get('hooks', {}).get(hook_type): | ||||||
|
@@ -518,20 +579,22 @@ def _run_hooks(self, args: dict[str, Any], hook_type: str) -> None: | |||||
|
||||||
def _run_command(self, args: dict[str, Any]) -> None: | ||||||
cmd = self.task_data.get('run', '').strip() | ||||||
if 'vars' not in self.group_data: | ||||||
self.group_data['vars'] = {} | ||||||
|
||||||
if not isinstance(self.group_data['vars'], dict): | ||||||
if not isinstance(self.group_data.get('vars', {}), dict): | ||||||
MakimLogs.raise_error( | ||||||
'`vars` attribute inside the group ' | ||||||
f'{self.group_name} is not a dictionary.', | ||||||
MakimError.MAKIM_VARS_ATTRIBUTE_INVALID, | ||||||
) | ||||||
|
||||||
env, variables = self._load_scoped_data('task') | ||||||
for k, v in env.items(): | ||||||
os.environ[k] = v | ||||||
# Get matrix configuration if it exists | ||||||
matrix_combinations: list[dict[str, Any]] = [{}] | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
if matrix_config := self.task_data.get('matrix', {}): | ||||||
matrix_combinations = self._generate_matrix_combinations( | ||||||
matrix_config | ||||||
) | ||||||
|
||||||
env, variables = self._load_scoped_data('task') | ||||||
self.env_scoped = deepcopy(env) | ||||||
|
||||||
args_input = {'file': self.file} | ||||||
|
@@ -565,36 +628,47 @@ def _run_command(self, args: dict[str, Any]) -> None: | |||||
MakimError.MAKIM_ARGUMENT_REQUIRED, | ||||||
) | ||||||
|
||||||
cmd = str(cmd) | ||||||
cmd = TEMPLATE.from_string(cmd).render( | ||||||
args=args_input, env=env, vars=variables | ||||||
) | ||||||
width, _ = get_terminal_size() | ||||||
|
||||||
if self.verbose: | ||||||
MakimLogs.print_info('=' * width) | ||||||
MakimLogs.print_info( | ||||||
'TARGET: ' + f'{self.group_name}.{self.task_name}' | ||||||
# Run command for each matrix combination | ||||||
for matrix_vars in matrix_combinations: | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
# Update environment variables | ||||||
for k, v in env.items(): | ||||||
os.environ[k] = v | ||||||
|
||||||
# Create a copy of variables and update with matrix values | ||||||
current_vars = deepcopy(variables) | ||||||
if matrix_vars: | ||||||
current_vars['matrix'] = matrix_vars | ||||||
|
||||||
# Render command with current matrix values | ||||||
current_cmd = TEMPLATE.from_string(cmd).render( | ||||||
args=args_input, env=env, vars=current_vars, matrix=matrix_vars | ||||||
) | ||||||
MakimLogs.print_info('ARGS:') | ||||||
MakimLogs.print_info(pprint.pformat(args_input)) | ||||||
MakimLogs.print_info('VARS:') | ||||||
MakimLogs.print_info(pprint.pformat(variables)) | ||||||
MakimLogs.print_info('ENV:') | ||||||
MakimLogs.print_info(str(env)) | ||||||
MakimLogs.print_info('-' * width) | ||||||
MakimLogs.print_info('>>> ' + cmd.replace('\n', '\n>>> ')) | ||||||
MakimLogs.print_info('=' * width) | ||||||
|
||||||
if not self.dry_run and cmd: | ||||||
self._call_shell_app(cmd) | ||||||
|
||||||
if self.verbose: | ||||||
# Prepare execution context for logging | ||||||
execution_context = { | ||||||
'args_input': args_input, | ||||||
'current_vars': current_vars, | ||||||
'env': env, | ||||||
'matrix_vars': matrix_vars, | ||||||
} | ||||||
# Log command execution details | ||||||
self._log_command_execution(execution_context, width) | ||||||
MakimLogs.print_info( | ||||||
'>>> ' + current_cmd.replace('\n', '\n>>> ') | ||||||
) | ||||||
MakimLogs.print_info('=' * width) | ||||||
|
||||||
if not self.dry_run and current_cmd: | ||||||
self._call_shell_app(current_cmd) | ||||||
|
||||||
# move back the environment variable to the previous values | ||||||
os.environ.clear() | ||||||
os.environ.update(self.env_scoped) | ||||||
|
||||||
# public methods | ||||||
|
||||||
# public methods | ||||||
def load( | ||||||
self, file: str, dry_run: bool = False, verbose: bool = False | ||||||
) -> None: | ||||||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
version: 1.0.0 | ||
groups: | ||
build: | ||
vars: | ||
app_name: "myapp" | ||
base_image: "node:18" | ||
tasks: | ||
setup: | ||
help: Setup development environment | ||
matrix: | ||
env: | ||
- dev | ||
- test | ||
- prod | ||
arch: | ||
- amd64 | ||
- arm64 | ||
run: | | ||
echo "[Setup] Environment: ${{ matrix.env }}, Architecture: ${{ matrix.arch }}" | ||
echo "[Setup] Using base image: ${{ vars.base_image }}" | ||
echo "[Setup] Installing dependencies for ${{ matrix.env }}" | ||
|
||
lint: | ||
help: Run linting checks | ||
matrix: | ||
type: | ||
- eslint | ||
- prettier | ||
path: | ||
- src | ||
- tests | ||
args: | ||
fix: | ||
type: bool | ||
default: false | ||
help: Automatically fix issues | ||
run: | | ||
echo "[Lint] Running ${{ matrix.type }} on ${{ matrix.path }}" | ||
{% if args.fix %} | ||
echo "[Lint] Auto-fixing enabled for ${{ matrix.type }}" | ||
{% endif %} | ||
|
||
test: | ||
vars: | ||
timeout: "30s" | ||
tasks: | ||
unit: | ||
help: Run unit tests across Node versions | ||
matrix: | ||
node: | ||
- "16" | ||
- "18" | ||
- "20" | ||
mode: | ||
- basic | ||
- coverage | ||
run: | | ||
echo "[Test] Using Node.js ${{ matrix.node }}" | ||
echo "[Test] Mode: ${{ matrix.mode }}" | ||
echo "[Test] Timeout set to ${{ vars.timeout }}" | ||
|
||
browser: | ||
help: Run browser tests | ||
matrix: | ||
browser: | ||
- chrome | ||
- firefox | ||
viewport: | ||
- desktop | ||
- mobile | ||
args: | ||
headless: | ||
type: bool | ||
default: true | ||
help: Run in headless mode | ||
run: | | ||
echo "[Browser Test] Testing on ${{ matrix.browser }}" | ||
echo "[Browser Test] Viewport: ${{ matrix.viewport }}" | ||
{% if args.headless %} | ||
echo "[Browser Test] Running in headless mode" | ||
{% else %} | ||
echo "[Browser Test] Running in GUI mode" | ||
{% endif %} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.