diff --git a/BUILD b/BUILD index 5085299..a8254d9 100644 --- a/BUILD +++ b/BUILD @@ -20,6 +20,8 @@ load("@rules_python//python:pip.bzl", "compile_pip_requirements") exports_files([ "mypy.ini", "requests.txt", + "README.header.txt", + "README.md", ]) compile_pip_requirements( diff --git a/README.md b/README.md index bbbfcd3..cf8cc05 100644 --- a/README.md +++ b/README.md @@ -40,50 +40,50 @@ bazel run //circleci:workflows -- combine --output=/tmp/circleci.csv "${PWD}/dat ### positional arguments: -input +`input` - List of CSV files generated from `workflow.py fetch`. +> List of CSV files generated from `workflow.py fetch`. ### options: --h, --help +`-h, --help` - show this help message and exit +> show this help message and exit ---circleci_server CIRCLECI_SERVER +`--circleci_server CIRCLECI_SERVER` - The circleci server url including protocol (defaults to environment - variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). +> The circleci server url including protocol (defaults to environment +> variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). ---circleci_token CIRCLECI_TOKEN +`--circleci_token CIRCLECI_TOKEN` - CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') +> CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') ---circleci_project_slug CIRCLECI_PROJECT_SLUG +`--circleci_project_slug CIRCLECI_PROJECT_SLUG` - CircleCI project-slug (defaults to environment variable - 'CIRCLECI_PROJECT_SLUG'). +> CircleCI project-slug (defaults to environment variable +> 'CIRCLECI_PROJECT_SLUG'). ---log_requests_to_file LOG_REQUESTS_TO_FILE +`--log_requests_to_file LOG_REQUESTS_TO_FILE` - Whether to log all requests for debugging purposes. +> Whether to log all requests for debugging purposes. ---log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']} +`--log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']}` - Comma separated list of LogRequestDetails. +> Comma separated list of LogRequestDetails. ---output OUTPUT +`--output OUTPUT` - Name of the output file. +> Name of the output file. ---fetch_workflow_details, --no-fetch_workflow_details +`--fetch_workflow_details, --no-fetch_workflow_details` - Whether workflow details should automatically be added (if not present). +> Whether workflow details should automatically be added (if not present). ---progress, --no-progress +`--progress, --no-progress` - Whether to indicate progress (defaults to True if - `--fetch_workflow_details` is active). +> Whether to indicate progress (defaults to True if +> `--fetch_workflow_details` is active). ## Command fetch @@ -110,71 +110,71 @@ bazel run //circleci:workflows -- fetch --output "${PWD}/data/circleci_workflows ### options: --h, --help +`-h, --help` - show this help message and exit +> show this help message and exit ---circleci_server CIRCLECI_SERVER +`--circleci_server CIRCLECI_SERVER` - The circleci server url including protocol (defaults to environment - variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). +> The circleci server url including protocol (defaults to environment +> variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). ---circleci_token CIRCLECI_TOKEN +`--circleci_token CIRCLECI_TOKEN` - CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') +> CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') ---circleci_project_slug CIRCLECI_PROJECT_SLUG +`--circleci_project_slug CIRCLECI_PROJECT_SLUG` - CircleCI project-slug (defaults to environment variable - 'CIRCLECI_PROJECT_SLUG'). +> CircleCI project-slug (defaults to environment variable +> 'CIRCLECI_PROJECT_SLUG'). ---log_requests_to_file LOG_REQUESTS_TO_FILE +`--log_requests_to_file LOG_REQUESTS_TO_FILE` - Whether to log all requests for debugging purposes. +> Whether to log all requests for debugging purposes. ---log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']} +`--log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']}` - Comma separated list of LogRequestDetails. +> Comma separated list of LogRequestDetails. ---workflow WORKFLOW +`--workflow WORKFLOW` - The name of the workflow(s) to read. Multiple workflows can be read by - separating with comma. If no workflow is set, then fetch all workflows. +> The name of the workflow(s) to read. Multiple workflows can be read by +> separating with comma. If no workflow is set, then fetch all workflows. ---output OUTPUT +`--output OUTPUT` - Name of the output file. +> Name of the output file. ---end END +`--end END` - End (newest) date/time in Python [ISO - 8601](https://en.wikipedia.org/wiki/ISO_8601) format, e.g. `200241224` - or as a negative time difference, e.g. `-10days` (for details see - [pytimeparse](https://github.com/wroberts/pytimeparse)). +> End (newest) date/time in Python [ISO +> 8601](https://en.wikipedia.org/wiki/ISO_8601) format, e.g. `200241224` +> or as a negative time difference, e.g. `-10days` (for details see +> [pytimeparse](https://github.com/wroberts/pytimeparse)). +> +> This defaults to `now`. - This defaults to `now`. +`--start START` ---start START +> Start (oldest) date/time in Python [ISO +> 8601](https://en.wikipedia.org/wiki/ISO_8601) format, e.g. `200241224` +> or as a negative time difference, e.g. `-10days` (for details see +> [pytimeparse](https://github.com/wroberts/pytimeparse)). +> +> This defaults to `-90days` (or `-89days` if --midnight is active). - Start (oldest) date/time in Python [ISO - 8601](https://en.wikipedia.org/wiki/ISO_8601) format, e.g. `200241224` - or as a negative time difference, e.g. `-10days` (for details see - [pytimeparse](https://github.com/wroberts/pytimeparse)). +`--midnight, --no-midnight` - This defaults to `-90days` (or `-89days` if --midnight is active). +> Adjust start and end date/time to midnight of the same day. ---midnight, --no-midnight +`--progress, --no-progress` - Adjust start and end date/time to midnight of the same day. +> Whether to indicate progress (defaults to True if +> `--fetch_workflow_details` is active). ---progress, --no-progress +`--fetch_workflow_details, --no-fetch_workflow_details` - Whether to indicate progress (defaults to True if - `--fetch_workflow_details` is active). - ---fetch_workflow_details, --no-fetch_workflow_details - - Whether workflow details should automatically be added. +> Whether workflow details should automatically be added. ## Command fetch_details @@ -186,43 +186,43 @@ bazel run //circleci:workflows -- fetch_details --input "${PWD}/data/circleci_wo ### options: --h, --help +`-h, --help` - show this help message and exit +> show this help message and exit ---circleci_server CIRCLECI_SERVER +`--circleci_server CIRCLECI_SERVER` - The circleci server url including protocol (defaults to environment - variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). +> The circleci server url including protocol (defaults to environment +> variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). ---circleci_token CIRCLECI_TOKEN +`--circleci_token CIRCLECI_TOKEN` - CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') +> CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') ---circleci_project_slug CIRCLECI_PROJECT_SLUG +`--circleci_project_slug CIRCLECI_PROJECT_SLUG` - CircleCI project-slug (defaults to environment variable - 'CIRCLECI_PROJECT_SLUG'). +> CircleCI project-slug (defaults to environment variable +> 'CIRCLECI_PROJECT_SLUG'). ---log_requests_to_file LOG_REQUESTS_TO_FILE +`--log_requests_to_file LOG_REQUESTS_TO_FILE` - Whether to log all requests for debugging purposes. +> Whether to log all requests for debugging purposes. ---log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']} +`--log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']}` - Comma separated list of LogRequestDetails. +> Comma separated list of LogRequestDetails. ---input INPUT +`--input INPUT` - A CSV file generated from `workflow.py fetch`. +> A CSV file generated from `workflow.py fetch`. ---output OUTPUT +`--output OUTPUT` - Name of the output file. +> Name of the output file. ---progress, --no-progress +`--progress, --no-progress` - Whether to indicate progress. +> Whether to indicate progress. ## Command filter @@ -234,53 +234,53 @@ bazel run //circleci:workflows -- filter --workflow default_workflow,pre_merge - ### options: --h, --help +`-h, --help` - show this help message and exit +> show this help message and exit ---workflow WORKFLOW +`--workflow WORKFLOW` - The name of the workflow(s) to accept. Multiple workflows can be userd - by separating with comma. If no workflow is set, then accept all - workflows. +> The name of the workflow(s) to accept. Multiple workflows can be userd +> by separating with comma. If no workflow is set, then accept all +> workflows. ---input INPUT +`--input INPUT` - CSV file generated from `workflow.py fetch`. +> CSV file generated from `workflow.py fetch`. ---output OUTPUT +`--output OUTPUT` - Name of the output file. +> Name of the output file. ---min_duration_sec MIN_DURATION_SEC +`--min_duration_sec MIN_DURATION_SEC` - Mininum duration to accept row in [sec]. +> Mininum duration to accept row in [sec]. ---output_duration_as_mins, --no-output_duration_as_mins +`--output_duration_as_mins, --no-output_duration_as_mins` - Whether to report duration values in minutes. +> Whether to report duration values in minutes. ---exclude_branches EXCLUDE_BRANCHES +`--exclude_branches EXCLUDE_BRANCHES` - Exclude branches by full regular expression match. +> Exclude branches by full regular expression match. ---exclude_incomplete_reruns, --no-exclude_incomplete_reruns +`--exclude_incomplete_reruns, --no-exclude_incomplete_reruns` - If workflow details are available, reject inomplete reruns (e.g.: rerun- - single-job, rerun-workflow-from-failed). +> If workflow details are available, reject inomplete reruns (e.g.: rerun- +> single-job, rerun-workflow-from-failed). ---only_branches ONLY_BRANCHES +`--only_branches ONLY_BRANCHES` - Accept branches by full regular expression match. +> Accept branches by full regular expression match. ---only_status ONLY_STATUS +`--only_status ONLY_STATUS` - Accept only listed status values (multiple separated by comma). +> Accept only listed status values (multiple separated by comma). ---only_weekdays ONLY_WEEKDAYS +`--only_weekdays ONLY_WEEKDAYS` - Accept only the listed days of the week as indexed 1=Monday through - 7=Sunday (ISO notation). +> Accept only the listed days of the week as indexed 1=Monday through +> 7=Sunday (ISO notation). ## Command request_branches @@ -295,36 +295,36 @@ bazel run //circleci:workflows -- request_branches ### options: --h, --help +`-h, --help` - show this help message and exit +> show this help message and exit ---circleci_server CIRCLECI_SERVER +`--circleci_server CIRCLECI_SERVER` - The circleci server url including protocol (defaults to environment - variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). +> The circleci server url including protocol (defaults to environment +> variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). ---circleci_token CIRCLECI_TOKEN +`--circleci_token CIRCLECI_TOKEN` - CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') +> CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') ---circleci_project_slug CIRCLECI_PROJECT_SLUG +`--circleci_project_slug CIRCLECI_PROJECT_SLUG` - CircleCI project-slug (defaults to environment variable - 'CIRCLECI_PROJECT_SLUG'). +> CircleCI project-slug (defaults to environment variable +> 'CIRCLECI_PROJECT_SLUG'). ---log_requests_to_file LOG_REQUESTS_TO_FILE +`--log_requests_to_file LOG_REQUESTS_TO_FILE` - Whether to log all requests for debugging purposes. +> Whether to log all requests for debugging purposes. ---log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']} +`--log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']}` - Comma separated list of LogRequestDetails. +> Comma separated list of LogRequestDetails. ---workflow WORKFLOW +`--workflow WORKFLOW` - The name of the workflow to read. Multiple workflows can be read by - separating with comma. +> The name of the workflow to read. Multiple workflows can be read by +> separating with comma. ## Command request_workflow @@ -336,35 +336,35 @@ bazel run //circleci:workflows -- request_workflow --workflow_id ### options: --h, --help +`-h, --help` - show this help message and exit +> show this help message and exit ---circleci_server CIRCLECI_SERVER +`--circleci_server CIRCLECI_SERVER` - The circleci server url including protocol (defaults to environment - variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). +> The circleci server url including protocol (defaults to environment +> variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). ---circleci_token CIRCLECI_TOKEN +`--circleci_token CIRCLECI_TOKEN` - CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') +> CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') ---circleci_project_slug CIRCLECI_PROJECT_SLUG +`--circleci_project_slug CIRCLECI_PROJECT_SLUG` - CircleCI project-slug (defaults to environment variable - 'CIRCLECI_PROJECT_SLUG'). +> CircleCI project-slug (defaults to environment variable +> 'CIRCLECI_PROJECT_SLUG'). ---log_requests_to_file LOG_REQUESTS_TO_FILE +`--log_requests_to_file LOG_REQUESTS_TO_FILE` - Whether to log all requests for debugging purposes. +> Whether to log all requests for debugging purposes. ---log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']} +`--log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']}` - Comma separated list of LogRequestDetails. +> Comma separated list of LogRequestDetails. ---workflow_id WORKFLOW_ID +`--workflow_id WORKFLOW_ID` - Workflow ID to request. +> Workflow ID to request. ## Command request_workflows @@ -376,28 +376,28 @@ bazel run //circleci:workflows -- request_workflows ### options: --h, --help +`-h, --help` - show this help message and exit +> show this help message and exit ---circleci_server CIRCLECI_SERVER +`--circleci_server CIRCLECI_SERVER` - The circleci server url including protocol (defaults to environment - variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). +> The circleci server url including protocol (defaults to environment +> variable 'CIRCLECI_SERVER' which defaults to 'https://circleci.com'). ---circleci_token CIRCLECI_TOKEN +`--circleci_token CIRCLECI_TOKEN` - CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') +> CircleCI Auth Token (defaults to environment variable 'CIRCLECI_TOKEN') ---circleci_project_slug CIRCLECI_PROJECT_SLUG +`--circleci_project_slug CIRCLECI_PROJECT_SLUG` - CircleCI project-slug (defaults to environment variable - 'CIRCLECI_PROJECT_SLUG'). +> CircleCI project-slug (defaults to environment variable +> 'CIRCLECI_PROJECT_SLUG'). ---log_requests_to_file LOG_REQUESTS_TO_FILE +`--log_requests_to_file LOG_REQUESTS_TO_FILE` - Whether to log all requests for debugging purposes. +> Whether to log all requests for debugging purposes. ---log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']} +`--log_requests_details {['REQUEST', 'RESPONSE_TEXT', 'STATUS_CODE']}` - Comma separated list of LogRequestDetails. +> Comma separated list of LogRequestDetails. diff --git a/circleci/BUILD b/circleci/BUILD index c7f25f8..71adc50 100644 --- a/circleci/BUILD +++ b/circleci/BUILD @@ -15,6 +15,7 @@ """Tool to fetch and analyze CircleCI workflows.""" +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@my_pip_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") @@ -72,3 +73,20 @@ py_binary( "//mbo/app:commands_py", ], ) + +genrule( + name = "readme_gen", + testonly = True, + srcs = ["//:README.header.txt"], + outs = ["readme.txt"], + cmd = """ + ./$(location :workflows) help --help_output_mode=markdown --all_commands --prefix="$(location //:README.header.txt)" > "$@" + """, + tools = [":workflows"], +) + +diff_test( + name = "readme_test", + file1 = "//:README.md", + file2 = ":readme.txt", +) diff --git a/circleci/workflows_lib.py b/circleci/workflows_lib.py index 8b3e740..407e523 100644 --- a/circleci/workflows_lib.py +++ b/circleci/workflows_lib.py @@ -37,7 +37,11 @@ from circleci.circleci_api_v2 import CircleCiApiV2, CircleCiApiV2Opts, LogRequestDetail from mbo.app.commands import Command, Die, DocOutdent, Log, OpenTextFile, Print -from mbo.app.flags import EnumListAction, ParseDateTimeOrDelta +from mbo.app.flags import ( + ActionDateTimeOrTimeDelta, + ActionEnumList, + ParseDateTimeOrTimeDelta, +) # Keys used by the `fetch` command. # Instead of `created_at` and `stopped_at` we provide `created`/`created_unix` @@ -73,6 +77,11 @@ def TimeRangeStr(start: datetime, end: datetime) -> str: + if (start.tzinfo is None) != (end.tzinfo is None): + if start.tzinfo: + end = datetime.combine(end.date(), end.time(), tzinfo=start.tzinfo) + else: + start = datetime.combine(start.date(), start.time(), tzinfo=end.tzinfo) return f"Time range: [{start} .. {end}] ({humanize.precisedelta(end - start)})." @@ -118,7 +127,7 @@ def __init__(self) -> None: type=LogRequestDetail, allow_empty=False, container_type=set, - action=EnumListAction, + action=ActionEnumList, help="Comma separated list of LogRequestDetails.", ) @@ -294,8 +303,8 @@ def __init__(self): ) self.parser.add_argument( "--end", - default="", - type=str, + action=ActionDateTimeOrTimeDelta, + verify_only=True, help="""End (newest) date/time in Python [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format, e.g. `200241224` or as a negative time difference, e.g. `-10days` (for details see [pytimeparse](https://github.com/wroberts/pytimeparse)). @@ -306,7 +315,8 @@ def __init__(self): self.parser.add_argument( "--start", default="", - type=str, + action=ActionDateTimeOrTimeDelta, + verify_only=True, help="""Start (oldest) date/time in Python [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format, e.g. `200241224` or as a negative time difference, e.g. `-10days` (for details see [pytimeparse](https://github.com/wroberts/pytimeparse)). @@ -336,20 +346,17 @@ def __init__(self): def Main(self) -> None: if self.args.fetch_workflow_details and self.args.progress == None: self.args.progress = True - now = datetime.now() - now = datetime.combine( - now.date(), now.time(), tzinfo=now.tzinfo or timezone.utc - ) - end = ParseDateTimeOrDelta( - arg=self.args.end, + now = datetime.now(timezone.utc) + end = ParseDateTimeOrTimeDelta( + value=self.args.end, midnight=self.args.midnight, default=now, reference=now, error_prefix="Bad flag `--end` value '", error_suffix="'.", ) - start = ParseDateTimeOrDelta( - arg=self.args.start, + start = ParseDateTimeOrTimeDelta( + value=self.args.start, midnight=self.args.midnight, default=end - timedelta(days=90), reference=end, @@ -362,14 +369,20 @@ def Main(self) -> None: if self.args.midnight: if self.args.start: Log("Adjusting to midnight from 89 days ago.") - start = datetime.now() - timedelta(days=89) + start = datetime.now(timezone.utc) - timedelta(days=89) start = datetime( - start.year, start.month, start.day, tzinfo=start.tzinfo + start.year, + start.month, + start.day, + tzinfo=start.tzinfo or timezone.utc, ) else: if self.args.start: Log("Adjusting to 90 days ago.") - start = datetime.now() - timedelta(days=90) + start = datetime.now(timezone.utc) - timedelta(days=90) + start = datetime.combine( + start.date(), start.time(), tzinfo=start.tzinfo or timezone.utc + ) if start >= end: Die(f"Specified start time {start} must be before end time {end}!") Log(TimeRangeStr(start, end)) diff --git a/mbo/app/commands.py b/mbo/app/commands.py index 47aa970..e0690d4 100644 --- a/mbo/app/commands.py +++ b/mbo/app/commands.py @@ -182,12 +182,18 @@ def _format_action_invocation(self, action): action ) if self.IsOutputMode(HelpOutputMode.MARKDOWN): - return "\n" + result + "\n\n" + return "\n`" + result + "`\n\n" return result def _format_action(self, action) -> str: result: str = super(CommandParagraphFormatter, self)._format_action(action) - # ATM this works without extra space in either mode. + if self.IsOutputMode(HelpOutputMode.MARKDOWN): + text = [] + for r in result.split("\n"): + if r.startswith(" "): + r = ("> " + r.lstrip()).rstrip() + text.append(r) + result = "\n".join(text) return result def _format_usage(self, usage, actions, groups, prefix): @@ -349,6 +355,7 @@ class Help(Command): def __init__(self) -> None: super(Help, self).__init__() + self.exit_code = 0 self.parser.add_argument( "--all_commands", action=argparse.BooleanOptionalAction, @@ -462,4 +469,4 @@ def Main(self) -> None: cmd.parser.prog = self.program + " " + name cmd.parser.usage = argparse.SUPPRESS self.Print(cmd.parser.format_help()) - exit(1) + exit(self.exit_code) diff --git a/mbo/app/flags.py b/mbo/app/flags.py index c10a79f..9f97c40 100644 --- a/mbo/app/flags.py +++ b/mbo/app/flags.py @@ -17,13 +17,19 @@ import argparse import collections -from datetime import datetime, time, timedelta, timezone +import re +import sys +from datetime import datetime, time, timedelta, timezone, tzinfo from enum import Enum -from typing import Callable, Iterable, Optional, cast +from typing import Any, Callable, Iterable, Optional, cast from pytimeparse.timeparse import timeparse +def _Log(message=Any): + print(message, flush=True, file=sys.stderr) + + class EnumAction(argparse.Action): """Argparse action that handles single Enum values.""" @@ -47,7 +53,7 @@ def EnumListParser(enum_type: type[Enum]) -> Callable[[str], list[Enum]]: In the argument definition default values can be specified as a list of the actual enum values. On the command line the values do not have to be upper case (lowercase and mixed case are fine). - Note: In many cases `EnumListAction` provides a better solution for flag parsing. + Note: In many cases `ActionEnumList` provides a better solution for flag parsing. Example: ``` @@ -57,7 +63,7 @@ def EnumListParser(enum_type: type[Enum]) -> Callable[[str], list[Enum]]: type=EnumListParser(enum_type=MyEnum), help="Comma separated list of MyEnum {}.".format(set(MyEnum.__members__.keys())), ) - args=parser.parse_args({"--nyenum", "my_default,my_other"}) + args=parser.parse_args(["--nyenum", "my_default,my_other"]) ``` """ return lambda values: [ @@ -65,7 +71,7 @@ def EnumListParser(enum_type: type[Enum]) -> Callable[[str], list[Enum]]: ] -class EnumListAction(argparse.Action): +class ActionEnumList(argparse.Action): """Argparse `action` for comma separated lists of Enum values. This action has the additional config: @@ -78,18 +84,18 @@ class EnumListAction(argparse.Action): "--myenum", default=[MyEnum.MY_DEFAULT], type=MyEnum, - action=EnumListAction, + action=ActionEnumList, allow_empty=False, container_type=set, help="Comma separated list of MyEnum values.", ) - args=parser.parse_args({"--nyenum", "my_default,my_other"}) + args=parser.parse_args(["--nyenum", "my_default,my_other"]) ``` """ class Choices: def __init__(self, action: argparse.Action): - self._action: EnumListAction = cast(EnumListAction, action) + self._action: ActionEnumList = cast(ActionEnumList, action) def choices(self) -> Iterable[str]: return sorted(self._action._enum_type.__members__.keys()) @@ -136,46 +142,83 @@ def __init__(self, **kwargs): ) kwargs.setdefault("choices", self.Choices(action=self)) - super(EnumListAction, self).__init__(**kwargs) + super(ActionEnumList, self).__init__(**kwargs) - def __call__(self, parser, namespace, values, option_string=None): + def __call__(self, parser, namespace, values, option_string=None) -> None: if isinstance(values, list): - values = ",".join(values) + values_str = ",".join(values) + elif type(values) == str: + values_str = values + else: + raise argparse.ArgumentError(self, "Unexpected value type {type(values)}.") value = self._container_type( [ self._enum_type.__getitem__(v.strip().upper()) - for v in values.split(",") + for v in values_str.split(",") if v ] ) setattr(namespace, self.dest, value) -def ParseDateTimeOrDelta( - arg: str, +def _MaybeMidnight( + value: datetime, midnight: bool = False, tz: tzinfo = timezone.utc +) -> datetime: + if midnight: + return datetime.combine( + value.date(), time(0, 0, 0, 0), tzinfo=value.tzinfo or tz + ) + else: + return datetime.combine(value.date(), value.time(), tzinfo=value.tzinfo or tz) + + +def _ParseDateTime( + value: str | datetime, midnight: bool = False, tz: tzinfo = timezone.utc +) -> datetime: + if value is datetime: + return _MaybeMidnight(cast(datetime, value), midnight=midnight, tz=tz) + v = str(value) + if re.fullmatch("[0-9]{8}", v): + try: + return datetime( + year=int(v[0:4]), + month=int(v[4:6]), + day=int(v[6:8]), + tzinfo=tz, + ) + except ValueError as err: + raise ValueError(f"Invalid date string: '{v}', {err}") + return _MaybeMidnight(datetime.fromisoformat(v), midnight=midnight, tz=tz) + + +def ParseDateTimeOrTimeDelta( + value: str, midnight: bool = False, default: Optional[datetime] = None, reference: Optional[datetime] = None, + tz: tzinfo = timezone.utc, error_prefix: Optional[str] = None, error_suffix: Optional[str] = None, ) -> datetime: - """Parse `arg` as date or time delta in relation to reference. + """Parse `value` as date or time delta in relation to reference. - If `arg` starts with either `-` or `+`, then it will be parsed as `timedelta`. - Otherwise `arg` will be parsed as Python [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). + If `value` starts with either `-` or `+`, then it will be parsed as `timedelta`. + Otherwise `value` will be parsed as Python [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). Note: The returned type will have its timezone set from (in order precedence)): * The input time. * The reference time. * `datetime.now()`. + * `tz`. * `timezone.utc`. Args: - arg: The argument to parse. If this starts with a '-', then + value: The argument to parse. If this starts with a '-', then the argument will be interpreted as a time delta. + default: The value to use if `value` is empty (defaults to `datetime.now`). midnight: Whether to adjust the date/time to midnight of the day. - default: The value to use if `arg` is empty (defaults to `datetime.now`). reference: The reference datetime to use for time deltas (defaults to `datetime.now`). + tz: Fallback timezone. error_prefix: An optional error prefix prepended to messages of raised errors. By default this is "Bad timedelta value '". Together with error_suffix which defaults to "'." this allows to provide additional error information. @@ -184,22 +227,126 @@ def ParseDateTimeOrDelta( error_suffix: See `error_prefix`. """ result: datetime - if arg.startswith(("-", "+")): - seconds: float | None = timeparse(arg) + if value.startswith(("-", "+")): + seconds: float | None = timeparse(value) if type(seconds) == type(None): if error_prefix is None: error_prefix = "Bad timedelta value '" if error_suffix is None: error_suffix = "'." - raise ValueError(f"{error_prefix}{arg}{error_suffix}") - result = (reference or datetime.now()) + timedelta(seconds=seconds or 0) - elif arg: - result = datetime.fromisoformat(arg) + raise ValueError(f"{error_prefix}{value}{error_suffix}") + result = (reference or datetime.now(tz=tz)) + timedelta(seconds=seconds or 0) + elif value: + result = _ParseDateTime(value, midnight=midnight, tz=tz) else: - result = default or datetime.now() + result = default or datetime.now(tz=tz) if not result.tzinfo: - tzinfo = (reference or datetime.now()).tzinfo or timezone.utc - result = datetime.combine(result.date(), result.time(), tzinfo=tzinfo) + result = datetime.combine( + result.date(), + result.time(), + tzinfo=(reference or datetime.now(tz=tz)).tzinfo or tz, + ) if midnight: return datetime.combine(result.date(), time(), tzinfo=result.tzinfo) return result + + +class ActionDateTimeOrTimeDelta(argparse.Action): + """Action to parse (or verify) a `datetime` with `timedelta` support. + + If `value` starts with either `-` or `+`, then it will be parsed as `timedelta`. + Otherwise `value` will be parsed as Python [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). + + This action has the additional config: + * default: The value to use if `value` is empty (defaults to `datetime.now`). + * midnight: Whether to adjust the date/time to midnight of the day. + * reference: The reference datetime to use for time deltas (defaults to `datetime.now`). + * tz: Fallback timezone. + * verify_only: If True (False is default), then the input will only be verified. The resulting + value is the input and its type is `str`. + + Example: + ``` + parser.add_argument( + "--time", + action=ActionDateTimeOrTimeDelta, + help="Parses flag as a `datetime` value; or a `timedelta` value relative to `reference`.", + ) + args=parser.parse_args(["--time", "+1week"]) + ``` + """ + + def __init__(self, **kwargs) -> None: + self._verify_only = kwargs.pop("verify_only", False) + self._midnight = kwargs.pop("midnight", False) + self._tz = kwargs.pop("tz", timezone.utc) + self._type = kwargs.pop("type", str if self._verify_only else datetime) + now = datetime.now(self._tz) + default_v = kwargs.pop("default", now) + reference = kwargs.pop("reference", default_v) + + super(ActionDateTimeOrTimeDelta, self).__init__(**kwargs) + if self._verify_only: + if self._type != str: + raise argparse.ArgumentError( + self, + f"Type (for verification) must be `str`, provided type is `{self._type}`.", + ) + else: + if self._type != datetime: + raise argparse.ArgumentError( + self, f"Type must be `datetime`, provided type is `{self._type}`." + ) + try: + if default_v: + default = _ParseDateTime( + default_v, midnight=self._midnight, tz=self._tz + ) + else: + default = None + except ValueError as error: + raise argparse.ArgumentError( + self, f"Default value `{default_v}` cannot be parsed as `datetime`." + ) + if self._verify_only: + self.default = str(default or default_v) + elif type(default) == datetime: + self.default = default + else: + raise argparse.ArgumentError( + self, + f"Default value must be of type datetime, provided is `{type(default)}`.", + ) + try: + self._reference: datetime = _ParseDateTime( + reference or now, midnight=self._midnight, tz=self._tz + ) + except ValueError as error: + raise argparse.ArgumentError( + self, f"Reference value `{reference}` cannot be parsed as `datetime`." + ) + + def _parse(self, value) -> datetime: + try: + return ParseDateTimeOrTimeDelta( + value=value, + default=self.default, + midnight=self._midnight, + reference=self._reference, + tz=self._tz, + ) + except ValueError as error: + raise argparse.ArgumentError(self, f"{error}") + + def __call__(self, parser, namespace, values, option_string=None) -> None: + result: Any + if values is list: + result = [self._parse(v) for v in values] + elif type(values) == str: + result = self._parse(values) + else: + raise argparse.ArgumentError(self, "Unexpected value type {type(values)}.") + if self._verify_only: + setattr(namespace, self.dest, values) + else: + setattr(namespace, self.dest, result) diff --git a/mbo/app/flags_test.py b/mbo/app/flags_test.py index 115472c..efba2e6 100644 --- a/mbo/app/flags_test.py +++ b/mbo/app/flags_test.py @@ -18,7 +18,7 @@ import argparse import unittest from dataclasses import dataclass, is_dataclass -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from typing import Any @@ -30,7 +30,7 @@ _NOW = "2024-08-28T14:15:16.123Z" -def ActionArgs(name: str = "--flag", **kwargs) -> dict[str, Any]: +def ActionArgs(name: str = "flag", **kwargs) -> dict[str, Any]: kwargs["name"] = name return kwargs @@ -58,7 +58,7 @@ class FlagsTest(unittest.TestCase): @dataclass_as_param @dataclass(kw_only=True) - class ParseDateTimeOrDeltaTest: + class ParseDateTimeOrTimeDeltaTest: expected: str expected_error: type | None = None input: str @@ -71,58 +71,58 @@ class ParseDateTimeOrDeltaTest: @parameterized.expand( [ - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-04-02T14:00:00Z", input="2024-04-02T14", ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-04-02T01:02:03.004Z", input="2024-04-02T01:02:03.004Z", ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-04-02T00:00:00Z", input="2024-04-02T13:14:15.123Z", midnight=True, ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-08-21T14:15:16.123Z", input="-1w", ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-08-29T14:15:16.123Z", input="+1d", ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-08-29T00:00:00Z", input="+1d", midnight=True, ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-08-14 01:00:00Z", input="+8h", reference="2024-08-13 17", midnight=False, ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-08-14 00:00:00Z", input="+8h", reference="2024-08-13 17", midnight=True, ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="2024-08-14 00:00:00Z", input="+0s", reference="2024-08-14 17", midnight=True, ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="Bad timedelta value '+0 NOPE'.", expected_error=ValueError, input="+0 NOPE", reference="2024-08-14 17", midnight=True, ), - ParseDateTimeOrDeltaTest( + ParseDateTimeOrTimeDeltaTest( expected="+0 NOPE", expected_error=ValueError, input="+0 NOPE", @@ -133,7 +133,7 @@ class ParseDateTimeOrDeltaTest: ), ] ) - def test_ParseDateTimeOrDelta(self, test: ParseDateTimeOrDeltaTest): + def test_ParseDateTimeOrTimeDelta(self, test: ParseDateTimeOrTimeDeltaTest): with freeze_time(datetime.fromisoformat(test.now)): try: self.assertEqual( @@ -142,8 +142,8 @@ def test_ParseDateTimeOrDelta(self, test: ParseDateTimeOrDeltaTest): if not test.expected_error else None ), - mbo.app.flags.ParseDateTimeOrDelta( - arg=test.input, + mbo.app.flags.ParseDateTimeOrTimeDelta( + value=test.input, midnight=test.midnight, default=( datetime.fromisoformat(test.default) @@ -167,99 +167,125 @@ def test_ParseDateTimeOrDelta(self, test: ParseDateTimeOrDeltaTest): @dataclass_as_param @dataclass(kw_only=True) - class EnumListActionTest: + class FlagTestData: test: str expected: Any expected_error: type | None = None action: dict[str, Any] input: list[str] + def FlagTest(self, test: FlagTestData) -> None: + try: + parser = argparse.ArgumentParser(exit_on_error=False) + name = test.action.pop("name", "flag") + parser.add_argument(name, **test.action) + args = parser.parse_args(test.input) + self.assertEqual( + test.expected, args.flag, "Bad value in test: " + test.test + ) + except argparse.ArgumentError as error: + self.assertIsNotNone(test.expected_error, error) + if test.expected_error: + self.assertEqual( + test.expected, str(error), "Bad error message in test: " + test.test + ) + self.assertEqual( + type(error), + test.expected_error, + "Bad error type in test: " + test.test, + ) + @parameterized.expand( [ - EnumListActionTest( + FlagTestData( test="Set a single value to a list.", expected=[TestEnum.ONE], action=ActionArgs( type=TestEnum, - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, ), - input=["--flag=one"], + input=["one"], ), - EnumListActionTest( + FlagTestData( test="Setting an empty vlaue requires `allow_empty=True`.", expected="argument --flag: Empty value is not allowed, chose at least one of [FOR, ONE, TRE, TWO].", expected_error=argparse.ArgumentError, action=ActionArgs( + "--flag", type=TestEnum, default=[], - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, ), input=["--flag="], ), - EnumListActionTest( + FlagTestData( test="Setting an empty vlaue requires `allow_empty=True` (not False).", expected="argument --flag: Empty value is not allowed, chose at least one of [FOR, ONE, TRE, TWO].", expected_error=argparse.ArgumentError, action=ActionArgs( + "--flag", type=TestEnum, default=[], - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, allow_empty=False, ), input=["--flag="], ), - EnumListActionTest( + FlagTestData( test="Setting an empty vlaue requires with `allow_empty=True` works.", expected=[], expected_error=argparse.ArgumentError, action=ActionArgs( + "--flag", type=TestEnum, default=[], - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, allow_empty=True, ), input=["--flag="], ), - EnumListActionTest( + FlagTestData( test="Default values work.", expected=[TestEnum.TWO], action=ActionArgs( + "--flag", type=TestEnum, default=[TestEnum.TWO], - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, ), input=[], ), - EnumListActionTest( + FlagTestData( test="Default values work: They can even bypass the type.", expected="Something else", action=ActionArgs( + "--flag", type=TestEnum, default="Something else", - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, ), input=[], ), - EnumListActionTest( + FlagTestData( test="Multile, possible repeated values and mixed case.", expected=[TestEnum.TWO, TestEnum.ONE, TestEnum.TWO], action=ActionArgs( type=TestEnum, - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, ), - input=["--flag=two,oNe,TWO"], + input=["two,oNe,TWO"], ), - EnumListActionTest( + FlagTestData( test="Multile values in a set.", expected=set([TestEnum.ONE, TestEnum.TWO]), action=ActionArgs( type=TestEnum, container_type=set, - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, ), - input=["--flag=two,oNe,TWO"], + input=["two,oNe,TWO"], ), - EnumListActionTest( + FlagTestData( test="Repeated flag for list.", expected=[ TestEnum.TWO, @@ -269,47 +295,105 @@ class EnumListActionTest: TestEnum.TWO, ], action=ActionArgs( - "flag", nargs="+", type=TestEnum, - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, ), input=["two,for", "one,tre", "TWO"], ), - EnumListActionTest( + FlagTestData( test="Repeated flag for list.", expected={TestEnum.TWO, TestEnum.FOR, TestEnum.ONE, TestEnum.TRE}, action=ActionArgs( - "flag", nargs="+", type=TestEnum, container_type=set, - action=mbo.app.flags.EnumListAction, + action=mbo.app.flags.ActionEnumList, ), input=["two,for", "one,tre", "TWO"], ), ] ) - def test_EnumListAction(self, test: EnumListActionTest): - try: - parser = argparse.ArgumentParser(exit_on_error=False) - name = test.action.pop("name", "--flag") - parser.add_argument(name, **test.action) - args = parser.parse_args(test.input) - self.assertEqual( - test.expected, args.flag, "Bad value in test: " + test.test - ) - except argparse.ArgumentError as error: - self.assertIsNotNone(test.expected_error, error) - if test.expected_error: - self.assertEqual( - test.expected, str(error), "Bad error message in test: " + test.test - ) - self.assertEqual( - type(error), - test.expected_error, - "Bad error type in test: " + test.test, - ) + def test_EnumListAction(self, test: FlagTestData): + self.FlagTest(test) + + @parameterized.expand( + [ + FlagTestData( + test="Parse from iso datetime.", + expected=datetime( + year=2024, + month=1, + day=30, + hour=13, + minute=14, + second=51, + tzinfo=timezone.utc, + ), + action=ActionArgs( + action=mbo.app.flags.ActionDateTimeOrTimeDelta, + ), + input=["2024-01-30T13:14:51"], + ), + FlagTestData( + test="Parse from iso datetime applying midnight.", + expected=datetime(year=2024, month=1, day=30, tzinfo=timezone.utc), + action=ActionArgs( + action=mbo.app.flags.ActionDateTimeOrTimeDelta, + midnight=True, + ), + input=["2024-01-30T13:14:51"], + ), + FlagTestData( + test="Parse from short datetime.", + expected=datetime(year=2024, month=1, day=30, tzinfo=timezone.utc), + action=ActionArgs( + action=mbo.app.flags.ActionDateTimeOrTimeDelta, + ), + input=["20240130"], + ), + FlagTestData( + test="Parse from short datetime, bad input.", + expected="argument flag: Invalid date string: '20240230', day is out of range for month", + expected_error=argparse.ArgumentError, + action=ActionArgs( + action=mbo.app.flags.ActionDateTimeOrTimeDelta, + ), + input=["20240230"], + ), + FlagTestData( + test="Parse from timedelta.", + expected=datetime(2024, 9, 4, 14, 15, 16, 123000, timezone.utc), + action=ActionArgs( + action=mbo.app.flags.ActionDateTimeOrTimeDelta, + reference=_NOW, + ), + input=["+1w"], + ), + FlagTestData( + test="Parse from timedelta applying midnight.", + expected=datetime(2024, 9, 4, 0, 0, 0, 0, timezone.utc), + action=ActionArgs( + action=mbo.app.flags.ActionDateTimeOrTimeDelta, + reference=_NOW, + midnight=True, + ), + input=["+1w"], + ), + FlagTestData( + test="Parse from negative timedelta.", + expected=datetime(2024, 8, 21, 14, 15, 16, 123000, timezone.utc), + action=ActionArgs( + "--flag", + action=mbo.app.flags.ActionDateTimeOrTimeDelta, + reference=_NOW, + ), + input=["--flag=-1w"], + ), + ] + ) + def test_DateTimeOrTimeDeltaAction(self, test: FlagTestData): + self.FlagTest(test) if __name__ == "__main__":