From 97b7a806684221fea9b2b5d52618c3f0ec0a5afe Mon Sep 17 00:00:00 2001 From: Nathan Freeman Date: Wed, 4 Oct 2023 10:50:50 -0500 Subject: [PATCH] add template type task to choices in model. Refactor requests. Sync schema to requests in api. Update refs after refactor of requests --- .../migrations/0023_alter_task_type.py | 19 ++ src/api/src/backend/views/http/requests.py | 149 +++++++++------- src/engine/src/owe_python_sdk/client.py | 2 +- src/engine/src/owe_python_sdk/schema.py | 167 +++++++++++------- 4 files changed, 208 insertions(+), 129 deletions(-) create mode 100644 src/api/src/backend/migrations/0023_alter_task_type.py diff --git a/src/api/src/backend/migrations/0023_alter_task_type.py b/src/api/src/backend/migrations/0023_alter_task_type.py new file mode 100644 index 00000000..1eab00e1 --- /dev/null +++ b/src/api/src/backend/migrations/0023_alter_task_type.py @@ -0,0 +1,19 @@ +# Generated by Django 4.1.2 on 2023-10-02 20:05 + +import backend.views.http.requests +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0022_task_uses'), + ] + + operations = [ + migrations.AlterField( + model_name='task', + name='type', + field=models.CharField(choices=[(backend.views.http.requests.EnumTaskType['Template'], backend.views.http.requests.EnumTaskType['Template']), (backend.views.http.requests.EnumTaskType['ImageBuild'], backend.views.http.requests.EnumTaskType['ImageBuild']), (backend.views.http.requests.EnumTaskType['Application'], backend.views.http.requests.EnumTaskType['Application']), (backend.views.http.requests.EnumTaskType['Request'], backend.views.http.requests.EnumTaskType['Request']), (backend.views.http.requests.EnumTaskType['Function'], backend.views.http.requests.EnumTaskType['Function']), (backend.views.http.requests.EnumTaskType['TapisJob'], backend.views.http.requests.EnumTaskType['TapisJob']), (backend.views.http.requests.EnumTaskType['TapisActor'], backend.views.http.requests.EnumTaskType['TapisActor'])], max_length=32), + ), + ] \ No newline at end of file diff --git a/src/api/src/backend/views/http/requests.py b/src/api/src/backend/views/http/requests.py index 56a41f30..57fe634e 100644 --- a/src/api/src/backend/views/http/requests.py +++ b/src/api/src/backend/views/http/requests.py @@ -180,17 +180,62 @@ def validate(cls, v): raise ValueError("Invalid 'id' format. Must start with alphanumeric chars followed by 0 or more alphanumeric chars, '_', and '-'") return v -class BaseValue(BaseModel): - type: EnumTaskIOTypes = EnumTaskIOTypes.String - value: Union[str, int, float, bool, bytes] = None - value_from: Dict[str, str] = None +class TaskOutputRef(BaseModel): + task_id: str + output_id: str + +LiteralHostRefTypes = Literal["kubernetes_secret", "kubernetes_config_map"] +class ValueFromHostRef(BaseModel): + type: LiteralHostRefTypes + name: str + field_selector: str = None + +class ValueFromSecretRef(BaseModel): + engine: str + pk: str + field_selector: str = None + +ValueFromEnv = Dict[Literal["env"], str] +ValueFromParams = Dict[Literal["params"], str] +ValueFromTaskOutput = Dict[Literal["task_output"], TaskOutputRef] +ValueFromHost = Dict[Literal["host"], ValueFromHostRef] +ValueFromSecret = Dict[Literal["secret"], ValueFromSecretRef] +ValueFromAll = Union[ + ValueFromEnv, + ValueFromParams, + ValueFromTaskOutput, + ValueFromHost, + ValueFromSecret +] + +EnvSpecValueFrom = Union[ + ValueFromSecret, + ValueFromHost +] + +# NOTE Same as EnvSpecValueFrom for now, but may diverge. Delete this NOTE once it does +# Also, no good name for the combination of the 2. EnvParamSpecValueFrom?? No thanks. +ParamSpecValueFrom = Union[ + ValueFromSecret, + ValueFromHost +] + +Value = Union[str, int, float, bool, bytes] + +class Spec(BaseModel): + required: bool = False + type: EnumTaskIOTypes + value: Value = None + value_from: ValueFromAll = None @root_validator(pre=True) def value_or_value_from(cls, values): # Either 'value' or 'value_from' must be set on every variable if ( - values.get("value", None) == None - and values.get("value_from", None) == None + ( + values.get("value", None) == None + and values.get("value_from", None) == None + ) and values.get("required") ): raise ValueError( "Missing 'value' or 'value_from' property in variable values") @@ -226,44 +271,25 @@ def value_from_type_validation(cls, value): "Task Input Value Error: 'value_from' property must be a single key-value pair where the key is oneOf ['env', 'params', 'task_input', 'host', 'secret'] and the value is a non-empty string if key is oneOf ['env', 'params'] or an object if key is oneOf ['task_input', 'host', 'secret']" ) return value - -class TaskOutputRef(BaseModel): - task_id: str - output_id: str +class TaskInputSpec(Spec): + value: Value = None + value_from: ValueFromAll = None -LiteralHostRefTypes = Literal["kubernetes_secret", "kubernetes_config_map"] -class ValueFromHostRef(BaseModel): - type: LiteralHostRefTypes - name: str - field_selector: str = None +class EnvSpec(Spec): + value: Value = None + value_from: EnvSpecValueFrom = None -class ValueFromSecretRef(BaseModel): - engine: str - pk: str - field_selector: str = None +Env = Dict[str, EnvSpec] -ValueFromEnv = Dict[Literal["env"], str] -ValueFromParams = Dict[Literal["params"], str] -ValueFromTaskOutput = Dict[Literal["task_output"], TaskOutputRef] -ValueFromHost = Dict[Literal["host"], ValueFromHostRef] -ValueFromSecret = Dict[Literal["secret"], ValueFromSecretRef] - -class TaskInputValue(BaseValue): +class ParamSpec(Spec): type: EnumTaskIOTypes - value: Union[str, int, float, bool, bytes] = None - value_from: Union[ - ValueFromEnv, - ValueFromParams, - ValueFromTaskOutput, - ValueFromHost, - ValueFromSecret - ] = None + value: Value = None + value_from: ParamSpecValueFrom = None -class ValueWithRequirements(BaseValue): - required: bool = False +Params = Dict[str, ParamSpec] -KeyVal = Dict[str, BaseValue] +KeyVal = Dict[str, Spec] ################## /Common ################### class S3Auth(BaseModel): @@ -362,7 +388,7 @@ class IdentityCreateRequest(BaseModel): DockerhubAuth ] -class BaseContext(BaseModel): +class Context(BaseModel): type: LiteralContextTypes branch: str = None build_file_path: str = None @@ -379,14 +405,14 @@ class BaseContext(BaseModel): class Config: extra: Extra.allow -class GithubContext(BaseContext): +class GithubContext(Context): type: Literal["github"] credentials: GithubAuth = None branch: str build_file_path: str = None sub_path: str = None -class DockerhubContext(BaseContext): +class DockerhubContext(Context): type: Literal["dockerhub"] credentials: DockerhubAuth = None @@ -427,7 +453,7 @@ class BaseEvent(BaseModel): username: str = None class APIEvent(BaseEvent): - params: KeyVal = {} + params: Params = {} directives: List[str] = None class WebhookEvent(BaseEvent): @@ -465,9 +491,8 @@ class TaskDependency(BaseModel): # Output ----------------------------------------------------------------- -class BaseOutputValue(BaseModel): +class TaskOutputSpec(BaseModel): type: EnumTaskIOTypes - value: Union[str, int, float, bool, bytes] # ------------------------------------------------------------------------ @@ -500,8 +525,14 @@ class ClonedGitRepository(GitRepository): directory: str class Uses(BaseModel): - name: str - git_repository: GitRepository + source: GitRepository + name: str = None + path: str = None + + @root_validator + def must_provide_name_or_path(cls, values): + if values.get("name", None) == None and values.get("path", None) == None: + raise ValueError("The 'uses' property of a Template Task must specify either a 'name' or 'path' property") class BaseTask(BaseModel): id: ID @@ -509,8 +540,8 @@ class BaseTask(BaseModel): depends_on: List[TaskDependency] = [] description: str = None execution_profile: TaskExecutionProfile = TaskExecutionProfile() - input: Dict[str, TaskInputValue] = {} - output: Dict[str, BaseOutputValue] = {} + input: Dict[str, TaskInputSpec] = {} + output: Dict[str, TaskOutputSpec] = {} class Config: arbitrary_types_allowed = True @@ -538,7 +569,7 @@ def backwards_compatibility_transforms(cls, values): return values class TemplateTask(BaseTask): - uses: Uses + uses: Union[str, Uses] class ApplicationTask(BaseTask): type: Literal["application", "container_run"] @@ -637,7 +668,7 @@ class FunctionTask(BaseTask): Field(discriminator="type") ] -class BasePipeline(BaseModel): +class Pipeline(BaseModel): id: ID type: EnumPipelineType = EnumPipelineType.Workflow tasks: List[ @@ -658,8 +689,8 @@ class BasePipeline(BaseModel): max_exec_time=DEFAULT_MAX_EXEC_TIME*3) cron: str = None archive_ids: List[str] = [] - env: KeyVal = {} - params: Dict[str, ValueWithRequirements] = {} + env: Env = {} + params: Params = {} # NOTE This pre validation transformer is for backwards-compatibility # Previous pipelines did not have environments or parmas @@ -681,10 +712,10 @@ def backwards_compatibility_transforms(cls, values): class Config: extra = Extra.allow -class WorkflowPipeline(BasePipeline): +class WorkflowPipeline(Pipeline): pass -class CIPipeline(BasePipeline): +class CIPipeline(Pipeline): cache: bool = False builder: str context: Annotated[ @@ -701,12 +732,6 @@ class CIPipeline(BasePipeline): ], Field(discriminator="type") ] = None - auth: dict = None - data: dict = None - headers: dict = None - http_method: str = None - query_params: dict = None - url: str = None # Pipeline runs and task executions # TODO rename ReqCreateTaskExecution @@ -748,10 +773,10 @@ class Group(BaseModel): class WorkflowSubmissionRequest(BaseModel): archives: List[Archive] = [] - env: KeyVal = {} - params: KeyVal = {} + env: Env = {} + params: Params = {} group: Group - pipeline: BasePipeline + pipeline: Pipeline pipeline_run: PipelineRun meta: WorkflowSubmissionRequestMeta diff --git a/src/engine/src/owe_python_sdk/client.py b/src/engine/src/owe_python_sdk/client.py index 8cb0904f..40f55186 100644 --- a/src/engine/src/owe_python_sdk/client.py +++ b/src/engine/src/owe_python_sdk/client.py @@ -61,7 +61,7 @@ def __init__( runner_class: Runner, runner_config_override={} ): - self.schema = owe_schema.BasePipeline(**schema) + self.schema = owe_schema.Pipeline(**schema) self._set_runner_class(runner_class, runner_config_override=runner_config_override) def add_task(self, task: Task, *tasks: List[Task]): diff --git a/src/engine/src/owe_python_sdk/schema.py b/src/engine/src/owe_python_sdk/schema.py index da0fbb4d..3a38c99d 100644 --- a/src/engine/src/owe_python_sdk/schema.py +++ b/src/engine/src/owe_python_sdk/schema.py @@ -115,9 +115,10 @@ class EnumTaskFlavor(str, Enum, metaclass=_EnumMeta): G1_NVD_MED = "g1nvdmed" G1_NVD_LRG = "g1nvdlrg" -LiteralTaskTypes = Literal["function", "application", "request", "image_build", "tapis_job", "tapis_actor"] +LiteralTaskTypes = Literal["template", "function", "application", "request", "image_build", "tapis_job", "tapis_actor"] TaskTypes = list(get_args(LiteralTaskTypes)) class EnumTaskType(str, Enum, metaclass=_EnumMeta): + Template = "template" ImageBuild = "image_build" Request = "request" Function = "function" @@ -179,17 +180,62 @@ def validate(cls, v): raise ValueError("Invalid 'id' format. Must start with alphanumeric chars followed by 0 or more alphanumeric chars, '_', and '-'") return v -class BaseValue(BaseModel): - type: EnumTaskIOTypes = EnumTaskIOTypes.String - value: Union[str, int, float, bool, bytes] = None - value_from: Dict[str, str] = None +class TaskOutputRef(BaseModel): + task_id: str + output_id: str + +LiteralHostRefTypes = Literal["kubernetes_secret", "kubernetes_config_map"] +class ValueFromHostRef(BaseModel): + type: LiteralHostRefTypes + name: str + field_selector: str = None + +class ValueFromSecretRef(BaseModel): + engine: str + pk: str + field_selector: str = None + +ValueFromEnv = Dict[Literal["env"], str] +ValueFromParams = Dict[Literal["params"], str] +ValueFromTaskOutput = Dict[Literal["task_output"], TaskOutputRef] +ValueFromHost = Dict[Literal["host"], ValueFromHostRef] +ValueFromSecret = Dict[Literal["secret"], ValueFromSecretRef] +ValueFromAll = Union[ + ValueFromEnv, + ValueFromParams, + ValueFromTaskOutput, + ValueFromHost, + ValueFromSecret +] + +EnvSpecValueFrom = Union[ + ValueFromSecret, + ValueFromHost +] + +# NOTE Same as EnvSpecValueFrom for now, but may diverge. Delete this NOTE once it does +# Also, no good name for the combination of the 2. EnvParamSpecValueFrom?? No thanks. +ParamSpecValueFrom = Union[ + ValueFromSecret, + ValueFromHost +] + +Value = Union[str, int, float, bool, bytes] + +class Spec(BaseModel): + required: bool = False + type: EnumTaskIOTypes + value: Value = None + value_from: ValueFromAll = None @root_validator(pre=True) def value_or_value_from(cls, values): # Either 'value' or 'value_from' must be set on every variable if ( - values.get("value", None) == None - and values.get("value_from", None) == None + ( + values.get("value", None) == None + and values.get("value_from", None) == None + ) and values.get("required") ): raise ValueError( "Missing 'value' or 'value_from' property in variable values") @@ -225,44 +271,25 @@ def value_from_type_validation(cls, value): "Task Input Value Error: 'value_from' property must be a single key-value pair where the key is oneOf ['env', 'params', 'task_input', 'host', 'secret'] and the value is a non-empty string if key is oneOf ['env', 'params'] or an object if key is oneOf ['task_input', 'host', 'secret']" ) return value - -class TaskOutputRef(BaseModel): - task_id: str - output_id: str +class TaskInputSpec(Spec): + value: Value = None + value_from: ValueFromAll = None -LiteralHostRefTypes = Literal["kubernetes_secret", "kubernetes_config_map"] -class ValueFromHostRef(BaseModel): - type: LiteralHostRefTypes - name: str - field_selector: str = None +class EnvSpec(Spec): + value: Value = None + value_from: EnvSpecValueFrom = None -class ValueFromSecretRef(BaseModel): - engine: str - pk: str - field_selector: str = None +Env = Dict[str, EnvSpec] -ValueFromEnv = Dict[Literal["env"], str] -ValueFromParams = Dict[Literal["params"], str] -ValueFromTaskOutput = Dict[Literal["task_output"], TaskOutputRef] -ValueFromHost = Dict[Literal["host"], ValueFromHostRef] -ValueFromSecret = Dict[Literal["secret"], ValueFromSecretRef] - -class TaskInputValue(BaseValue): +class ParamSpec(Spec): type: EnumTaskIOTypes - value: Union[str, int, float, bool, bytes] = None - value_from: Union[ - ValueFromEnv, - ValueFromParams, - ValueFromTaskOutput, - ValueFromHost, - ValueFromSecret - ] = None + value: Value = None + value_from: ParamSpecValueFrom = None -class ValueWithRequirements(BaseValue): - required: bool = False +Params = Dict[str, ParamSpec] -KeyVal = Dict[str, BaseValue] +KeyVal = Dict[str, Spec] ################## /Common ################### class S3Auth(BaseModel): @@ -361,7 +388,7 @@ class IdentityCreateRequest(BaseModel): DockerhubAuth ] -class BaseContext(BaseModel): +class Context(BaseModel): type: LiteralContextTypes branch: str = None build_file_path: str = None @@ -378,14 +405,14 @@ class BaseContext(BaseModel): class Config: extra: Extra.allow -class GithubContext(BaseContext): +class GithubContext(Context): type: Literal["github"] credentials: GithubAuth = None branch: str build_file_path: str = None sub_path: str = None -class DockerhubContext(BaseContext): +class DockerhubContext(Context): type: Literal["dockerhub"] credentials: DockerhubAuth = None @@ -426,7 +453,7 @@ class BaseEvent(BaseModel): username: str = None class APIEvent(BaseEvent): - params: KeyVal = {} + params: Params = {} directives: List[str] = None class WebhookEvent(BaseEvent): @@ -464,9 +491,8 @@ class TaskDependency(BaseModel): # Output ----------------------------------------------------------------- -class BaseOutputValue(BaseModel): +class TaskOutputSpec(BaseModel): type: EnumTaskIOTypes - value: Union[str, int, float, bool, bytes] # ------------------------------------------------------------------------ @@ -492,18 +518,30 @@ class TaskExecutionProfile(BaseExecutionProfile): class GitRepository(BaseModel): url: str - branch: str = None # If no branch specified, the default branch will be used + branch: str = None + auth: GithubAuth = None + +class ClonedGitRepository(GitRepository): directory: str +class Uses(BaseModel): + source: GitRepository + name: str = None + path: str = None + + @root_validator + def must_provide_name_or_path(cls, values): + if values.get("name", None) == None and values.get("path", None) == None: + raise ValueError("The 'uses' property of a Template Task must specify either a 'name' or 'path' property") + class BaseTask(BaseModel): id: ID type: LiteralTaskTypes depends_on: List[TaskDependency] = [] description: str = None execution_profile: TaskExecutionProfile = TaskExecutionProfile() - input: Dict[str, TaskInputValue] = {} - _if: str = None - output: Dict[str, BaseOutputValue] = {} + input: Dict[str, TaskInputSpec] = {} + output: Dict[str, TaskOutputSpec] = {} class Config: arbitrary_types_allowed = True @@ -529,6 +567,9 @@ def backwards_compatibility_transforms(cls, values): } return values + +class TemplateTask(BaseTask): + uses: Union[str, Uses] class ApplicationTask(BaseTask): type: Literal["application", "container_run"] @@ -605,7 +646,7 @@ class RequestTask(BaseTask): class FunctionTask(BaseTask): type: Literal["function"] - git_repositories: List[GitRepository] = [] + git_repositories: List[ClonedGitRepository] = [] runtime: EnumRuntimeEnvironment packages: List[str] = [] installer: EnumInstaller @@ -616,6 +657,7 @@ class FunctionTask(BaseTask): Task = Annotated[ Union[ + TemplateTask, ApplicationTask, ImageBuildTask, FunctionTask, @@ -626,12 +668,13 @@ class FunctionTask(BaseTask): Field(discriminator="type") ] -class BasePipeline(BaseModel): +class Pipeline(BaseModel): id: ID type: EnumPipelineType = EnumPipelineType.Workflow tasks: List[ Annotated[ Union[ + TemplateTask, ApplicationTask, ImageBuildTask, FunctionTask, @@ -646,8 +689,8 @@ class BasePipeline(BaseModel): max_exec_time=DEFAULT_MAX_EXEC_TIME*3) cron: str = None archive_ids: List[str] = [] - env: KeyVal = {} - params: Dict[str, ValueWithRequirements] = {} + env: Env = {} + params: Params = {} # NOTE This pre validation transformer is for backwards-compatibility # Previous pipelines did not have environments or parmas @@ -669,10 +712,10 @@ def backwards_compatibility_transforms(cls, values): class Config: extra = Extra.allow -class WorkflowPipeline(BasePipeline): +class WorkflowPipeline(Pipeline): pass -class CIPipeline(BasePipeline): +class CIPipeline(Pipeline): cache: bool = False builder: str context: Annotated[ @@ -689,12 +732,6 @@ class CIPipeline(BasePipeline): ], Field(discriminator="type") ] = None - auth: dict = None - data: dict = None - headers: dict = None - http_method: str = None - query_params: dict = None - url: str = None # Pipeline runs and task executions # TODO rename ReqCreateTaskExecution @@ -736,10 +773,10 @@ class Group(BaseModel): class WorkflowSubmissionRequest(BaseModel): archives: List[Archive] = [] - env: KeyVal = {} - params: KeyVal = {} + env: Env = {} + params: Params = {} group: Group - pipeline: BasePipeline + pipeline: Pipeline pipeline_run: PipelineRun meta: WorkflowSubmissionRequestMeta @@ -779,6 +816,4 @@ def __init__( self.is_valid = is_valid self.body = body self.message = message - self.failure_view = failure_view - - + self.failure_view = failure_view \ No newline at end of file