Skip to content

Commit

Permalink
add MCDecoratorSyncResponse model, fix typing errors for cronjobs?
Browse files Browse the repository at this point in the history
  • Loading branch information
ikreymer committed Jul 26, 2024
1 parent f843038 commit 2fe6159
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 18 deletions.
36 changes: 19 additions & 17 deletions backend/btrixcloud/operator/cronjobs.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
""" Operator handler for crawl CronJobs """

from uuid import UUID
from typing import Optional, Any
from typing import Optional
import yaml

from btrixcloud.utils import to_k8s_date, dt_now
from .models import MCDecoratorSyncData, CJS
from .models import MCDecoratorSyncData, CJS, MCDecoratorSyncResponse
from .baseoperator import BaseOperator

from ..models import CrawlConfig
Expand All @@ -20,12 +20,14 @@ def init_routes(self, app):
"""init routes for crawl CronJob decorator"""

@app.post("/op/cronjob/sync")
async def mc_sync_cronjob_crawls(data: MCDecoratorSyncData):
async def mc_sync_cronjob_crawls(
data: MCDecoratorSyncData,
) -> MCDecoratorSyncResponse:
return await self.sync_cronjob_crawl(data)

def get_finished_response(
self, metadata: dict[str, str], set_status=True, finished: Optional[str] = None
):
) -> MCDecoratorSyncResponse:
"""get final response to indicate cronjob created job is finished"""

if not finished:
Expand All @@ -40,14 +42,12 @@ def get_finished_response(
"completionTime": finished,
}

res = {
"attachments": [],
return MCDecoratorSyncResponse(
attachments=[],
# set on job to match default behavior when job finishes
"annotations": {"finished": finished},
"status": status,
}
print(res)
return res
annotations={"finished": finished},
status=status,
)

# pylint: disable=too-many-arguments
async def make_new_crawljob(
Expand All @@ -58,7 +58,7 @@ async def make_new_crawljob(
crawl_id: str,
metadata: dict[str, str],
state: Optional[str],
) -> list[dict[str, Any]]:
) -> MCDecoratorSyncResponse:
"""declare new CrawlJob from cid, based on db data"""
# cronjob doesn't exist yet
crawlconfig: CrawlConfig
Expand Down Expand Up @@ -127,9 +127,11 @@ async def make_new_crawljob(
profile_filename=profile_filename or "",
)

return list(yaml.safe_load_all(crawljob))
return MCDecoratorSyncResponse(attachments=list(yaml.safe_load_all(crawljob)))

async def sync_cronjob_crawl(self, data: MCDecoratorSyncData):
async def sync_cronjob_crawl(
self, data: MCDecoratorSyncData
) -> MCDecoratorSyncResponse:
"""create crawljobs from a job object spawned by cronjob"""

metadata = data.object["metadata"]
Expand Down Expand Up @@ -163,7 +165,7 @@ async def sync_cronjob_crawl(self, data: MCDecoratorSyncData):
crawljob_id = f"crawljob-{crawl_id}"

if crawljob_id not in crawljobs:
attachments = await self.make_new_crawljob(
response = await self.make_new_crawljob(
UUID(cid),
UUID(oid) if oid else None,
UUID(userid) if userid else None,
Expand All @@ -182,6 +184,6 @@ async def sync_cronjob_crawl(self, data: MCDecoratorSyncData):
except KeyError:
pass

attachments = [crawljob]
response = MCDecoratorSyncResponse(attachments=[crawljob])

return {"attachments": attachments}
return response
11 changes: 10 additions & 1 deletion backend/btrixcloud/operator/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from collections import defaultdict
from uuid import UUID
from typing import Optional, DefaultDict, Literal, Annotated
from typing import Optional, DefaultDict, Literal, Annotated, Any
from pydantic import BaseModel, Field
from kubernetes.utils import parse_quantity
from btrixcloud.models import StorageRef, TYPE_ALL_CRAWL_STATES
Expand Down Expand Up @@ -53,6 +53,15 @@ class MCDecoratorSyncData(BaseModel):
finalizing: bool = False


# ============================================================================
class MCDecoratorSyncResponse(BaseModel):
"""Response model for decoratorcontroller sync api"""

attachments: list[dict[str, Any]]
status: Optional[dict[str, Any]] = None
annotations: Optional[dict[str, str]] = None


# ============================================================================
class CrawlSpec(BaseModel):
"""spec from k8s CrawlJob object"""
Expand Down

0 comments on commit 2fe6159

Please sign in to comment.