From 7e9d06d0629d7b03afd5aca3034bf917a5f0c945 Mon Sep 17 00:00:00 2001 From: Shahar Glazner Date: Sun, 15 Sep 2024 09:33:49 +0300 Subject: [PATCH 1/9] feat: don't allow update/edit provisioned resources (#1916) --- docs/deployment/provision/overview.mdx | 29 +++ docs/deployment/provision/provider.mdx | 58 +++++ docs/deployment/provision/workflow.mdx | 36 +++ docs/mint.json | 8 + keep-ui/app/providers/provider-form.tsx | 51 +++- keep-ui/app/providers/provider-tile.tsx | 10 + keep-ui/app/providers/providers.tsx | 1 + keep-ui/app/workflows/models.tsx | 1 + keep-ui/app/workflows/workflow-menu.tsx | 30 ++- keep-ui/app/workflows/workflow-tile.tsx | 14 +- keep/api/api.py | 12 +- keep/api/core/db.py | 127 +++++++--- .../versions/2024-09-13-10-48_938b1aa62d5c.py | 52 ++++ keep/api/models/db/provider.py | 1 + keep/api/models/db/workflow.py | 2 + keep/api/models/provider.py | 1 + keep/api/models/workflow.py | 2 + keep/api/routes/workflows.py | 84 ++++--- keep/providers/providers_factory.py | 1 + keep/providers/providers_service.py | 54 ++++- keep/workflowmanager/workflowstore.py | 187 ++++++++++++--- tests/fixtures/client.py | 11 + .../workflows_1/provision_example_1.yml | 20 ++ .../workflows_1/provision_example_2.yml | 29 +++ .../workflows_1/provision_example_3.yml | 14 ++ .../workflows_2/provision_example_1.yml | 20 ++ .../workflows_2/provision_example_2.yml | 29 +++ tests/test_auth.py | 16 +- tests/test_enrichments.py | 4 +- tests/test_extraction_rules.py | 23 +- tests/test_metrics.py | 34 ++- tests/test_provisioning.py | 223 ++++++++++++++++++ tests/test_rules_api.py | 92 ++++---- tests/test_search_alerts.py | 10 +- tests/test_search_alerts_configuration.py | 6 +- 35 files changed, 1060 insertions(+), 232 deletions(-) create mode 100644 docs/deployment/provision/overview.mdx create mode 100644 docs/deployment/provision/provider.mdx create mode 100644 docs/deployment/provision/workflow.mdx create mode 100644 keep/api/models/db/migrations/versions/2024-09-13-10-48_938b1aa62d5c.py create mode 100644 tests/provision/workflows_1/provision_example_1.yml create mode 100644 tests/provision/workflows_1/provision_example_2.yml create mode 100644 tests/provision/workflows_1/provision_example_3.yml create mode 100644 tests/provision/workflows_2/provision_example_1.yml create mode 100644 tests/provision/workflows_2/provision_example_2.yml create mode 100644 tests/test_provisioning.py diff --git a/docs/deployment/provision/overview.mdx b/docs/deployment/provision/overview.mdx new file mode 100644 index 000000000..f48d4d81e --- /dev/null +++ b/docs/deployment/provision/overview.mdx @@ -0,0 +1,29 @@ +--- +title: "Overview" +--- + +Keep supports various deployment and provisioning strategies to accommodate different environments and use cases, from development setups to production deployments. + +### Provisioning Options + +Keep offers two main provisioning options: + +1. [**Provider Provisioning**](/deployment/provision/provider) - Set up and manage data providers for Keep. +2. [**Workflow Provisioning**](/deployment/provision/workflow) - Configure and manage workflows within Keep. + +Choosing the right provisioning strategy depends on your specific use case, deployment environment, and scalability requirements. You can read more about each provisioning option in their respective sections. + +### How To Configure Provisioning + + +Some provisioning options require additional environment variables. These will be covered in detail on the specific provisioning pages. + + +Provisioning in Keep is controlled through environment variables and configuration files. The main environment variables for provisioning are: + +| Provisioning Type | Environment Variable | Purpose | +|-------------------|----------------------|---------| +| **Provider** | `KEEP_PROVIDERS` | JSON string containing provider configurations | +| **Workflow** | `KEEP_WORKFLOWS_DIRECTORY` | Directory path containing workflow configuration files | + +For more details on each provisioning strategy, including setup instructions and implications, refer to the respective sections. diff --git a/docs/deployment/provision/provider.mdx b/docs/deployment/provision/provider.mdx new file mode 100644 index 000000000..f6993aabf --- /dev/null +++ b/docs/deployment/provision/provider.mdx @@ -0,0 +1,58 @@ +--- +title: "Providers Provisioning" +--- + +For any questions or issues related to provider provisioning, please join our [Slack](https://slack.keephq.dev) community. + +Provider provisioning in Keep allows you to set up and manage data providers dynamically. This feature enables you to configure various data sources that Keep can interact with, such as monitoring systems, databases, or other services. + +### Configuring Providers + +To provision providers, set the `KEEP_PROVIDERS` environment variable with a JSON string containing the provider configurations. Here's an example: + +```json +{ + "keepVictoriaMetrics": { + "type": "victoriametrics", + "authentication": { + "VMAlertHost": "http://localhost", + "VMAlertPort": 1234 + } + }, + "keepClickhouse1": { + "type": "clickhouse", + "authentication": { + "host": "http://localhost", + "port": 1234, + "username": "keep", + "password": "keep", + "database": "keep-db" + } + } +} +``` + +Spin up Keep with this `KEEP_PROVIDERS` value: +```json +# ENV +KEEP_PROVIDERS={"keepVictoriaMetrics":{"type":"victoriametrics","authentication":{"VMAlertHost":"http://localhost","VMAlertPort": 1234}},"keepClickhouse1":{"type":"clickhouse","authentication":{"host":"http://localhost","port":"4321","username":"keep","password":"1234","database":"keepdb"}}} +``` + +### Supported Providers + +Keep supports a wide range of provider types. Each provider type has its own specific configuration requirements. +To see the full list of supported providers and their detailed configuration options, please refer to our comprehensive provider documentation. + + +### Update Provisioned Providers + +Provider configurations can be updated dynamically by changing the `KEEP_PROVIDERS` environment variable. + +On every restart, Keep reads this environment variable and determines which providers need to be added or removed. + +This process allows for flexible management of data sources without requiring manual intervention. By simply updating the `KEEP_PROVIDERS` variable and restarting the application, you can efficiently add new providers, remove existing ones, or modify their configurations. + +The high-level provisioning mechanism: +1. Keep reads the `KEEP_PROVIDERS` value. +2. Keep checks if there are any provisioned providers that are no longer in the `KEEP_PROVIDERS` value, and deletes them. +3. Keep installs all providers from the `KEEP_PROVIDERS` value. diff --git a/docs/deployment/provision/workflow.mdx b/docs/deployment/provision/workflow.mdx new file mode 100644 index 000000000..134704dc4 --- /dev/null +++ b/docs/deployment/provision/workflow.mdx @@ -0,0 +1,36 @@ +--- +title: "Workflow Provisioning" +--- + +For any questions or issues related to workflow provisioning, please join our [Slack](https://slack.keephq.dev) community. + +Workflow provisioning in Keep allows you to set up and manage workflows dynamically. This feature enables you to configure various automated processes and tasks within your Keep deployment. + +### Configuring Workflows + +To provision workflows, follow these steps: + +1. Set the `KEEP_WORKFLOWS_DIRECTORY` environment variable to the path of your workflow configuration directory. +2. Create workflow configuration files in the specified directory. + +Example directory structure: +``` +/path/to/workflows/ +├── workflow1.yaml +├── workflow2.yaml +└── workflow3.yaml +``` +### Update Provisioned Workflows + +On every restart, Keep reads the `KEEP_WORKFLOWS_DIRECTORY` environment variable and determines which workflows need to be added, removed, or updated. + +This process allows for flexible management of workflows without requiring manual intervention. By simply updating the workflow files in the `KEEP_WORKFLOWS_DIRECTORY` and restarting the application, you can efficiently add new workflows, remove existing ones, or modify their configurations. + +The high-level provisioning mechanism: +1. Keep reads the `KEEP_WORKFLOWS_DIRECTORY` value. +2. Keep lists all workflow files under the `KEEP_WORKFLOWS_DIRECTORY` directory. +3. Keep compares the current workflow files with the previously provisioned workflows: + - New workflow files are provisioned. + - Missing workflow files are deprovisioned. + - Updated workflow files are re-provisioned with the new configuration. +4. Keep updates its internal state to reflect the current set of provisioned workflows. diff --git a/docs/mint.json b/docs/mint.json index 99e9a8b8a..1f874c84e 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -65,6 +65,14 @@ "deployment/authentication/keycloak-auth" ] }, + { + "group": "Provision", + "pages": [ + "deployment/provision/overview", + "deployment/provision/provider", + "deployment/provision/workflow" + ] + }, "deployment/secret-manager", "deployment/docker", "deployment/kubernetes", diff --git a/keep-ui/app/providers/provider-form.tsx b/keep-ui/app/providers/provider-form.tsx index 3e2da8ea2..f3f8e1ce6 100644 --- a/keep-ui/app/providers/provider-form.tsx +++ b/keep-ui/app/providers/provider-form.tsx @@ -26,7 +26,7 @@ import { Accordion, AccordionHeader, AccordionBody, - + Badge, } from "@tremor/react"; import { ExclamationCircleIcon, @@ -520,6 +520,7 @@ const ProviderForm = ({ onChange={(value) => handleInputChange({ target: { name: configKey, value } })} placeholder={method.placeholder || `Select ${configKey}`} error={Object.keys(inputErrors).includes(configKey)} + disabled={provider.provisioned} > {method.options.map((option) => ( @@ -541,6 +542,7 @@ const ProviderForm = ({ color="orange" size="xs" onClick={addEntry(configKey)} + disabled={provider.provisioned} > Add Entry @@ -550,6 +552,7 @@ const ProviderForm = ({ value={formValues[configKey] || []} onChange={(value) => handleDictInputChange(configKey, value)} error={Object.keys(inputErrors).includes(configKey)} + disabled={provider.provisioned} /> ); @@ -565,6 +568,7 @@ const ProviderForm = ({ inputFileRef.current.click(); }} icon={ArrowDownOnSquareIcon} + disabled={provider.provisioned} > {selectedFile ? `File Chosen: ${selectedFile}` : `Upload a ${method.name}`} @@ -581,6 +585,7 @@ const ProviderForm = ({ } handleInputChange(e); }} + disabled={provider.provisioned} /> ); @@ -597,6 +602,7 @@ const ProviderForm = ({ autoComplete="off" error={Object.keys(inputErrors).includes(configKey)} placeholder={method.placeholder || `Enter ${configKey}`} + disabled={provider.provisioned} /> ); @@ -694,6 +700,13 @@ const ProviderForm = ({
Connect to {provider.display_name} + {/* Display the Provisioned Badge if the provider is provisioned */} + {provider.provisioned && ( + + Provisioned + + )} +
+ {provider.provisioned && +
+ + + Editing provisioned providers is not possible from UI. + + +
+ } + {provider.provider_description && ( {provider.provider_description} )} @@ -885,7 +912,7 @@ const ProviderForm = ({ variant="secondary" color="orange" className="mt-2.5" - disabled={!installOrUpdateWebhookEnabled} + disabled={!installOrUpdateWebhookEnabled || provider.provisioned} tooltip={ !installOrUpdateWebhookEnabled ? "Fix required webhook scopes and refresh scopes to enable" @@ -928,16 +955,20 @@ const ProviderForm = ({ {installedProvidersMode && Object.keys(provider.config).length > 0 && ( <> - - +
+ +
)} {!installedProvidersMode && Object.keys(provider.config).length > 0 && ( diff --git a/keep-ui/app/providers/provider-tile.tsx b/keep-ui/app/providers/provider-tile.tsx index 51d0a4865..cd59f0790 100644 --- a/keep-ui/app/providers/provider-tile.tsx +++ b/keep-ui/app/providers/provider-tile.tsx @@ -19,6 +19,7 @@ import { import "./provider-tile.css"; import moment from "moment"; import ImageWithFallback from "@/components/ImageWithFallback"; +import { FaCode } from "react-icons/fa"; interface Props { provider: Provider; @@ -200,6 +201,15 @@ export default function ProviderTile({ provider, onClick }: Props) { Linked ) : null} + {provider.provisioned ? ( + + ) : null}
diff --git a/keep-ui/app/providers/providers.tsx b/keep-ui/app/providers/providers.tsx index a75229645..ed060ead2 100644 --- a/keep-ui/app/providers/providers.tsx +++ b/keep-ui/app/providers/providers.tsx @@ -90,6 +90,7 @@ export interface Provider { tags: TProviderLabels[]; alertsDistribution?: AlertDistritbuionData[]; alertExample?: { [key: string]: string }; + provisioned?: boolean; } export type Providers = Provider[]; diff --git a/keep-ui/app/workflows/models.tsx b/keep-ui/app/workflows/models.tsx index 7b548a518..5e77ddcfa 100644 --- a/keep-ui/app/workflows/models.tsx +++ b/keep-ui/app/workflows/models.tsx @@ -44,6 +44,7 @@ export type Workflow = { WorkflowExecution, "execution_time" | "status" | "started" >[]; + provisioned?: boolean; }; export type MockProvider = { diff --git a/keep-ui/app/workflows/workflow-menu.tsx b/keep-ui/app/workflows/workflow-menu.tsx index d13e85aa7..2362fa094 100644 --- a/keep-ui/app/workflows/workflow-menu.tsx +++ b/keep-ui/app/workflows/workflow-menu.tsx @@ -12,7 +12,8 @@ interface WorkflowMenuProps { onDownload?: () => void; onBuilder?: () => void; isRunButtonDisabled: boolean; - runButtonToolTip?: string; + runButtonToolTip?: string; + provisioned?: boolean; } @@ -24,6 +25,7 @@ export default function WorkflowMenu({ onBuilder, isRunButtonDisabled, runButtonToolTip, + provisioned, }: WorkflowMenuProps) { const stopPropagation = (e: React.MouseEvent<HTMLButtonElement>) => { e.stopPropagation(); @@ -115,15 +117,23 @@ export default function WorkflowMenu({ </Menu.Item> <Menu.Item> {({ active }) => ( - <button - onClick={(e) => { stopPropagation(e); onDelete?.(); }} - className={`${ - active ? "bg-slate-200" : "text-gray-900" - } group flex w-full items-center rounded-md px-2 py-2 text-xs`} - > - <TrashIcon className="mr-2 h-4 w-4" aria-hidden="true" /> - Delete - </button> + <div className="relative group"> + <button + disabled={provisioned} + onClick={(e) => { stopPropagation(e); onDelete?.(); }} + className={`${ + active ? 'bg-slate-200' : 'text-gray-900' + } flex w-full items-center rounded-md px-2 py-2 text-xs ${provisioned ? 'cursor-not-allowed opacity-50' : ''}`} + > + <TrashIcon className="mr-2 h-4 w-4" aria-hidden="true" /> + Delete + </button> + {provisioned && ( + <div className="absolute bottom-full transform -translate-x-1/2 bg-black text-white text-xs rounded px-4 py-1 z-10 opacity-0 group-hover:opacity-100"> + Cannot delete a provisioned workflow + </div> + )} + </div> )} </Menu.Item> </div> diff --git a/keep-ui/app/workflows/workflow-tile.tsx b/keep-ui/app/workflows/workflow-tile.tsx index c462454d1..73da76321 100644 --- a/keep-ui/app/workflows/workflow-tile.tsx +++ b/keep-ui/app/workflows/workflow-tile.tsx @@ -50,6 +50,7 @@ function WorkflowMenuSection({ onBuilder, isRunButtonDisabled, runButtonToolTip, + provisioned, }: { onDelete: () => Promise<void>; onRun: () => Promise<void>; @@ -58,6 +59,7 @@ function WorkflowMenuSection({ onBuilder: () => void; isRunButtonDisabled: boolean; runButtonToolTip?: string; + provisioned?: boolean; }) { // Determine if all providers are installed @@ -70,6 +72,7 @@ function WorkflowMenuSection({ onBuilder={onBuilder} isRunButtonDisabled={isRunButtonDisabled} runButtonToolTip={runButtonToolTip} + provisioned={provisioned} /> ); } @@ -547,7 +550,7 @@ function WorkflowTile({ workflow }: { workflow: Workflow }) { <Loading /> </div> )} - <Card + <Card className="relative flex flex-col justify-between bg-white rounded shadow p-2 h-full hover:border-orange-400 hover:border-2" onClick={(e)=>{ e.stopPropagation(); @@ -557,7 +560,12 @@ function WorkflowTile({ workflow }: { workflow: Workflow }) { } }} > - <div className="absolute top-0 right-0 mt-2 mr-2 mb-2"> + <div className="absolute top-0 right-0 mt-2 mr-2 mb-2 flex items-center"> + {workflow.provisioned && ( + <Badge color="orange" size="xs" className="mr-2"> + Provisioned + </Badge> + )} {!!handleRunClick && WorkflowMenuSection({ onDelete: handleDeleteClick, onRun: handleRunClick, @@ -566,6 +574,7 @@ function WorkflowTile({ workflow }: { workflow: Workflow }) { onBuilder: handleBuilderClick, runButtonToolTip: message, isRunButtonDisabled: !!isRunButtonDisabled, + provisioned: workflow.provisioned, })} </div> <div className="m-2 flex flex-col justify-around item-start flex-wrap"> @@ -862,6 +871,7 @@ export function WorkflowTileOld({ workflow }: { workflow: Workflow }) { onBuilder: handleBuilderClick, runButtonToolTip: message, isRunButtonDisabled: !!isRunButtonDisabled, + provisioned: workflow.provisioned, })} </div> diff --git a/keep/api/api.py b/keep/api/api.py index f04d01365..bed995b86 100644 --- a/keep/api/api.py +++ b/keep/api/api.py @@ -59,7 +59,12 @@ from keep.event_subscriber.event_subscriber import EventSubscriber from keep.identitymanager.identitymanagerfactory import IdentityManagerFactory from keep.posthog.posthog import get_posthog_client + +# load all providers into cache +from keep.providers.providers_factory import ProvidersFactory +from keep.providers.providers_service import ProvidersService from keep.workflowmanager.workflowmanager import WorkflowManager +from keep.workflowmanager.workflowstore import WorkflowStore load_dotenv(find_dotenv()) keep.api.logging.setup_logging() @@ -242,15 +247,14 @@ def get_app( @app.on_event("startup") async def on_startup(): - # load all providers into cache - from keep.providers.providers_factory import ProvidersFactory - from keep.providers.providers_service import ProvidersService - logger.info("Loading providers into cache") ProvidersFactory.get_all_providers() # provision providers from env. relevant only on single tenant. + logger.info("Provisioning providers and workflows") ProvidersService.provision_providers_from_env(SINGLE_TENANT_UUID) logger.info("Providers loaded successfully") + WorkflowStore.provision_workflows_from_directory(SINGLE_TENANT_UUID) + logger.info("Workflows provisioned successfully") # Start the services logger.info("Starting the services") # Start the scheduler diff --git a/keep/api/core/db.py b/keep/api/core/db.py index 9d7d5c40b..0e0111f54 100644 --- a/keep/api/core/db.py +++ b/keep/api/core/db.py @@ -278,6 +278,8 @@ def add_or_update_workflow( interval, workflow_raw, is_disabled, + provisioned=False, + provisioned_file=None, updated_by=None, ) -> Workflow: with Session(engine, expire_on_commit=False) as session: @@ -301,7 +303,9 @@ def add_or_update_workflow( existing_workflow.revision += 1 # Increment the revision existing_workflow.last_updated = datetime.now() # Update last_updated existing_workflow.is_deleted = False - existing_workflow.is_disabled= is_disabled + existing_workflow.is_disabled = is_disabled + existing_workflow.provisioned = provisioned + existing_workflow.provisioned_file = provisioned_file else: # Create a new workflow @@ -313,8 +317,10 @@ def add_or_update_workflow( created_by=created_by, updated_by=updated_by, # Set updated_by to the provided value interval=interval, - is_disabled =is_disabled, + is_disabled=is_disabled, workflow_raw=workflow_raw, + provisioned=provisioned, + provisioned_file=provisioned_file, ) session.add(workflow) @@ -461,6 +467,27 @@ def get_all_workflows(tenant_id: str) -> List[Workflow]: return workflows +def get_all_provisioned_workflows(tenant_id: str) -> List[Workflow]: + with Session(engine) as session: + workflows = session.exec( + select(Workflow) + .where(Workflow.tenant_id == tenant_id) + .where(Workflow.provisioned == True) + .where(Workflow.is_deleted == False) + ).all() + return workflows + + +def get_all_provisioned_providers(tenant_id: str) -> List[Provider]: + with Session(engine) as session: + providers = session.exec( + select(Provider) + .where(Provider.tenant_id == tenant_id) + .where(Provider.provisioned == True) + ).all() + return providers + + def get_all_workflows_yamls(tenant_id: str) -> List[str]: with Session(engine) as session: workflows = session.exec( @@ -499,6 +526,7 @@ def get_raw_workflow(tenant_id: str, workflow_id: str) -> str: return None return workflow.workflow_raw + def update_provider_last_pull_time(tenant_id: str, provider_id: str): extra = {"tenant_id": tenant_id, "provider_id": provider_id} logger.info("Updating provider last pull time", extra=extra) @@ -568,18 +596,22 @@ def finish_workflow_execution(tenant_id, workflow_id, execution_id, status, erro session.commit() -def get_workflow_executions(tenant_id, workflow_id, limit=50, offset=0, tab=2, status: Optional[Union[str, List[str]]] = None, +def get_workflow_executions( + tenant_id, + workflow_id, + limit=50, + offset=0, + tab=2, + status: Optional[Union[str, List[str]]] = None, trigger: Optional[Union[str, List[str]]] = None, - execution_id: Optional[str] = None): + execution_id: Optional[str] = None, +): with Session(engine) as session: - query = ( - session.query( - WorkflowExecution, - ) - .filter( - WorkflowExecution.tenant_id == tenant_id, - WorkflowExecution.workflow_id == workflow_id - ) + query = session.query( + WorkflowExecution, + ).filter( + WorkflowExecution.tenant_id == tenant_id, + WorkflowExecution.workflow_id == workflow_id, ) now = datetime.now(tz=timezone.utc) @@ -593,48 +625,51 @@ def get_workflow_executions(tenant_id, workflow_id, limit=50, offset=0, tab=2, s start_of_day = now.replace(hour=0, minute=0, second=0, microsecond=0) query = query.filter( WorkflowExecution.started >= start_of_day, - WorkflowExecution.started <= now + WorkflowExecution.started <= now, ) if timeframe: - query = query.filter( - WorkflowExecution.started >= timeframe - ) + query = query.filter(WorkflowExecution.started >= timeframe) if isinstance(status, str): status = [status] elif status is None: - status = [] - + status = [] + # Normalize trigger to a list if isinstance(trigger, str): trigger = [trigger] - if execution_id: query = query.filter(WorkflowExecution.id == execution_id) if status and len(status) > 0: query = query.filter(WorkflowExecution.status.in_(status)) if trigger and len(trigger) > 0: - conditions = [WorkflowExecution.triggered_by.like(f"{trig}%") for trig in trigger] + conditions = [ + WorkflowExecution.triggered_by.like(f"{trig}%") for trig in trigger + ] query = query.filter(or_(*conditions)) - total_count = query.count() status_count_query = query.with_entities( - WorkflowExecution.status, - func.count().label('count') + WorkflowExecution.status, func.count().label("count") ).group_by(WorkflowExecution.status) status_counts = status_count_query.all() statusGroupbyMap = {status: count for status, count in status_counts} - pass_count = statusGroupbyMap.get('success', 0) - fail_count = statusGroupbyMap.get('error', 0) + statusGroupbyMap.get('timeout', 0) - avgDuration = query.with_entities(func.avg(WorkflowExecution.execution_time)).scalar() + pass_count = statusGroupbyMap.get("success", 0) + fail_count = statusGroupbyMap.get("error", 0) + statusGroupbyMap.get( + "timeout", 0 + ) + avgDuration = query.with_entities( + func.avg(WorkflowExecution.execution_time) + ).scalar() avgDuration = avgDuration if avgDuration else 0.0 - query = query.order_by(desc(WorkflowExecution.started)).limit(limit).offset(offset) - + query = ( + query.order_by(desc(WorkflowExecution.started)).limit(limit).offset(offset) + ) + # Execute the query workflow_executions = query.all() @@ -654,6 +689,19 @@ def delete_workflow(tenant_id, workflow_id): session.commit() +def delete_workflow_by_provisioned_file(tenant_id, provisioned_file): + with Session(engine) as session: + workflow = session.exec( + select(Workflow) + .where(Workflow.tenant_id == tenant_id) + .where(Workflow.provisioned_file == provisioned_file) + ).first() + + if workflow: + workflow.is_deleted = True + session.commit() + + def get_workflow_id(tenant_id, workflow_name): with Session(engine) as session: workflow = session.exec( @@ -1532,10 +1580,7 @@ def get_rule_incidents_count_db(tenant_id): query = ( session.query(Incident.rule_id, func.count(Incident.id)) .select_from(Incident) - .filter( - Incident.tenant_id == tenant_id, - col(Incident.rule_id).isnot(None) - ) + .filter(Incident.tenant_id == tenant_id, col(Incident.rule_id).isnot(None)) .group_by(Incident.rule_id) ) return dict(query.all()) @@ -1611,15 +1656,26 @@ def get_all_filters(tenant_id): def get_last_alert_hash_by_fingerprint(tenant_id, fingerprint): + from sqlalchemy.dialects import mssql + # get the last alert for a given fingerprint # to check deduplication with Session(engine) as session: - alert_hash = session.exec( + query = ( select(Alert.alert_hash) .where(Alert.tenant_id == tenant_id) .where(Alert.fingerprint == fingerprint) .order_by(Alert.timestamp.desc()) - ).first() + .limit(1) # Add LIMIT 1 for MSSQL + ) + + # Compile the query and log it + compiled_query = query.compile( + dialect=mssql.dialect(), compile_kwargs={"literal_binds": True} + ) + logger.info(f"Compiled query: {compiled_query}") + + alert_hash = session.exec(query).first() return alert_hash @@ -2110,8 +2166,7 @@ def get_last_incidents( # .options(joinedload(Incident.alerts)) .filter( Incident.tenant_id == tenant_id, Incident.is_confirmed == is_confirmed - ) - .order_by(desc(Incident.creation_time)) + ).order_by(desc(Incident.creation_time)) ) if timeframe: diff --git a/keep/api/models/db/migrations/versions/2024-09-13-10-48_938b1aa62d5c.py b/keep/api/models/db/migrations/versions/2024-09-13-10-48_938b1aa62d5c.py new file mode 100644 index 000000000..d7cacc71a --- /dev/null +++ b/keep/api/models/db/migrations/versions/2024-09-13-10-48_938b1aa62d5c.py @@ -0,0 +1,52 @@ +"""Provisioned + +Revision ID: 938b1aa62d5c +Revises: 710b4ff1d19e +Create Date: 2024-09-13 10:48:16.112419 + +""" + +import sqlalchemy as sa +import sqlmodel +from alembic import op + +# revision identifiers, used by Alembic. +revision = "938b1aa62d5c" +down_revision = "710b4ff1d19e" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "provider", + sa.Column( + "provisioned", sa.Boolean(), nullable=False, server_default=sa.false() + ), + ) + op.add_column( + "workflow", + sa.Column( + "provisioned", sa.Boolean(), nullable=False, server_default=sa.false() + ), + ) + op.add_column( + "workflow", + sa.Column( + "provisioned_file", sqlmodel.sql.sqltypes.AutoString(), nullable=True + ), + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("workflow", schema=None) as batch_op: + batch_op.drop_column("provisioned") + + with op.batch_alter_table("provider", schema=None) as batch_op: + batch_op.drop_column("provisioned") + + # ### end Alembic commands ### diff --git a/keep/api/models/db/provider.py b/keep/api/models/db/provider.py index 78ef3c68e..37d8d05fa 100644 --- a/keep/api/models/db/provider.py +++ b/keep/api/models/db/provider.py @@ -21,6 +21,7 @@ class Provider(SQLModel, table=True): ) # scope name is key and value is either True if validated or string with error message, e.g: {"read": True, "write": "error message"} consumer: bool = False last_pull_time: Optional[datetime] + provisioned: bool = Field(default=False) class Config: orm_mode = True diff --git a/keep/api/models/db/workflow.py b/keep/api/models/db/workflow.py index f2c575863..3426a9560 100644 --- a/keep/api/models/db/workflow.py +++ b/keep/api/models/db/workflow.py @@ -19,6 +19,8 @@ class Workflow(SQLModel, table=True): is_disabled: bool = Field(default=False) revision: int = Field(default=1, nullable=False) last_updated: datetime = Field(default_factory=datetime.utcnow) + provisioned: bool = Field(default=False) + provisioned_file: Optional[str] = None class Config: orm_mode = True diff --git a/keep/api/models/provider.py b/keep/api/models/provider.py index 78df4eb62..76307de20 100644 --- a/keep/api/models/provider.py +++ b/keep/api/models/provider.py @@ -44,3 +44,4 @@ class Provider(BaseModel): ] = [] alertsDistribution: dict[str, int] | None = None alertExample: dict | None = None + provisioned: bool = False diff --git a/keep/api/models/workflow.py b/keep/api/models/workflow.py index 6beb16c80..68965a2fc 100644 --- a/keep/api/models/workflow.py +++ b/keep/api/models/workflow.py @@ -39,6 +39,8 @@ class WorkflowDTO(BaseModel): invalid: bool = False # whether the workflow is invalid or not (for UI purposes) last_executions: List[dict] = None last_execution_started: datetime = None + provisioned: bool = False + provisioned_file: str = None @property def workflow_raw_id(self): diff --git a/keep/api/routes/workflows.py b/keep/api/routes/workflows.py index 47d44f6ae..621e6d843 100644 --- a/keep/api/routes/workflows.py +++ b/keep/api/routes/workflows.py @@ -107,30 +107,39 @@ def get_workflows( try: providers_dto, triggers = workflowstore.get_workflow_meta_data( - tenant_id=tenant_id, workflow=workflow, installed_providers_by_type=installed_providers_by_type) + tenant_id=tenant_id, + workflow=workflow, + installed_providers_by_type=installed_providers_by_type, + ) except Exception as e: logger.error(f"Error fetching workflow meta data: {e}") providers_dto, triggers = [], [] # Default in case of failure # create the workflow DTO - workflow_dto = WorkflowDTO( - id=workflow.id, - name=workflow.name, - description=workflow.description or "[This workflow has no description]", - created_by=workflow.created_by, - creation_time=workflow.creation_time, - last_execution_time=workflow_last_run_time, - last_execution_status=workflow_last_run_status, - interval=workflow.interval, - providers=providers_dto, - triggers=triggers, - workflow_raw=workflow.workflow_raw, - revision=workflow.revision, - last_updated=workflow.last_updated, - last_executions=last_executions, - last_execution_started=last_execution_started, - disabled=workflow.is_disabled, - ) + try: + workflow_dto = WorkflowDTO( + id=workflow.id, + name=workflow.name, + description=workflow.description + or "[This workflow has no description]", + created_by=workflow.created_by, + creation_time=workflow.creation_time, + last_execution_time=workflow_last_run_time, + last_execution_status=workflow_last_run_status, + interval=workflow.interval, + providers=providers_dto, + triggers=triggers, + workflow_raw=workflow.workflow_raw, + revision=workflow.revision, + last_updated=workflow.last_updated, + last_executions=last_executions, + last_execution_started=last_execution_started, + disabled=workflow.is_disabled, + provisioned=workflow.provisioned, + ) + except Exception as e: + logger.error(f"Error creating workflow DTO: {e}") + continue workflows_dto.append(workflow_dto) return workflows_dto @@ -422,6 +431,10 @@ async def update_workflow_by_id( extra={"tenant_id": tenant_id}, ) raise HTTPException(404, "Workflow not found") + + if workflow_from_db.provisioned: + raise HTTPException(403, detail="Cannot update a provisioned workflow") + workflow = await __get_workflow_raw_data(request, None) parser = Parser() workflow_interval = parser.parse_interval(workflow) @@ -543,24 +556,27 @@ def get_workflow_by_id( workflowstore = WorkflowStore() try: providers_dto, triggers = workflowstore.get_workflow_meta_data( - tenant_id=tenant_id, workflow=workflow, installed_providers_by_type=installed_providers_by_type) + tenant_id=tenant_id, + workflow=workflow, + installed_providers_by_type=installed_providers_by_type, + ) except Exception as e: logger.error(f"Error fetching workflow meta data: {e}") providers_dto, triggers = [], [] # Default in case of failure - + final_workflow = WorkflowDTO( - id=workflow.id, - name=workflow.name, - description=workflow.description or "[This workflow has no description]", - created_by=workflow.created_by, - creation_time=workflow.creation_time, - interval=workflow.interval, - providers=providers_dto, - triggers=triggers, - workflow_raw=workflow.workflow_raw, - last_updated=workflow.last_updated, - disabled=workflow.is_disabled, - ) + id=workflow.id, + name=workflow.name, + description=workflow.description or "[This workflow has no description]", + created_by=workflow.created_by, + creation_time=workflow.creation_time, + interval=workflow.interval, + providers=providers_dto, + triggers=triggers, + workflow_raw=workflow.workflow_raw, + last_updated=workflow.last_updated, + disabled=workflow.is_disabled, + ) return WorkflowExecutionsPaginatedResultsDto( limit=limit, offset=offset, @@ -569,7 +585,7 @@ def get_workflow_by_id( passCount=pass_count, failCount=fail_count, avgDuration=avgDuration, - workflow=final_workflow + workflow=final_workflow, ) diff --git a/keep/providers/providers_factory.py b/keep/providers/providers_factory.py index 6e9f6a70e..5fb75e755 100644 --- a/keep/providers/providers_factory.py +++ b/keep/providers/providers_factory.py @@ -399,6 +399,7 @@ def get_installed_providers( provider_copy.installed_by = p.installed_by provider_copy.installation_time = p.installation_time provider_copy.last_pull_time = p.last_pull_time + provider_copy.provisioned = p.provisioned try: provider_auth = {"name": p.name} if include_details: diff --git a/keep/providers/providers_service.py b/keep/providers/providers_service.py index 804fd17dc..9dbb0a3ba 100644 --- a/keep/providers/providers_service.py +++ b/keep/providers/providers_service.py @@ -8,7 +8,7 @@ from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select -from keep.api.core.db import engine, get_provider_by_name +from keep.api.core.db import engine, get_all_provisioned_providers, get_provider_by_name from keep.api.models.db.provider import Provider from keep.api.models.provider import Provider as ProviderModel from keep.contextmanager.contextmanager import ContextManager @@ -45,6 +45,8 @@ def install_provider( provider_name: str, provider_type: str, provider_config: Dict[str, Any], + provisioned: bool = False, + validate_scopes: bool = True, ) -> Dict[str, Any]: provider_unique_id = uuid.uuid4().hex logger.info( @@ -69,7 +71,10 @@ def install_provider( except Exception as e: raise HTTPException(status_code=400, detail=str(e)) - validated_scopes = provider.validate_scopes() + if validate_scopes: + validated_scopes = provider.validate_scopes() + else: + validated_scopes = {} secret_manager = SecretManagerFactory.get_secret_manager(context_manager) secret_name = f"{tenant_id}_{provider_type}_{provider_unique_id}" @@ -89,6 +94,7 @@ def install_provider( configuration_key=secret_name, validatedScopes=validated_scopes, consumer=provider.is_consumer, + provisioned=provisioned, ) try: session.add(provider_model) @@ -129,6 +135,9 @@ def update_provider( if not provider: raise HTTPException(404, detail="Provider not found") + if provider.provisioned: + raise HTTPException(403, detail="Cannot update a provisioned provider") + provider_config = { "authentication": provider_info, "name": provider.name, @@ -160,7 +169,9 @@ def update_provider( } @staticmethod - def delete_provider(tenant_id: str, provider_id: str, session: Session): + def delete_provider( + tenant_id: str, provider_id: str, session: Session, allow_provisioned=False + ): provider = session.exec( select(Provider).where( (Provider.tenant_id == tenant_id) & (Provider.id == provider_id) @@ -170,6 +181,9 @@ def delete_provider(tenant_id: str, provider_id: str, session: Session): if not provider: raise HTTPException(404, detail="Provider not found") + if provider.provisioned and not allow_provisioned: + raise HTTPException(403, detail="Cannot delete a provisioned provider") + context_manager = ContextManager(tenant_id=tenant_id) secret_manager = SecretManagerFactory.get_secret_manager(context_manager) @@ -231,6 +245,18 @@ def provision_providers_from_env(tenant_id: str): context_manager = ContextManager(tenant_id=tenant_id) parser._parse_providers_from_env(context_manager) env_providers = context_manager.providers_context + + # first, remove any provisioned providers that are not in the env + prev_provisioned_providers = get_all_provisioned_providers(tenant_id) + for provider in prev_provisioned_providers: + if provider.name not in env_providers: + with Session(engine) as session: + logger.info(f"Deleting provider {provider.name}") + ProvidersService.delete_provider( + tenant_id, provider.id, session, allow_provisioned=True + ) + logger.info(f"Provider {provider.name} deleted") + for provider_name, provider_config in env_providers.items(): logger.info(f"Provisioning provider {provider_name}") # if its already installed, skip @@ -238,12 +264,18 @@ def provision_providers_from_env(tenant_id: str): logger.info(f"Provider {provider_name} already installed") continue logger.info(f"Installing provider {provider_name}") - ProvidersService.install_provider( - tenant_id=tenant_id, - installed_by="system", - provider_id=provider_config["type"], - provider_name=provider_name, - provider_type=provider_config["type"], - provider_config=provider_config["authentication"], - ) + try: + ProvidersService.install_provider( + tenant_id=tenant_id, + installed_by="system", + provider_id=provider_config["type"], + provider_name=provider_name, + provider_type=provider_config["type"], + provider_config=provider_config["authentication"], + provisioned=True, + validate_scopes=False, + ) + except Exception: + logger.exception(f"Failed to provision provider {provider_name}") + continue logger.info(f"Provider {provider_name} provisioned") diff --git a/keep/workflowmanager/workflowstore.py b/keep/workflowmanager/workflowstore.py index cfeb7021d..27fd8c898 100644 --- a/keep/workflowmanager/workflowstore.py +++ b/keep/workflowmanager/workflowstore.py @@ -1,8 +1,8 @@ import io import logging import os -import uuid import random +import uuid import requests import validators @@ -12,20 +12,21 @@ from keep.api.core.db import ( add_or_update_workflow, delete_workflow, + delete_workflow_by_provisioned_file, + get_all_provisioned_workflows, get_all_workflows, get_all_workflows_yamls, get_raw_workflow, + get_workflow, get_workflow_execution, get_workflows_with_last_execution, get_workflows_with_last_executions_v2, ) from keep.api.models.db.workflow import Workflow as WorkflowModel +from keep.api.models.workflow import ProviderDTO from keep.parser.parser import Parser -from keep.workflowmanager.workflow import Workflow from keep.providers.providers_factory import ProvidersFactory -from keep.api.models.workflow import ( - ProviderDTO, -) +from keep.workflowmanager.workflow import Workflow class WorkflowStore: @@ -62,11 +63,19 @@ def create_workflow(self, tenant_id: str, created_by, workflow: dict): def delete_workflow(self, tenant_id, workflow_id): self.logger.info(f"Deleting workflow {workflow_id}") + workflow = get_workflow(tenant_id, workflow_id) + if not workflow: + raise HTTPException( + status_code=404, detail=f"Workflow {workflow_id} not found" + ) + if workflow.provisioned: + raise HTTPException(403, detail="Cannot delete a provisioned workflow") try: delete_workflow(tenant_id, workflow_id) - except Exception: + except Exception as e: + self.logger.exception(f"Error deleting workflow {workflow_id}: {str(e)}") raise HTTPException( - status_code=404, detail=f"Workflow {workflow_id} not found" + status_code=500, detail=f"Failed to delete workflow {workflow_id}" ) def _parse_workflow_to_dict(self, workflow_path: str) -> dict: @@ -133,7 +142,9 @@ def get_all_workflows(self, tenant_id: str) -> list[WorkflowModel]: workflows = get_all_workflows(tenant_id) return workflows - def get_all_workflows_with_last_execution(self, tenant_id: str, is_v2: bool = False) -> list[dict]: + def get_all_workflows_with_last_execution( + self, tenant_id: str, is_v2: bool = False + ) -> list[dict]: # list all tenant's workflows if is_v2: workflows = get_workflows_with_last_executions_v2(tenant_id, 15) @@ -226,6 +237,101 @@ def _get_workflows_from_directory( ) return workflows + @staticmethod + def provision_workflows_from_directory( + tenant_id: str, workflows_dir: str = None + ) -> list[Workflow]: + """ + Provision workflows from a directory. + + Args: + tenant_id (str): The tenant ID. + workflows_dir (str, optional): A directory containing workflow YAML files. + If not provided, it will be read from the WORKFLOWS_DIR environment variable. + + Returns: + list[Workflow]: A list of provisioned Workflow objects. + """ + logger = logging.getLogger(__name__) + parser = Parser() + provisioned_workflows = [] + + if not workflows_dir: + workflows_dir = os.environ.get("KEEP_WORKFLOWS_DIRECTORY") + if not workflows_dir: + logger.info( + "No workflows directory provided - no provisioning will be done" + ) + return [] + + if not os.path.isdir(workflows_dir): + raise FileNotFoundError(f"Directory {workflows_dir} does not exist") + + # Get all existing provisioned workflows + provisioned_workflows = get_all_provisioned_workflows(tenant_id) + + # Check for workflows that are no longer in the directory or outside the workflows_dir and delete them + for workflow in provisioned_workflows: + if ( + not os.path.exists(workflow.provisioned_file) + or not os.path.commonpath([workflows_dir, workflow.provisioned_file]) + == workflows_dir + ): + logger.info( + f"Deprovisioning workflow {workflow.id} as its file no longer exists or is outside the workflows directory" + ) + delete_workflow_by_provisioned_file( + tenant_id, workflow.provisioned_file + ) + logger.info(f"Workflow {workflow.id} deprovisioned successfully") + + # Provision new workflows + for file in os.listdir(workflows_dir): + if file.endswith((".yaml", ".yml")): + logger.info(f"Provisioning workflow from {file}") + workflow_path = os.path.join(workflows_dir, file) + + try: + with open(workflow_path, "r") as yaml_file: + workflow_yaml = yaml.safe_load(yaml_file) + if "workflow" in workflow_yaml: + workflow_yaml = workflow_yaml["workflow"] + # backward compatibility + elif "alert" in workflow_yaml: + workflow_yaml = workflow_yaml["alert"] + + workflow_name = workflow_yaml.get("name") or workflow_yaml.get("id") + if not workflow_name: + logger.error(f"Workflow from {file} does not have a name or id") + continue + workflow_id = str(uuid.uuid4()) + workflow_description = workflow_yaml.get("description") + workflow_interval = parser.parse_interval(workflow_yaml) + workflow_disabled = parser.parse_disabled(workflow_yaml) + + add_or_update_workflow( + id=workflow_id, + name=workflow_name, + tenant_id=tenant_id, + description=workflow_description, + created_by="system", + interval=workflow_interval, + is_disabled=workflow_disabled, + workflow_raw=yaml.dump(workflow_yaml), + provisioned=True, + provisioned_file=workflow_path, + ) + provisioned_workflows.append(workflow_yaml) + + logger.info(f"Workflow from {file} provisioned successfully") + except Exception as e: + logger.error( + f"Error provisioning workflow from {file}", + extra={"exception": e}, + ) + + return provisioned_workflows + def _read_workflow_from_stream(self, stream) -> dict: """ Parse a workflow from an IO stream. @@ -247,7 +353,9 @@ def _read_workflow_from_stream(self, stream) -> dict: raise e return workflow - def get_random_workflow_templates(self, tenant_id: str, workflows_dir: str, limit: int) -> list[dict]: + def get_random_workflow_templates( + self, tenant_id: str, workflows_dir: str, limit: int + ) -> list[dict]: """ Get random workflows from a directory. Args: @@ -261,7 +369,9 @@ def get_random_workflow_templates(self, tenant_id: str, workflows_dir: str, limi if not os.path.isdir(workflows_dir): raise FileNotFoundError(f"Directory {workflows_dir} does not exist") - workflow_yaml_files = [f for f in os.listdir(workflows_dir) if f.endswith(('.yaml', '.yml'))] + workflow_yaml_files = [ + f for f in os.listdir(workflows_dir) if f.endswith((".yaml", ".yml")) + ] if not workflow_yaml_files: raise FileNotFoundError(f"No workflows found in directory {workflows_dir}") @@ -275,15 +385,17 @@ def get_random_workflow_templates(self, tenant_id: str, workflows_dir: str, limi file_path = os.path.join(workflows_dir, file) workflow_yaml = self._parse_workflow_to_dict(file_path) if "workflow" in workflow_yaml: - workflow_yaml['name'] = workflow_yaml['workflow']['id'] - workflow_yaml['workflow_raw'] = yaml.dump(workflow_yaml) - workflow_yaml['workflow_raw_id'] = workflow_yaml['workflow']['id'] + workflow_yaml["name"] = workflow_yaml["workflow"]["id"] + workflow_yaml["workflow_raw"] = yaml.dump(workflow_yaml) + workflow_yaml["workflow_raw_id"] = workflow_yaml["workflow"]["id"] workflows.append(workflow_yaml) count += 1 self.logger.info(f"Workflow from {file} fetched successfully") except Exception as e: - self.logger.error(f"Error parsing or fetching workflow from {file}: {e}") + self.logger.error( + f"Error parsing or fetching workflow from {file}: {e}" + ) return workflows def group_last_workflow_executions(self, workflows: list[dict]) -> list[dict]: @@ -294,7 +406,7 @@ def group_last_workflow_executions(self, workflows: list[dict]) -> list[dict]: self.logger.info(f"workflow_executions: {workflows}") workflow_dict = {} for item in workflows: - workflow,started,execution_time,status = item + workflow, started, execution_time, status = item workflow_id = workflow.id # Initialize the workflow if not already in the dictionary @@ -304,14 +416,14 @@ def group_last_workflow_executions(self, workflows: list[dict]) -> list[dict]: "workflow_last_run_started": None, "workflow_last_run_time": None, "workflow_last_run_status": None, - "workflow_last_executions": [] + "workflow_last_executions": [], } # Update the latest execution details if available - if workflow_dict[workflow_id]["workflow_last_run_started"] is None : + if workflow_dict[workflow_id]["workflow_last_run_started"] is None: workflow_dict[workflow_id]["workflow_last_run_status"] = status workflow_dict[workflow_id]["workflow_last_run_started"] = started - workflow_dict[workflow_id]["workflow_last_run_time"] = started + workflow_dict[workflow_id]["workflow_last_run_time"] = started # Add the execution to the list of executions if started is not None: @@ -319,7 +431,7 @@ def group_last_workflow_executions(self, workflows: list[dict]) -> list[dict]: { "status": status, "execution_time": execution_time, - "started": started + "started": started, } ) # Convert the dictionary to a list of results @@ -329,14 +441,16 @@ def group_last_workflow_executions(self, workflows: list[dict]) -> list[dict]: "workflow_last_run_status": workflow_info["workflow_last_run_status"], "workflow_last_run_time": workflow_info["workflow_last_run_time"], "workflow_last_run_started": workflow_info["workflow_last_run_started"], - "workflow_last_executions": workflow_info["workflow_last_executions"] + "workflow_last_executions": workflow_info["workflow_last_executions"], } for workflow_id, workflow_info in workflow_dict.items() ] return results - def get_workflow_meta_data(self, tenant_id: str, workflow: dict, installed_providers_by_type: dict): + def get_workflow_meta_data( + self, tenant_id: str, workflow: dict, installed_providers_by_type: dict + ): providers_dto = [] triggers = [] @@ -354,19 +468,28 @@ def get_workflow_meta_data(self, tenant_id: str, workflow: dict, installed_provi # Parse the workflow YAML safely workflow_yaml = yaml.safe_load(workflow_raw_data) if not workflow_yaml: - self.logger.error(f"Parsed workflow_yaml is empty or invalid: {workflow_raw_data}") + self.logger.error( + f"Parsed workflow_yaml is empty or invalid: {workflow_raw_data}" + ) return providers_dto, triggers providers = self.parser.get_providers_from_workflow(workflow_yaml) except Exception as e: # Improved logging to capture more details about the error - self.logger.error(f"Failed to parse workflow in get_workflow_meta_data: {e}, workflow: {workflow}") - return providers_dto, triggers # Return empty providers and triggers in case of error + self.logger.error( + f"Failed to parse workflow in get_workflow_meta_data: {e}, workflow: {workflow}" + ) + return ( + providers_dto, + triggers, + ) # Return empty providers and triggers in case of error # Step 2: Process providers and add them to DTO for provider in providers: try: - provider_data = installed_providers_by_type[provider.get("type")][provider.get("name")] + provider_data = installed_providers_by_type[provider.get("type")][ + provider.get("name") + ] provider_dto = ProviderDTO( name=provider_data.name, type=provider_data.type, @@ -377,9 +500,13 @@ def get_workflow_meta_data(self, tenant_id: str, workflow: dict, installed_provi except KeyError: # Handle case where the provider is not installed try: - conf = ProvidersFactory.get_provider_required_config(provider.get("type")) + conf = ProvidersFactory.get_provider_required_config( + provider.get("type") + ) except ModuleNotFoundError: - self.logger.warning(f"Non-existing provider in workflow: {provider.get('type')}") + self.logger.warning( + f"Non-existing provider in workflow: {provider.get('type')}" + ) conf = None # Handle providers based on whether they require config @@ -387,11 +514,13 @@ def get_workflow_meta_data(self, tenant_id: str, workflow: dict, installed_provi name=provider.get("name"), type=provider.get("type"), id=None, - installed=(conf is None), # Consider it installed if no config is required + installed=( + conf is None + ), # Consider it installed if no config is required ) providers_dto.append(provider_dto) # Step 3: Extract triggers from workflow triggers = self.parser.get_triggers_from_workflow(workflow_yaml) - return providers_dto, triggers \ No newline at end of file + return providers_dto, triggers diff --git a/tests/fixtures/client.py b/tests/fixtures/client.py index bc7df1872..c49b6ece5 100644 --- a/tests/fixtures/client.py +++ b/tests/fixtures/client.py @@ -1,3 +1,4 @@ +import asyncio import hashlib import importlib import sys @@ -29,6 +30,11 @@ def test_app(monkeypatch, request): for module in list(sys.modules): if module.startswith("keep.api.routes"): del sys.modules[module] + + # this is a fucking bug in db patching ffs it ruined my saturday + elif module.startswith("keep.providers.providers_service"): + importlib.reload(sys.modules[module]) + if "keep.api.api" in sys.modules: importlib.reload(sys.modules["keep.api.api"]) @@ -36,6 +42,11 @@ def test_app(monkeypatch, request): from keep.api.api import get_app app = get_app() + + # Manually trigger the startup event + for event_handler in app.router.on_startup: + asyncio.run(event_handler()) + return app diff --git a/tests/provision/workflows_1/provision_example_1.yml b/tests/provision/workflows_1/provision_example_1.yml new file mode 100644 index 000000000..aeeb7f140 --- /dev/null +++ b/tests/provision/workflows_1/provision_example_1.yml @@ -0,0 +1,20 @@ +workflow: + id: aks-example + description: aks-example + triggers: + - type: manual + steps: + # get all pods + - name: get-pods + provider: + type: aks + config: "{{ providers.aks }}" + with: + command_type: get_pods + actions: + - name: echo-pod-status + foreach: "{{ steps.get-pods.results }}" + provider: + type: console + with: + alert_message: "Pod name: {{ foreach.value.metadata.name }} || Namespace: {{ foreach.value.metadata.namespace }} || Status: {{ foreach.value.status.phase }}" diff --git a/tests/provision/workflows_1/provision_example_2.yml b/tests/provision/workflows_1/provision_example_2.yml new file mode 100644 index 000000000..4b5518ef9 --- /dev/null +++ b/tests/provision/workflows_1/provision_example_2.yml @@ -0,0 +1,29 @@ +workflow: + id: Resend-Python-service + description: Python Resend Mail + triggers: + - type: manual + owners: [] + services: [] + steps: + - name: run-script + provider: + config: '{{ providers.default-bash }}' + type: bash + with: + command: python3 test.py + timeout: 5 + actions: + - condition: + - assert: '{{ steps.run-script.results.return_code }} == 0' + name: assert-condition + type: assert + name: trigger-resend + provider: + type: resend + config: "{{ providers.resend-test }}" + with: + _from: "onboarding@resend.dev" + to: "youremail.dev@gmail.com" + subject: "Python test is up!" + html: <p>Python test is up!</p> diff --git a/tests/provision/workflows_1/provision_example_3.yml b/tests/provision/workflows_1/provision_example_3.yml new file mode 100644 index 000000000..3e6e85cf4 --- /dev/null +++ b/tests/provision/workflows_1/provision_example_3.yml @@ -0,0 +1,14 @@ +workflow: + id: autosupress + strategy: parallel + description: demonstrates how to automatically suppress alerts + triggers: + - type: alert + actions: + - name: dismiss-alert + provider: + type: mock + with: + enrich_alert: + - key: dismissed + value: "true" diff --git a/tests/provision/workflows_2/provision_example_1.yml b/tests/provision/workflows_2/provision_example_1.yml new file mode 100644 index 000000000..aeeb7f140 --- /dev/null +++ b/tests/provision/workflows_2/provision_example_1.yml @@ -0,0 +1,20 @@ +workflow: + id: aks-example + description: aks-example + triggers: + - type: manual + steps: + # get all pods + - name: get-pods + provider: + type: aks + config: "{{ providers.aks }}" + with: + command_type: get_pods + actions: + - name: echo-pod-status + foreach: "{{ steps.get-pods.results }}" + provider: + type: console + with: + alert_message: "Pod name: {{ foreach.value.metadata.name }} || Namespace: {{ foreach.value.metadata.namespace }} || Status: {{ foreach.value.status.phase }}" diff --git a/tests/provision/workflows_2/provision_example_2.yml b/tests/provision/workflows_2/provision_example_2.yml new file mode 100644 index 000000000..4b5518ef9 --- /dev/null +++ b/tests/provision/workflows_2/provision_example_2.yml @@ -0,0 +1,29 @@ +workflow: + id: Resend-Python-service + description: Python Resend Mail + triggers: + - type: manual + owners: [] + services: [] + steps: + - name: run-script + provider: + config: '{{ providers.default-bash }}' + type: bash + with: + command: python3 test.py + timeout: 5 + actions: + - condition: + - assert: '{{ steps.run-script.results.return_code }} == 0' + name: assert-condition + type: assert + name: trigger-resend + provider: + type: resend + config: "{{ providers.resend-test }}" + with: + _from: "onboarding@resend.dev" + to: "youremail.dev@gmail.com" + subject: "Python test is up!" + html: <p>Python test is up!</p> diff --git a/tests/test_auth.py b/tests/test_auth.py index 09ce7956a..86ad7a385 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -53,7 +53,7 @@ def get_mock_jwt_payload(token, *args, **kwargs): @pytest.mark.parametrize( "test_app", ["SINGLE_TENANT", "MULTI_TENANT", "NO_AUTH"], indirect=True ) -def test_api_key_with_header(client, db_session, test_app): +def test_api_key_with_header(db_session, client, test_app): """Tests the API key authentication with the x-api-key/digest""" auth_type = os.getenv("AUTH_TYPE") valid_api_key = "valid_api_key" @@ -95,7 +95,7 @@ def test_api_key_with_header(client, db_session, test_app): @pytest.mark.parametrize( "test_app", ["SINGLE_TENANT", "MULTI_TENANT", "NO_AUTH"], indirect=True ) -def test_bearer_token(client, db_session, test_app): +def test_bearer_token(db_session, client, test_app): """Tests the bearer token authentication""" auth_type = os.getenv("AUTH_TYPE") # Test bearer tokens @@ -121,7 +121,7 @@ def test_bearer_token(client, db_session, test_app): @pytest.mark.parametrize( "test_app", ["SINGLE_TENANT", "MULTI_TENANT", "NO_AUTH"], indirect=True ) -def test_webhook_api_key(client, db_session, test_app): +def test_webhook_api_key(db_session, client, test_app): """Tests the webhook API key authentication""" auth_type = os.getenv("AUTH_TYPE") valid_api_key = "valid_api_key" @@ -167,7 +167,7 @@ def test_webhook_api_key(client, db_session, test_app): # sanity check with keycloak @pytest.mark.parametrize("test_app", ["KEYCLOAK"], indirect=True) -def test_keycloak_sanity(keycloak_client, keycloak_token, client, test_app): +def test_keycloak_sanity(db_session, keycloak_client, keycloak_token, client, test_app): """Tests the keycloak sanity check""" # Use the token to make a request to the Keep API headers = {"Authorization": f"Bearer {keycloak_token}"} @@ -182,7 +182,7 @@ def test_keycloak_sanity(keycloak_client, keycloak_token, client, test_app): ], indirect=True, ) -def test_api_key_impersonation_without_admin(client, db_session, test_app): +def test_api_key_impersonation_without_admin(db_session, client, test_app): """Tests the API key impersonation with different environment settings""" valid_api_key = "valid_admin_api_key" @@ -207,7 +207,7 @@ def test_api_key_impersonation_without_admin(client, db_session, test_app): ], indirect=True, ) -def test_api_key_impersonation_without_user_provision(client, db_session, test_app): +def test_api_key_impersonation_without_user_provision(db_session, client, test_app): """Tests the API key impersonation with different environment settings""" valid_api_key = "valid_admin_api_key" @@ -239,7 +239,7 @@ def test_api_key_impersonation_without_user_provision(client, db_session, test_a ], indirect=True, ) -def test_api_key_impersonation_with_user_provision(client, db_session, test_app): +def test_api_key_impersonation_with_user_provision(db_session, client, test_app): """Tests the API key impersonation with different environment settings""" valid_api_key = "valid_admin_api_key" @@ -272,7 +272,7 @@ def test_api_key_impersonation_with_user_provision(client, db_session, test_app) indirect=True, ) def test_api_key_impersonation_provisioned_user_cant_login( - client, db_session, test_app + db_session, client, test_app ): """Tests the API key impersonation with different environment settings""" diff --git a/tests/test_enrichments.py b/tests/test_enrichments.py index 74ea6878c..e75c99a24 100644 --- a/tests/test_enrichments.py +++ b/tests/test_enrichments.py @@ -457,7 +457,7 @@ def test_mapping_rule_with_elsatic(mock_session, mock_alert_dto, setup_alerts): @pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) -def test_enrichment(client, db_session, test_app, mock_alert_dto, elastic_client): +def test_enrichment(db_session, client, test_app, mock_alert_dto, elastic_client): # add some rule rule = MappingRule( id=1, @@ -495,7 +495,7 @@ def test_enrichment(client, db_session, test_app, mock_alert_dto, elastic_client @pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) -def test_disposable_enrichment(client, db_session, test_app, mock_alert_dto): +def test_disposable_enrichment(db_session, client, test_app, mock_alert_dto): # SHAHAR: there is a voodoo so that you must do something with the db_session to kick it off rule = MappingRule( id=1, diff --git a/tests/test_extraction_rules.py b/tests/test_extraction_rules.py index 24759b6e2..190220a34 100644 --- a/tests/test_extraction_rules.py +++ b/tests/test_extraction_rules.py @@ -1,18 +1,15 @@ from time import sleep import pytest - from isodate import parse_datetime -from tests.fixtures.client import client, test_app, setup_api_key +from tests.fixtures.client import client, setup_api_key, test_app # noqa VALID_API_KEY = "valid_api_key" -@pytest.mark.parametrize( - "test_app", ["NO_AUTH"], indirect=True -) -def test_create_extraction_rule(client, test_app, db_session): +@pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) +def test_create_extraction_rule(db_session, client, test_app): setup_api_key(db_session, VALID_API_KEY, role="webhook") # Try to create invalid extraction @@ -33,10 +30,8 @@ def test_create_extraction_rule(client, test_app, db_session): assert response.status_code == 200 -@pytest.mark.parametrize( - "test_app", ["NO_AUTH"], indirect=True -) -def test_extraction_rule_updated_at(client, test_app, db_session): +@pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) +def test_extraction_rule_updated_at(db_session, client, test_app): setup_api_key(db_session, VALID_API_KEY, role="webhook") rule_dict = { @@ -66,7 +61,9 @@ def test_extraction_rule_updated_at(client, test_app, db_session): # Without it update can happen in the same second, so we will not see any changes sleep(1) updated_response = client.put( - f"/extraction/{rule_id}", json=updated_rule_dict, headers={"x-api-key": VALID_API_KEY} + f"/extraction/{rule_id}", + json=updated_rule_dict, + headers={"x-api-key": VALID_API_KEY}, ) assert updated_response.status_code == 200 @@ -75,7 +72,3 @@ def test_extraction_rule_updated_at(client, test_app, db_session): new_updated_at = parse_datetime(updated_response_data["updated_at"]) assert new_updated_at > updated_at - - - - diff --git a/tests/test_metrics.py b/tests/test_metrics.py index e9c400022..deb0b1621 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,36 +1,32 @@ - import pytest from keep.api.core.db import ( add_alerts_to_incident_by_incident_id, - create_incident_from_dict + create_incident_from_dict, ) - -from tests.fixtures.client import client, setup_api_key, test_app +from tests.fixtures.client import client, setup_api_key, test_app # noqa -@pytest.mark.parametrize( - "test_app", ["NO_AUTH"], indirect=True -) -def test_add_remove_alert_to_incidents(client, db_session, test_app, setup_stress_alerts_no_elastic): +@pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) +def test_add_remove_alert_to_incidents( + db_session, client, test_app, setup_stress_alerts_no_elastic +): alerts = setup_stress_alerts_no_elastic(14) - incident = create_incident_from_dict("keep", {"user_generated_name": "test", "description": "test"}) + incident = create_incident_from_dict( + "keep", {"user_generated_name": "test", "description": "test"} + ) valid_api_key = "valid_api_key" setup_api_key(db_session, valid_api_key) - add_alerts_to_incident_by_incident_id( - "keep", - incident.id, - [a.id for a in alerts] - ) + add_alerts_to_incident_by_incident_id("keep", incident.id, [a.id for a in alerts]) - response = client.get( - "/metrics", - headers={"X-API-KEY": "valid_api_key"} - ) + response = client.get("/metrics", headers={"X-API-KEY": "valid_api_key"}) # Checking for alert_total metric - assert f"alerts_total{{incident_name=\"test\" incident_id=\"{incident.id}\"}} 14" in response.text.split("\n") + assert ( + f'alerts_total{{incident_name="test" incident_id="{incident.id}"}} 14' + in response.text.split("\n") + ) # Checking for open_incidents_total metric assert "open_incidents_total 1" in response.text.split("\n") diff --git a/tests/test_provisioning.py b/tests/test_provisioning.py new file mode 100644 index 000000000..4b49ca854 --- /dev/null +++ b/tests/test_provisioning.py @@ -0,0 +1,223 @@ +import asyncio +import importlib +import sys + +import pytest +from fastapi.testclient import TestClient + +from tests.fixtures.client import client, setup_api_key, test_app # noqa + +# Mock data for workflows +MOCK_WORKFLOW_ID = "123e4567-e89b-12d3-a456-426614174000" +MOCK_PROVISIONED_WORKFLOW = { + "id": MOCK_WORKFLOW_ID, + "name": "Test Workflow", + "description": "A provisioned test workflow", + "provisioned": True, +} + + +# Test for deleting a provisioned workflow +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + "KEEP_WORKFLOWS_DIRECTORY": "./tests/provision/workflows_1", + }, + ], + indirect=True, +) +def test_provisioned_workflows(db_session, client, test_app): + response = client.get("/workflows", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + # 3 workflows and 3 provisioned workflows + workflows = response.json() + provisioned_workflows = [w for w in workflows if w.get("provisioned")] + assert len(provisioned_workflows) == 3 + + +# Test for deleting a provisioned workflow +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + "KEEP_WORKFLOWS_DIRECTORY": "./tests/provision/workflows_2", + }, + ], + indirect=True, +) +def test_provisioned_workflows_2(db_session, client, test_app): + response = client.get("/workflows", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + # 3 workflows and 3 provisioned workflows + workflows = response.json() + provisioned_workflows = [w for w in workflows if w.get("provisioned")] + assert len(provisioned_workflows) == 2 + + +# Test for deleting a provisioned workflow +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + "KEEP_WORKFLOWS_DIRECTORY": "./tests/provision/workflows_1", + }, + ], + indirect=True, +) +def test_delete_provisioned_workflow(db_session, client, test_app): + response = client.get("/workflows", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + # 3 workflows and 3 provisioned workflows + workflows = response.json() + provisioned_workflows = [w for w in workflows if w.get("provisioned")] + workflow_id = provisioned_workflows[0].get("id") + response = client.delete( + f"/workflows/{workflow_id}", headers={"x-api-key": "someapikey"} + ) + # can't delete a provisioned workflow + assert response.status_code == 403 + assert response.json() == {"detail": "Cannot delete a provisioned workflow"} + + +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + "KEEP_WORKFLOWS_DIRECTORY": "./tests/provision/workflows_1", + }, + ], + indirect=True, +) +def test_update_provisioned_workflow(db_session, client, test_app): + response = client.get("/workflows", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + # 3 workflows and 3 provisioned workflows + workflows = response.json() + provisioned_workflows = [w for w in workflows if w.get("provisioned")] + workflow_id = provisioned_workflows[0].get("id") + response = client.put( + f"/workflows/{workflow_id}", headers={"x-api-key": "someapikey"} + ) + # can't delete a provisioned workflow + assert response.status_code == 403 + assert response.json() == {"detail": "Cannot update a provisioned workflow"} + + +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + "KEEP_WORKFLOWS_DIRECTORY": "./tests/provision/workflows_1", + }, + ], + indirect=True, +) +def test_reprovision_workflow(monkeypatch, db_session, client, test_app): + response = client.get("/workflows", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + # 3 workflows and 3 provisioned workflows + workflows = response.json() + provisioned_workflows = [w for w in workflows if w.get("provisioned")] + assert len(provisioned_workflows) == 3 + + # Step 2: Change environment variables (simulating new provisioning) + monkeypatch.setenv("KEEP_WORKFLOWS_DIRECTORY", "./tests/provision/workflows_2") + + # Reload the app to apply the new environment changes + importlib.reload(sys.modules["keep.api.api"]) + + # Reinitialize the TestClient with the new app instance + from keep.api.api import get_app + + app = get_app() + + # Manually trigger the startup event + for event_handler in app.router.on_startup: + asyncio.run(event_handler()) + + client = TestClient(get_app()) + + response = client.get("/workflows", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + # 2 workflows and 2 provisioned workflows + workflows = response.json() + provisioned_workflows = [w for w in workflows if w.get("provisioned")] + assert len(provisioned_workflows) == 2 + + +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + "KEEP_PROVIDERS": '{"keepVictoriaMetrics":{"type":"victoriametrics","authentication":{"VMAlertHost":"http://localhost","VMAlertPort": 1234}},"keepClickhouse1":{"type":"clickhouse","authentication":{"host":"http://localhost","port":1234,"username":"keep","password":"keep","database":"keep-db"}}}', + }, + ], + indirect=True, +) +def test_provision_provider(db_session, client, test_app): + response = client.get("/providers", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + # 3 workflows and 3 provisioned workflows + providers = response.json() + provisioned_providers = [ + p for p in providers.get("installed_providers") if p.get("provisioned") + ] + assert len(provisioned_providers) == 2 + + +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + "KEEP_PROVIDERS": '{"keepVictoriaMetrics":{"type":"victoriametrics","authentication":{"VMAlertHost":"http://localhost","VMAlertPort": 1234}},"keepClickhouse1":{"type":"clickhouse","authentication":{"host":"http://localhost","port":1234,"username":"keep","password":"keep","database":"keep-db"}}}', + }, + ], + indirect=True, +) +def test_reprovision_provider(monkeypatch, db_session, client, test_app): + response = client.get("/providers", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + # 3 workflows and 3 provisioned workflows + providers = response.json() + provisioned_providers = [ + p for p in providers.get("installed_providers") if p.get("provisioned") + ] + assert len(provisioned_providers) == 2 + + # Step 2: Change environment variables (simulating new provisioning) + monkeypatch.setenv( + "KEEP_PROVIDERS", + '{"keepPrometheus":{"type":"prometheus","authentication":{"url":"http://localhost","port":9090}}}', + ) + + # Reload the app to apply the new environment changes + importlib.reload(sys.modules["keep.api.api"]) + + # Reinitialize the TestClient with the new app instance + from keep.api.api import get_app + + app = get_app() + + # Manually trigger the startup event + for event_handler in app.router.on_startup: + asyncio.run(event_handler()) + + client = TestClient(app) + + # Step 3: Verify if the new provider is provisioned after reloading + response = client.get("/providers", headers={"x-api-key": "someapikey"}) + assert response.status_code == 200 + providers = response.json() + provisioned_providers = [ + p for p in providers.get("installed_providers") if p.get("provisioned") + ] + assert len(provisioned_providers) == 1 + assert provisioned_providers[0]["type"] == "prometheus" diff --git a/tests/test_rules_api.py b/tests/test_rules_api.py index f29e2992d..ff039950d 100644 --- a/tests/test_rules_api.py +++ b/tests/test_rules_api.py @@ -1,8 +1,7 @@ import pytest -from keep.api.core.dependencies import SINGLE_TENANT_UUID from keep.api.core.db import create_rule as create_rule_db - +from keep.api.core.dependencies import SINGLE_TENANT_UUID from tests.fixtures.client import client, setup_api_key, test_app # noqa TEST_RULE_DATA = { @@ -18,28 +17,36 @@ "created_by": "test@keephq.dev", } -INVALID_DATA_STEPS = [{ - "update": {"sqlQuery": {"sql": "", "params": []}}, - "error": "SQL is required", -}, { - "update": {"sqlQuery": {"sql": "SELECT", "params": []}}, - "error": "Params are required", -}, { - "update": {"celQuery": ""}, - "error": "CEL is required", -}, { - "update": {"ruleName": ""}, - "error": "Rule name is required", -}, { - "update": {"timeframeInSeconds": 0}, - "error": "Timeframe is required", -}, { - "update": {"timeUnit": ""}, - "error": "Timeunit is required", -}] +INVALID_DATA_STEPS = [ + { + "update": {"sqlQuery": {"sql": "", "params": []}}, + "error": "SQL is required", + }, + { + "update": {"sqlQuery": {"sql": "SELECT", "params": []}}, + "error": "Params are required", + }, + { + "update": {"celQuery": ""}, + "error": "CEL is required", + }, + { + "update": {"ruleName": ""}, + "error": "Rule name is required", + }, + { + "update": {"timeframeInSeconds": 0}, + "error": "Timeframe is required", + }, + { + "update": {"timeUnit": ""}, + "error": "Timeunit is required", + }, +] + @pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) -def test_get_rules_api(client, db_session, test_app): +def test_get_rules_api(db_session, client, test_app): rule = create_rule_db(**TEST_RULE_DATA) response = client.get( @@ -50,7 +57,7 @@ def test_get_rules_api(client, db_session, test_app): assert response.status_code == 200 data = response.json() assert len(data) == 1 - assert data[0]['id'] == str(rule.id) + assert data[0]["id"] == str(rule.id) rule2 = create_rule_db(**TEST_RULE_DATA) @@ -62,27 +69,26 @@ def test_get_rules_api(client, db_session, test_app): assert response2.status_code == 200 data = response2.json() assert len(data) == 2 - assert data[0]['id'] == str(rule.id) - assert data[1]['id'] == str(rule2.id) + assert data[0]["id"] == str(rule.id) + assert data[1]["id"] == str(rule2.id) @pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) -def test_create_rule_api(client, db_session, test_app): +def test_create_rule_api(db_session, client, test_app): rule_data = { "ruleName": "test rule", - "sqlQuery": {"sql": "SELECT * FROM alert where severity = %s", "params": ["critical"]}, + "sqlQuery": { + "sql": "SELECT * FROM alert where severity = %s", + "params": ["critical"], + }, "celQuery": "severity = 'critical'", "timeframeInSeconds": 300, "timeUnit": "seconds", "requireApprove": False, } - response = client.post( - "/rules", - headers={"x-api-key": "some-key"}, - json=rule_data - ) + response = client.post("/rules", headers={"x-api-key": "some-key"}, json=rule_data) assert response.status_code == 200 data = response.json() @@ -92,9 +98,7 @@ def test_create_rule_api(client, db_session, test_app): invalid_rule_data = {k: v for k, v in rule_data.items() if k != "ruleName"} invalid_data_response = client.post( - "/rules", - headers={"x-api-key": "some-key"}, - json=invalid_rule_data + "/rules", headers={"x-api-key": "some-key"}, json=invalid_rule_data ) assert invalid_data_response.status_code == 422 @@ -109,7 +113,7 @@ def test_create_rule_api(client, db_session, test_app): invalid_data_response_2 = client.post( "/rules", headers={"x-api-key": "some-key"}, - json=dict(rule_data, **invalid_data_step["update"]) + json=dict(rule_data, **invalid_data_step["update"]), ) assert invalid_data_response_2.status_code == 400, current_step @@ -119,7 +123,7 @@ def test_create_rule_api(client, db_session, test_app): @pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) -def test_delete_rule_api(client, db_session, test_app): +def test_delete_rule_api(db_session, client, test_app): rule = create_rule_db(**TEST_RULE_DATA) response = client.delete( @@ -143,15 +147,17 @@ def test_delete_rule_api(client, db_session, test_app): assert data["detail"] == "Rule not found" - @pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) -def test_update_rule_api(client, db_session, test_app): +def test_update_rule_api(db_session, client, test_app): rule = create_rule_db(**TEST_RULE_DATA) rule_data = { "ruleName": "test rule", - "sqlQuery": {"sql": "SELECT * FROM alert where severity = %s", "params": ["critical"]}, + "sqlQuery": { + "sql": "SELECT * FROM alert where severity = %s", + "params": ["critical"], + }, "celQuery": "severity = 'critical'", "timeframeInSeconds": 300, "timeUnit": "seconds", @@ -159,9 +165,7 @@ def test_update_rule_api(client, db_session, test_app): } response = client.put( - "/rules/{}".format(rule.id), - headers={"x-api-key": "some-key"}, - json=rule_data + "/rules/{}".format(rule.id), headers={"x-api-key": "some-key"}, json=rule_data ) assert response.status_code == 200 @@ -174,7 +178,7 @@ def test_update_rule_api(client, db_session, test_app): invalid_data_response_2 = client.put( "/rules/{}".format(rule.id), headers={"x-api-key": "some-key"}, - json=dict(rule_data, **invalid_data_step["update"]) + json=dict(rule_data, **invalid_data_step["update"]), ) assert invalid_data_response_2.status_code == 400, current_step diff --git a/tests/test_search_alerts.py b/tests/test_search_alerts.py index bb45436ff..a7cc3b867 100644 --- a/tests/test_search_alerts.py +++ b/tests/test_search_alerts.py @@ -736,7 +736,7 @@ def test_special_characters_in_strings(db_session, setup_alerts): # tests 10k alerts @pytest.mark.parametrize( - "setup_stress_alerts", [{"num_alerts": 10000}], indirect=True + "setup_stress_alerts", [{"num_alerts": 1000}], indirect=True ) # Generate 10,000 alerts def test_filter_large_dataset(db_session, setup_stress_alerts): search_query = SearchQuery( @@ -745,7 +745,7 @@ def test_filter_large_dataset(db_session, setup_stress_alerts): "params": {"source_1": "source_1", "severity_1": "critical"}, }, cel_query='(source == "source_1") && (severity == "critical")', - limit=10000, + limit=1000, ) # first, use elastic os.environ["ELASTIC_ENABLED"] = "true" @@ -764,10 +764,10 @@ def test_filter_large_dataset(db_session, setup_stress_alerts): # compare assert len(elastic_filtered_alerts) == len(db_filtered_alerts) print( - "time taken for 10k alerts with elastic: ", + "time taken for 1k alerts with elastic: ", elastic_end_time - elastic_start_time, ) - print("time taken for 10k alerts with db: ", db_end_time - db_start_time) + print("time taken for 1k alerts with db: ", db_end_time - db_start_time) @pytest.mark.parametrize("setup_stress_alerts", [{"num_alerts": 10000}], indirect=True) @@ -1312,7 +1312,7 @@ def test_severity_comparisons( @pytest.mark.parametrize("test_app", ["NO_AUTH"], indirect=True) -def test_alerts_enrichment_in_search(client, db_session, test_app, elastic_client): +def test_alerts_enrichment_in_search(db_session, client, test_app, elastic_client): rule = MappingRule( id=1, diff --git a/tests/test_search_alerts_configuration.py b/tests/test_search_alerts_configuration.py index 7d3bb700e..bf4b1ab9a 100644 --- a/tests/test_search_alerts_configuration.py +++ b/tests/test_search_alerts_configuration.py @@ -12,7 +12,7 @@ @pytest.mark.parametrize("test_app", ["SINGLE_TENANT"], indirect=True) def test_single_tenant_configuration_with_elastic( - client, elastic_client, db_session, test_app + db_session, client, elastic_client, test_app ): valid_api_key = "valid_api_key" setup_api_key(db_session, valid_api_key) @@ -30,7 +30,7 @@ def test_single_tenant_configuration_with_elastic( ], indirect=True, ) -def test_single_tenant_configuration_without_elastic(client, db_session, test_app): +def test_single_tenant_configuration_without_elastic(db_session, client, test_app): valid_api_key = "valid_api_key" setup_api_key(db_session, valid_api_key) response = client.get("/preset/feed/alerts", headers={"x-api-key": valid_api_key}) @@ -39,7 +39,7 @@ def test_single_tenant_configuration_without_elastic(client, db_session, test_ap @pytest.mark.parametrize("test_app", ["MULTI_TENANT"], indirect=True) def test_multi_tenant_configuration_with_elastic( - client, elastic_client, db_session, test_app + db_session, client, elastic_client, test_app ): valid_api_key = "valid_api_key" valid_api_key_2 = "valid_api_key_2" From 3ea029e1d40ebe2a0021ce4ce35da4b2a3eedc43 Mon Sep 17 00:00:00 2001 From: Shahar Glazner <shaharglazner@gmail.com> Date: Sun, 15 Sep 2024 09:49:35 +0300 Subject: [PATCH 2/9] fix: default workflows directory (#1919) --- keep/api/routes/workflows.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/keep/api/routes/workflows.py b/keep/api/routes/workflows.py index 621e6d843..7ed8dfc2b 100644 --- a/keep/api/routes/workflows.py +++ b/keep/api/routes/workflows.py @@ -387,6 +387,19 @@ def get_random_workflow_templates( "KEEP_WORKFLOWS_PATH", os.path.join(os.path.dirname(__file__), "../../../examples/workflows"), ) + if not os.path.exists(default_directory): + # on the container we use the following path + fallback_directory = "/examples/workflows" + logger.warning( + f"{default_directory} does not exist, using fallback: {fallback_directory}" + ) + if os.path.exists(fallback_directory): + default_directory = fallback_directory + else: + logger.error(f"Neither {default_directory} nor {fallback_directory} exist") + raise FileNotFoundError( + f"Neither {default_directory} nor {fallback_directory} exist" + ) workflows = workflowstore.get_random_workflow_templates( tenant_id=tenant_id, workflows_dir=default_directory, limit=6 ) From c83ec96ac4a50d8dc07443b382add0631409bf4d Mon Sep 17 00:00:00 2001 From: Shahar Glazner <shaharglazner@gmail.com> Date: Sun, 15 Sep 2024 09:52:37 +0300 Subject: [PATCH 3/9] chore(release): 0.24.4 (#1921) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8833cb00e..82ee85d7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "keep" -version = "0.24.3" +version = "0.24.4" description = "Alerting. for developers, by developers." authors = ["Keep Alerting LTD"] readme = "README.md" From c9dc74ab95e96aeddc941ed45a87a817d541e14a Mon Sep 17 00:00:00 2001 From: Shahar Glazner <shaharglazner@gmail.com> Date: Sun, 15 Sep 2024 12:20:05 +0300 Subject: [PATCH 4/9] fix: don't fail the server if api key already in the secret manager (#1923) --- keep/api/core/db_on_start.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/keep/api/core/db_on_start.py b/keep/api/core/db_on_start.py index bb6a9edee..a39abf1e7 100644 --- a/keep/api/core/db_on_start.py +++ b/keep/api/core/db_on_start.py @@ -137,10 +137,17 @@ def try_create_single_tenant(tenant_id: str) -> None: secret_manager = SecretManagerFactory.get_secret_manager( context_manager ) - secret_manager.write_secret( - secret_name=f"{tenant_id}-{api_key_name}", - secret_value=api_key_secret, - ) + try: + secret_manager.write_secret( + secret_name=f"{tenant_id}-{api_key_name}", + secret_value=api_key_secret, + ) + # probably 409 if the secret already exists, but we don't want to fail on that + except Exception: + logger.exception( + f"Failed to write secret for api key {api_key_name}" + ) + pass logger.info(f"Api key {api_key_name} provisioned") logger.info("Api keys provisioned") # commit the changes @@ -162,7 +169,7 @@ def migrate_db(): if os.environ.get("SKIP_DB_CREATION", "false") == "true": logger.info("Skipping running migrations...") return None - + logger.info("Running migrations...") config_path = os.path.dirname(os.path.abspath(__file__)) + "/../../" + "alembic.ini" config = alembic.config.Config(file_=config_path) From 5ac65ca64a773d81a6aa916c2ba290f722ac6cf2 Mon Sep 17 00:00:00 2001 From: Shahar Glazner <shaharglazner@gmail.com> Date: Sun, 15 Sep 2024 15:23:08 +0300 Subject: [PATCH 5/9] feat: add the option to run workflow manually but with alert payload (#1925) --- .../workflows/builder/ReactFlowBuilder.tsx | 11 +++-- .../app/workflows/builder/ReactFlowEditor.tsx | 5 ++- keep-ui/app/workflows/builder/builder.tsx | 1 + keep-ui/app/workflows/builder/editors.tsx | 15 +++++-- .../workflow-run-with-alert-modal.tsx | 11 ++--- keep-ui/utils/hooks/useWorkflowRun.ts | 41 ++++++++++++------- 6 files changed, 56 insertions(+), 28 deletions(-) diff --git a/keep-ui/app/workflows/builder/ReactFlowBuilder.tsx b/keep-ui/app/workflows/builder/ReactFlowBuilder.tsx index ba303a9e2..77b7b861f 100644 --- a/keep-ui/app/workflows/builder/ReactFlowBuilder.tsx +++ b/keep-ui/app/workflows/builder/ReactFlowBuilder.tsx @@ -14,12 +14,14 @@ const nodeTypes = { custom: CustomNode as any }; const edgeTypes: EdgeTypesType = { "custom-edge": CustomEdge as React.ComponentType<any> }; const ReactFlowBuilder = ({ + providers, installedProviders, toolboxConfiguration, definition, onDefinitionChange, validatorConfiguration }: { + providers: Provider[] | undefined | null; installedProviders: Provider[] | undefined | null; toolboxConfiguration: Record<string, any>; definition: any; @@ -29,7 +31,7 @@ const ReactFlowBuilder = ({ }; onDefinitionChange:(def: Definition) => void; }) => { - + const { nodes, edges, @@ -60,13 +62,14 @@ const ReactFlowBuilder = ({ nodeTypes={nodeTypes} edgeTypes={edgeTypes} fitView - > + > <Controls orientation="horizontal"/> <Background/> </ReactFlow> )} - <ReactFlowEditor - providers={installedProviders} + <ReactFlowEditor + providers={providers} + installedProviders={installedProviders} onDefinitionChange= {onDefinitionChange} validatorConfiguration= {validatorConfiguration} /> diff --git a/keep-ui/app/workflows/builder/ReactFlowEditor.tsx b/keep-ui/app/workflows/builder/ReactFlowEditor.tsx index 30ce85c5f..6848dd13c 100644 --- a/keep-ui/app/workflows/builder/ReactFlowEditor.tsx +++ b/keep-ui/app/workflows/builder/ReactFlowEditor.tsx @@ -9,10 +9,12 @@ import debounce from "lodash.debounce"; const ReactFlowEditor = ({ providers, + installedProviders, validatorConfiguration, onDefinitionChange }: { providers: Provider[] | undefined | null; + installedProviders: Provider[] | undefined | null; validatorConfiguration: { step: (step: V2Step, parent?: V2Step, defnition?: ReactFlowDefinition) => boolean; root: (def: Definition) => boolean; @@ -114,7 +116,7 @@ const ReactFlowEditor = ({ <div style={{ width: "300px" }}> <GlobalEditorV2 synced={synced} /> {!selectedNode?.includes('empty') && !isTrigger && <Divider ref={stepEditorRef} />} - {!selectedNode?.includes('empty') && !isTrigger && <StepEditorV2 installedProviders={providers} setSynced={setSynced} />} + {!selectedNode?.includes('empty') && !isTrigger && <StepEditorV2 providers={providers} installedProviders={installedProviders} setSynced={setSynced} />} </div> </div> </div> @@ -124,4 +126,3 @@ const ReactFlowEditor = ({ }; export default ReactFlowEditor; - diff --git a/keep-ui/app/workflows/builder/builder.tsx b/keep-ui/app/workflows/builder/builder.tsx index 3b33d8470..020d13bf4 100644 --- a/keep-ui/app/workflows/builder/builder.tsx +++ b/keep-ui/app/workflows/builder/builder.tsx @@ -313,6 +313,7 @@ function Builder({ <div className="h-[94%]"> <ReactFlowProvider> <ReactFlowBuilder + providers={providers} installedProviders={installedProviders} definition={definition} validatorConfiguration={ValidatorConfigurationV2} diff --git a/keep-ui/app/workflows/builder/editors.tsx b/keep-ui/app/workflows/builder/editors.tsx index 8ebcc7d76..53283f26b 100644 --- a/keep-ui/app/workflows/builder/editors.tsx +++ b/keep-ui/app/workflows/builder/editors.tsx @@ -54,6 +54,7 @@ export function GlobalEditorV2({synced}: {synced: boolean}) { interface keepEditorProps { properties: V2Properties; updateProperty: ((key: string, value: any) => void); + providers?: Provider[] | null | undefined; installedProviders?: Provider[] | null | undefined; providerType?: string; type?: string; @@ -64,6 +65,7 @@ function KeepStepEditor({ properties, updateProperty, installedProviders, + providers, providerType, type, }: keepEditorProps) { @@ -86,6 +88,9 @@ function KeepStepEditor({ const installedProviderByType = installedProviders?.filter( (p) => p.type === providerType ); + const isThisProviderNeedsInstallation = providers?.some( + (p) => p.type === providerType && p.config && Object.keys(p.config).length > 0 + ) ?? false; const DynamicIcon = (props: any) => ( <svg @@ -141,12 +146,13 @@ function KeepStepEditor({ error={ providerConfig !== "" && providerConfig !== undefined && + isThisProviderNeedsInstallation && installedProviderByType?.find( (p) => p.details?.name === providerConfig ) === undefined } errorMessage={`${ - providerConfig && + providerConfig && isThisProviderNeedsInstallation && installedProviderByType?.find( (p) => p.details?.name === providerConfig ) === undefined @@ -412,14 +418,16 @@ function WorkflowEditorV2({ export function StepEditorV2({ + providers, installedProviders, setSynced }: { + providers: Provider[] | undefined | null; installedProviders?: Provider[] | undefined | null; setSynced: (sync:boolean) => void; }) { const [formData, setFormData] = useState<{ name?: string; properties?: V2Properties, type?:string }>({}); - const { + const { selectedNode, updateSelectedNodeData, setOpneGlobalEditor, @@ -475,6 +483,7 @@ export function StepEditorV2({ <KeepStepEditor properties={formData.properties} updateProperty={handlePropertyChange} + providers={providers} installedProviders={installedProviders} providerType={providerType} type={formData.type} @@ -503,4 +512,4 @@ export function StepEditorV2({ </button> </EditorLayout> ); -} \ No newline at end of file +} diff --git a/keep-ui/app/workflows/workflow-run-with-alert-modal.tsx b/keep-ui/app/workflows/workflow-run-with-alert-modal.tsx index bbf64d458..47b27321a 100644 --- a/keep-ui/app/workflows/workflow-run-with-alert-modal.tsx +++ b/keep-ui/app/workflows/workflow-run-with-alert-modal.tsx @@ -166,16 +166,17 @@ export default function AlertTriggerModal({ return ( <Modal isOpen={isOpen} onClose={onClose} title="Build Alert Payload"> <form onSubmit={handleSubmit}> - <Card className="mb-4"> - <Text className="mb-2">Fields Defined As Workflow Filters</Text> - {Array.isArray(staticFields) && - staticFields.map((field, index) => ( + {Array.isArray(staticFields) && staticFields.length > 0 && ( + <Card className="mb-4"> + <Text className="mb-2">Fields Defined As Workflow Filters</Text> + {staticFields.map((field, index) => ( <div key={field.key} className="flex gap-2 mb-2"> <TextInput placeholder="Key" value={field.key} disabled /> <TextInput placeholder="Value" value={field.value} disabled /> </div> ))} - </Card> + </Card> + )} <Card className="mb-4"> <Text className="mb-2"> diff --git a/keep-ui/utils/hooks/useWorkflowRun.ts b/keep-ui/utils/hooks/useWorkflowRun.ts index a35ca86bc..3d5890cc5 100644 --- a/keep-ui/utils/hooks/useWorkflowRun.ts +++ b/keep-ui/utils/hooks/useWorkflowRun.ts @@ -2,7 +2,14 @@ import { useState } from "react"; import { useSession } from "next-auth/react"; import { getApiURL } from "utils/apiUrl"; import { useRouter } from "next/navigation"; +import { useProviders } from "./useProviders"; import { Filter, Workflow } from "app/workflows/models"; +import { Provider } from "app/providers/providers"; + +interface ProvidersData { + providers: { [key: string]: { providers: Provider[] } }; + } + export const useWorkflowRun = (workflow: Workflow) => { @@ -15,14 +22,23 @@ export const useWorkflowRun = (workflow: Workflow) => { const [alertFilters, setAlertFilters] = useState<Filter[]>([]); const [alertDependencies, setAlertDependencies] = useState<string[]>([]); + const { data: providersData = { providers: {} } as ProvidersData } = useProviders(); + const providers = providersData.providers; + + const apiUrl = getApiURL(); if (!workflow) { return {}; } - const allProvidersInstalled = workflow?.providers?.every( - (provider) => provider.installed - ); + + const notInstalledProviders = workflow?.providers?.filter((workflowProvider) => + !workflowProvider.installed && Object.values(providers || {}).some(provider => + provider.type === workflowProvider.type && (provider.config && Object.keys(provider.config).length > 0) + ) + ).map(provider => provider.type); + + const allProvidersInstalled = notInstalledProviders.length === 0; // Check if there is a manual trigger const hasManualTrigger = workflow?.triggers?.some( @@ -36,7 +52,7 @@ export const useWorkflowRun = (workflow: Workflow) => { const isWorkflowDisabled = !!workflow?.disabled const getDisabledTooltip = () => { - if (!allProvidersInstalled) return "Not all providers are installed."; + if (!allProvidersInstalled) return `Not all providers are installed: ${notInstalledProviders.join(", ")}`; if (!hasManualTrigger) return "No manual trigger available."; if(isWorkflowDisabled) { return "Workflow is Disabled"; @@ -107,28 +123,25 @@ export const useWorkflowRun = (workflow: Workflow) => { if (!workflow) { return; } - const hasAlertTrigger = workflow?.triggers?.some( - (trigger) => trigger.type === "alert" - ); + const dependencies = extractAlertDependencies(workflow?.workflow_raw); + const hasDependencies = dependencies.length > 0; - // if it needs alert payload, than open the modal - if (hasAlertTrigger) { + // if it has dependencies, open the modal + if (hasDependencies) { + setAlertDependencies(dependencies); // extract the filters // TODO: support more than one trigger - for (const trigger of workflow?.triggers) { - // at least one trigger is alert, o/w hasAlertTrigger was false + for (const trigger of workflow?.triggers || []) { if (trigger.type === "alert") { const staticAlertFilters = trigger.filters || []; setAlertFilters(staticAlertFilters); break; } } - const dependencies = extractAlertDependencies(workflow?.workflow_raw); - setAlertDependencies(dependencies); setIsAlertTriggerModalOpen(true); return; } - // else, manual trigger, just run it + // else, no dependencies, just run it else { runWorkflow({}); } From 690830a7104234fcb71e61b19ab06e036d340d55 Mon Sep 17 00:00:00 2001 From: Shahar Glazner <shaharglazner@gmail.com> Date: Sun, 15 Sep 2024 15:48:32 +0300 Subject: [PATCH 6/9] chore(release): 0.24.5 (#1928) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 82ee85d7e..d62fcacdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "keep" -version = "0.24.4" +version = "0.24.5" description = "Alerting. for developers, by developers." authors = ["Keep Alerting LTD"] readme = "README.md" From 9a5a7eefb0f19e43a4c08dafdd4a9827f39309cc Mon Sep 17 00:00:00 2001 From: Jay Kumar <70096901+35C4n0r@users.noreply.github.com> Date: Sun, 15 Sep 2024 20:06:01 +0530 Subject: [PATCH 7/9] feat: add sumologic provider (#1924) Signed-off-by: 35C4n0r <jaykumar20march@gmail.com> Co-authored-by: Tal <tal@keephq.dev> --- README.md | 2 + docs/mint.json | 1 + .../documentation/sumologic-provider.mdx | 36 ++ docs/providers/overview.mdx | 6 + keep-ui/public/icons/sumologic-icon.png | Bin 0 -> 13535 bytes keep/providers/sumologic_provider/__init__.py | 0 .../connection_template.json | 20 + .../sumologic_provider/sumologic_provider.py | 439 ++++++++++++++++++ 8 files changed, 504 insertions(+) create mode 100644 docs/providers/documentation/sumologic-provider.mdx create mode 100644 keep-ui/public/icons/sumologic-icon.png create mode 100644 keep/providers/sumologic_provider/__init__.py create mode 100644 keep/providers/sumologic_provider/connection_template.json create mode 100644 keep/providers/sumologic_provider/sumologic_provider.py diff --git a/README.md b/README.md index 04db95c11..2a655a673 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,8 @@ Workflow triggers can either be executed manually when an alert is activated or            <img width=32 height=32 src="https://github.com/keephq/keep/blob/main/keep-ui/public/icons/prometheus-icon.png?raw=true"/>            + <img width=32 height=32 src="https://github.com/keephq/keep/blob/main/keep-ui/public/icons/sumologic-icon.png?raw=true"/> +            <img width=32 height=32 src="https://github.com/keephq/keep/blob/main/keep-ui/public/icons/victoriametrics-icon.png?raw=true"/>            <img width=32 height=32 src="https://github.com/keephq/keep/blob/main/keep-ui/public/icons/zabbix-icon.png?raw=true"/> diff --git a/docs/mint.json b/docs/mint.json index 1f874c84e..b7022144e 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -167,6 +167,7 @@ "providers/documentation/squadcast-provider", "providers/documentation/ssh-provider", "providers/documentation/statuscake-provider", + "providers/documentation/sumologic-provider", "providers/documentation/teams-provider", "providers/documentation/telegram-provider", "providers/documentation/template", diff --git a/docs/providers/documentation/sumologic-provider.mdx b/docs/providers/documentation/sumologic-provider.mdx new file mode 100644 index 000000000..6e1be21b2 --- /dev/null +++ b/docs/providers/documentation/sumologic-provider.mdx @@ -0,0 +1,36 @@ +--- +title: "SumoLogic Provider" +sidebarTitle: "SumoLogic Provider" +description: "The SumoLogic provider enables webhook installations for receiving alerts in keep" +--- + +## Overview + +The SumoLogic provider facilitates receiving alerts from Monitors in SumoLogic using a Webhook Connection. + +## Authentication Parameters + +- `sumoLogicAccessId`: API key for authenticating with SumoLogic's API. +- `sumoLogicAccessKey`: API key for authenticating with SumoLogic's API. +- `deployment`: API key for authenticating with SumoLogic's API. + +## Scopes + +- `authenticated`: Mandatory for all operations, ensures the user is authenticated. +- `authorized`: Mandatory for querying incidents, ensures the user has read access. + +## Connecting with the Provider + +1. Follow the instructions [here](https://help.sumologic.com/docs/manage/security/access-keys/) to get your Access Key & Access ID +2. Make sure the user has roles with the following capabilities: + - `manageScheduledViews` + - `manageConnections` + - `manageUsersAndRoles` +3. Find your `deployment` from [here](https://api.sumologic.com/docs/#section/Getting-Started/API-Endpoints), keep will automatically figure out your endpoint. + +## Useful Links + +- [SumoLogic API Documentation](https://api.sumologic.com/docs/#section/Getting-Started) +- [SumoLogic Access_Keys](https://help.sumologic.com/docs/manage/security/access-keys/) +- [SumoLogic Roles Management](https://help.sumologic.com/docs/manage/users-roles/roles/create-manage-roles/) +- [SumoLogic Deployments](https://api.sumologic.com/docs/#section/Getting-Started/API-Endpoints) diff --git a/docs/providers/overview.mdx b/docs/providers/overview.mdx index 36beeb12a..60f754022 100644 --- a/docs/providers/overview.mdx +++ b/docs/providers/overview.mdx @@ -354,6 +354,12 @@ By leveraging Keep Providers, users are able to deeply integrate Keep with the t icon={ <img src="https://img.logo.dev/statuscake.com?token=pk_dfXfZBoKQMGDTIgqu7LvYg" /> } ></Card> +<Card + title="SumoLogic" + href="/providers/documentation/sumologic-provider" + icon={ <img src="https://img.logo.dev/sumologic.com?token=pk_dfXfZBoKQMGDTIgqu7LvYg" /> } +></Card> + <Card title="Teams" href="/providers/documentation/teams-provider" diff --git a/keep-ui/public/icons/sumologic-icon.png b/keep-ui/public/icons/sumologic-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..4a6e02ae22fefa4c8504a7d20f357bc69959fa88 GIT binary patch literal 13535 zcmZ|01yCGK)Gj;=i@Q541Pw01JrLZ2L$Kf$G{IqU2o4Fsf;%KQgy0g~-8Dea#chGT zy#HVIRo%MxR@cm#>YnHHnXWn0r_b~gt*NengGq@A0054XqTJhOS^nRFj`V!4C4N2o zEFhMOZ`A<6ml*({VE}OdTm}6H03Ue(;J_RJMAHC(%r)z`miTi6>U$LhIRNor$!#l1 zd9GpdlsE9SvhcJPwRE?B762a)p8zM12q*6=9bO($0e(>dK6V}+Q68S+21V2V4dCoz zWpCs2{|3zeJKcH)u>MaEp7zex9-bD?uKyQ|;Q!x=XYCc&Ge-D7%xf2CXX_81!1EYD zEL}c)vUc~hwt{%sd)h(lJUv}KM7g=GeQoV+xGY_qxxK94yK;KEaJqi5HLg4!c-BGq zpAKyo8&59_cWdB(?#%6}GXMa7S0y<a9q*;%0G})--J}yTH+8NXz18xAgYaT|nOsAi zUK}0&&l`L%S-E%)cs}rNi^j*Z#xC<9OJ2p@qkf|x(ZP515zQ}%Z{fh?+KAx5d^rKZ zY4bYv8Wd;QdQ3{c+5J%@RQ>^3_iWhAYp%_QUn5)CO+71$n*se+;Dj9mNc0@*sxvEX z^%HuRkHV}xAh7mnE;IGErqn+AF(xB*p`&zg>;}g+wxsClDNqT!no9|>#nff0)Nyq9 ziF?vNi9-pI&jcF$po~ywl3YHsb@adUk3O*#2xV89>iA_pJ|Adg1g8_P2c|j%E~H%T zSf<uFcUsskMPNQ-aE{5xP&>TqM-`%{&@ZXX2Y*ysd!OXh>V7^Jl?CqBcNk}^PQ7;v z%4_~r1f7`r`>gNTs8sUb_1D}VGW^y%>KBc{^^DJU9wn!?r`aprlf7E}`6{q@Ja8}w zk;??)yn~amVJHG@RgbNO$J|^8sZS!_|JmU^cp?ioK3|tAwO6Vod&OB3q%}<lCwujA z`C)J7JarC^#=`mhV8u+=+zv+X5&6t5R<nyD$M&$J_|#thpDRM2NOpWQU_6JzC!<sn z2Apcxnr;lMId<H}d9f%1C%hNd{Ns7ZpN0ch+CyyMt~Y%kSXV<!v0#FG;AAZOkW;3| z%n3ncnSy1W(3IIFs`U$qUnzUQu{7>q=obj^3%(OhPRW|qmFy<#+?6a?w-*$@@zD7F zOfK=$aPWf(dcKK^C>4QB9X$m5+v=!ioDQi03i5}(FU^`*o_GuEAKG}pQNiRu9?0Fa z3F{szT92{%8}Y>cQkRmJcFE-`@b&B0NjS_bEMBp3>gtT6Z(RHD)wV6S`?xN~km&J^ zd!<=LRI-RyjROCf3e>BG=kC2Q_cnAcmt1pFV|@!qqhBIdk|PV`>XtE!*Qt0Evzr4Q zPUhb3TU-^<!K2V?%rjRuB$+sTBwEoQi&z50r2fa_h0{B@Hu9g;H-uv|l#r_fv=~IM zvr=+yL&zkqv=WRyZM?R?u0-L-4n`k=m_MTaBJOzP4D{T&GP|GhI!yUt<I!C(SW18t zjuzI%*H(9(t(@X7h&C{=P)x$!{bKG!sFlNs2`4xlhE~bT-t{%JYMoE=124RORC+ps z<eO@GuFnV&?R$IiQyxqpJCV|j+#m>TwWV3+omTomHejb8w+i=aH4pw^l97cn0~XN( zr3u&=pv~JpcAnp_KdjcN2b^iatNxVloC2VHSK6%&nl17q6sQ{-YplE+IZcmOhjGWo z1NYNqzdmxLfhj5m2`V~(jsfU)cyV%h;Xst*i2?o$t%krgg*Bgh_R(9$h)A)KXaL33 z{oo_wu_1@x<xWF|CMmfClyo5A^flr%q9c+QUGu%vhQ!)kyp-8(I{U&*F>@negL@52 zzZ?&I#bm|&pRfAz-0cmsbV0Kehc%II$-Vzhq6O~e3yEL4>agE^RG(rz!zMCte6jVb z09n_0xQ9xI6AF~H|GKYCF?(D^h}^BzaT^5no8%d?P2s%~9rra0WJg7wa)_f8BMfKH z#}sf$Lly?HPdLS1_9JVLdK$rEY>pfO?C<q5vVsDnZ)9eW%<(Z5(YIJAe<V4)0_*1e z2)7;W&Il+hNMb{VFy!uEt?3{xD{ZUppKhKGHWhYS$1@1sG;5eK8LJ5iIi&G}Uz!uV ziQ<bU|H)v5J%2i10z;%6j)^?(Xb>zNcsk~NQM#@)w7)T~g32l`FVXqs^pU88WrT|q zp`tvvu|g&iVA<(;C*SNg6%@rm%n7Gg8Lkt=TM+v~uD1%V5Io&rmzPG;Gy`vNI2T6N zV_0*>&^4KKU1T|ys}LgN64z7Xk7~~7Ycg-ciz)WVyCc%#K(>$L=E$<`&OBgi?>8}% z05fsm9j5=*H21>_*|>2VqTGb>%bFO&H;WDQRQy__A;~cOu-c{<7SN9@EqEaGEMmW` z#cbri1<>T9C#(_5T(XRyc#=lIn2LEYm8TE^hgDT@MmEmKffaHAAQ4cuYU}x;(H3Qa z+s%_(i;iy&OzW|+B+H42?+PDveA;Q`Tvoy+0M&arB1ZbyOA6DI7#U|2Q58G?@v8zD zQJsu(s5~jLN(l!?ZOY1ppT7t*p;F*WfARd(jifpCdJQJ?zRAZoS5XsV%pYDcPYVVd zsQy;?n6XKdn!CHI2^<#hn3K!I@`Tl(Bk>e#gkgHqF>|)=#B#6t4m(efph!x<&n3O` z`2?ViKd|UA3^v&^v-rjgOJ4%YP`#QT--n&HJC%5|VsHq6+Ksnw{f7jVu|6cgDBvhi zObYgX$;}Pc5F{(rYm*oDD1z0!O{)>(f&V$x`<|*m56iZ-(jLyL$`u#HtJlMKVYbx% zODc4Tv;y8WFvTTGDYEuQ^6q{qcX$&woTNIL6+m~$1us$U=kS_xs7OJlu1TZ?@Al&K zrR!cnAz-_9%S=fm0r*02^2$gBc7q%kDFu`01rTyXq-nEiWbpG+=f2JR1*UqiG$?#9 z!i)M1?XO*1<F(T!I_kyS#Cz{`MpTIi2Du+Lh=`V>2tSrUzwIEihvq8ry}u<-+Z5v_ zemtKT$tgY#M;wt!ik$^}NI4z_uF0=fJel58{*DG!K&4fU<5sD3E3Lj4%1VvS5l_iV z@Il#}{z{y?UoNaFln*%p=(*L4*Y>T8I$a8qw^x31k(S$h(Z!Lr-dj4y3t!d+)>al2 zk7jfb;W_s?mI(o?i?DlMEL>A-mv#c%4tS^=qWZ07<&E{DBkkJH>K7Og)Lt+Hc_N5l zDdzTrc(iIU1fPI6jTQbVS!AkkSL*xf57cGaoZ2`-!@)Azw-m$?m8)*VxsHvbvn?}g z?i5yOC?@RMTQY{28ys)HWSQR@twQwnUek;YDk&9p1fg$?Jm@Pfh<j?56(K?EEvGHa zjeLpc;)PsX%1WsB8!ltW1O=SsRRCRTpT9{2Ay3NGXfk%P$XEtRoh^y)s4ZUc{CoG& zim0IB{CCBCj`zx<5W<Gt6{3lJ^!Kk$<FA)S88AlJGO^K-5ZL*JMqp3KsQ5hW8cJNa zLi?uDd?!2F8a5ZG2QV_oO_OBIh`^2)_ry_+nlC-fvZ$Q}+G0j{9I-JJG2U|;E1F@& z&0yi<GoU8bA|auKysvRNoI*`Yz-L0z2@B&(-LsShUQ6!=k3{g3iL-aC9?@<Y!b^DQ zE@NN(LxGp@O3%73QzTM^PTKx3njwKn!3Mmt;Mwdymt1FeI$F1{lgv!KGQC2RDZL3b ztkO*qA@pJ3*F2bnQ2OvR6{YnQp95vKTh8OVoO5G-2-v{BPM9kG4t=3*FcNUJC^nZ@ z(s<{35Zqq{*T7LZ<l~<>kHSXhLCrRo&GMXD($4Al<VkHGLRHoORe!~M+r_2ju21Ti zysWO~6qPUr46tE`25{XXl@iyMWE>BiQ0;V(0m^T;Z2h#yRwf?|FxOxw<psw<(ICAy zyx%UvNar@i0*G();r)l_aaB&D2dA@9G2aul$x&GO7(Y9JnS|vV@Rkw~9~t8PmL?Mm z0zwXh4y%4Yd=px6-5dYe{?RG2*odO?L`e4r?0)(@Cw&nc9z+V4u+dO`oMvk>!UTft zP?uOegSo1BWTHt#t}W+<#-6l++-_y*fg@0n0tdAhDdAy1|Cn{F<Qy5T_hd=?U5K`+ zU+)V|ef&H<CMYCZ^L&mV76chmhrH)No8;BtNeEaWYV+G_^V-HT`*pU3#ob~%Jxh;E zXWHJVrO{8h4DC|S+4o;Su$jSS7LSnVk55}OB`{MBe+2xd+TcWCE44H|ntbSx9%l`` z5bZzDdvs{nR5NaD+r%MTuc4Iz+js|)vBvJc!LzX@?Rjs2l#Drn7CqG~QetKPg8G^# z>td}S$n)554be6K2AN98xr0ruiLjgEO|6e_i^(VrADV}l_=soEevfs0yY{Z>D5A~n zR<twe)sNn|Sasi9st*Q;*pjx*d@L7AI)wt1s#7S7cYgQU>a61kI<j$G8Mm|<fK~YU zBiiP%EzQm2u+*oK8Z?2`R3kG@+09WQH9C;^VI+smH$NObPraM{6#uGM2ozs~8qkY* zNu7X#N%wBjidO7|69bPq;||xUI;n><#Dhs&d=f?>63Sbd<Zd>Bx>DX&X=-d~C|VW+ zh};fP`(qjZhbafqUqDy)$=|`U3l}eW#U_x>UZ4L{dZ|Hcb7tJ}#rJ(rChfP}Xk8%^ z5tx(XS!wu5BSX$4dJPAU<%JFFqe2Ne9w$r+Quj2r-|u!{{(cuZus2rt5$2tD?}7xw z$2&Nk-d(!yQu{1y-gcsm{j1>N{*qIlwH*TW8S<9>t*IElT8y%Gt2Ovk&e0owp#z}B z(H|Mls@)9exBq%d9)IAAzo!kD*-Q|<pD2;Iz)0{HkhnA_N5;hmKI<eQVe=J%HjK!< zL?1?Mx7hmWJa+J0U}?Z^%ommR`42yB1!Xd>yRw3Io9vA7m9R<ZX_-{rs!A1}0`bW0 zup1Wbv>Bl&yivjU*>A_GuRsljB<~QJHHSeL(=l96xRVR?k+>2ET6hv?`~8ylylAu@ zM}Y9_aJxCrw>(NftojEH`@0$q;e(-J!~|2pg}f0Tvsz}S2ZB2bt%rcky=BxB(>xs= zJ<lQ?gfe5jJ`qx5CgNqE5*#r>-^ofTa-n1Hgr5QV@@BfiBRVGpqgO@6Lem0Vj_mco z)^^b-6?9lDmUhrj+jMfNFMa_tZTh(rwE#`QrlH6;ia{n(RbBa!0kEzV`{%s37UdD^ zQ|E940}bdzAJZVZ0u&_a$I3l5T3FbX@17<`#`wRL+M?G!?BclwNxkM2@cT?bY4A$4 z9R%>u-iQds<MW6mNCz0dH|#>f>ZMRk1?<i*a-0#f4SVa}J=5zqU3+1?()2wbNG(St zSTlcsv*e<9+D=e@Cs^S))1gVoP9cB+*TLebRh<Xb_7FwAkVy%PT<o%XajzIf=wf6n zJG{UteIDQ3oRB<+dY#vR3J^r<xF0~_e(XwEZ{O`K_F6_#!MDM|MjES`M2eCYkix6j z?GzUO+krSNiNml`U%l=e%Bn6*^v6GbX|lL>NeHvjDJ#~3vPkX|SQTSx6j*7gW%Ub2 z=Q5FtY=$A@1<An$E5m!tY?~N41vu+1wJ0cMDQ3WGqDMSkg+Jm0Po>uHK7XTjJr#Yp zw1xTpVb?HUau_96`tB}YZek)X1ztHF^!E5HLcMBwsZATuRVU@O#ta?n$Z}K3D~F)( z{4iJ(QthnM2$XBcfKpAznn3Y0I=vhCo@zTJEbh#yJ{xc(!Q)ANg9e*9l94rVu+JBI z>BfXFX68H2HgOW#7xZ7={w-1e{clU?$Gc`u+<r$A%mFa9)tL)zKF{N9-q#*>?Brnb z$AEvZ&^+-c_PQOaP}c6wPJrm_p<OHDmHt?4Z3MF01cv~c=_{%6HIF|IS%~zTs>PrQ z2$KLw9C6C*XPR=|kX*wUg>nCz7_$fc1Tat!wfgxqP7<1}=`w<a-GDu>d~(+^pYy=I z_rML16!5;oe${!A-E^PfD0#ZepZ)P40d@cK4~ppSns$c*J*7&T?{+H<AN$p`e-%wN z?bLmH7@qj7E+@|3dSL;eE{uq6*QH$Be5JUIG<Na89YIaRR?ARe?a_ZBzRj(deHGt9 zX)0+Gf)ip473-%IN^H9rDCkZI*rHoIZ|rCMK?;gxPE^%?41-*}FJB@$P)j7s<>fi6 zOri6-_KUB2OWGrYSpg|P=bWPryFn~(Kkb_&z-PzLw%J`;56UYVot~F=B3c!*df41_ zrMl`qCv4oJS6ZQ=)dUOPiPA`bsd!I4;fn7tr<{$2-s2It-1oR)o_X-MgIqXl-BmJ0 z2*sj7>LGKUL+a5QcE8_X3|p<AY?ulT4lbbmwD5v9x-)lEGo~P-DzeY*WMNC3t7h78 zMWte0wyuMSo@pcKX`U>&+6@zaWj)*i0zz;Ah}v{s@CHRta0#{7B_a?pNkn8<1PN&{ z4{1P0M)%>?-V(sxIy!Oa*8405>(vN!=lAr%U=yPwB_3^w+646Sj9r?0UdaSt<EkW_ zLTd$;wDa4$=&2DiU;8QrVi%A)2fW|^;+>FMaJbjwkB;OBmiw{*6@^D-vn=h)u$8Qw z?g1Bl^Jc&IX^++;qruzDOgCYei2w!?*j4&7E2_9dfB|pVCOtN8uHNe+Od<w!^N<PD z$E#G|2n#T(8&t#pVW<U$S~%xMxgHs(!xixTI54nK5w<FD9gVmxun=G#=)4sWQbiVo zmZ4Hdvi0g(lA&-}5fFSNFONWe4BhgtTO+5WWS{|~`tzSC`q%))-AWP(<%jBP?p>A- zOWq#|OCLwM=cR58ZG%qOm#0J{9I{j)6ric@c6xblFqxQ$9s>&20*NJhgk=aufv~Qs z2yi=~-~vxbFd<!z9C!!P^IMeIS+juBv_kH0gZ%f`TJXVk0t9ceifJ8kK{K*4t|M$O zEQDocT{psJUZ5EBFU-T3c^Zqtanb*TYD<UZe&3?N^ZbVQ+md9awpS9v$dY}`x95&d zcyFnOE$mSm-Py8WDMHYp;!0--%@Re=ytaXnAS_HWMkB!C5WWUC>jafm<n=uHV4M!j zNX(D@;nOiB&|2@cx<8nIH|1pUehNl@RmN{rk55@1QBE%~^4)mRfSF$~+bU(H<!PV& z<^wfKr`BsLv|npl$ZRT_Q|cyl8}*$wIoC-qm>mm<et&Ff+aQt>KZ!>8F6uXVqx|`P ztZfg5gzEY*(0hB*Y_r{ycXZ%XJ;D)zhsIBq%Z=#djivV4tCqAyENJXjeOZ%z-sPn_ zk?nqe5^@l&H_mHL5s|75NFcdQ3vmjWPYdUp!Cjlxyx~o4I}sXYrv;qGKhQgo%VLOm z>oGAP&3;Nc!FN#_t2hS~GyHWS@&G9Ghd}dALZ<1HK3(95C7#IRWs17b6R6_)bwRjw zk0KbNr5hYo9`*U9F_C5>>r9tXZ0vLXV>NL9j~RT`MJS0D9UBk~|K?AUCmZ>j+XL#k zE<+q|A>rxChjEs87%1qwg!MGeZu;>+Y|ZU<fl-b?7hgfTPZSAReZuMU=QnfjyuSKx z1O@G!J3h4*BQVyuZ*bRM{Jd1!#n-KdD1n#GB{_ljSn62cS5V~Lzkv?`X0SAg<{x6s zrqX_rc{3Fa@w3SUvFoTvx{n(c3S5U(7Owa;`)Q-VN*jf-P>W;L<EloG`}R${hlwR$ zqMp+T=GAav3R;o)l7FU;b!o}LPm=DFvH}TQ>W!FpW)44i=iD^u-q|2dX1V>3Y7XNA zb;Ku}VRUr;bI!j`ew#fUWn^|-7nvD~$W0>yUK19Qu?m%Hid3ma5sCuWe3x*k*dlU5 zvoi;sw(=7$Cr_{SqS5QHk(uQ&|LKrgDL-FyVNDtxH0oa9RRj}|9!ZR(hEkS0sn0tb zSbmvqJ$%2dT$-;AvJPHtwTJJjEi<0EIm>&LfAxE03hM(%c&gB=rNMyW@<LUcb&x$0 zgy1W~D`VVUQbuVaJHutz%1q}*XD}KqdA}Q6l1k8Nq1C5A`E40)MM4HA8aRx+l)V@_ zH`HYvSEGf59;}e&enjw6m(ef&Am9dcbg?2it(XRW31PrQri!Y*d6{vkf1mkd(5B)~ zfjcZhXY%sp#0Z}G;>?z})MpW;w}R2XP(3kErIFG?ri(k1M_fV@u4nR5?kA8j<*Xb= zDjf<Y)Z_0mU6ybS#yCZ~OZzr^79ncJ<wK_-Vd*`t3(di-)yDmOoDmqRJhWT#Vr&z> zgip;IM=+pTfRqF(pWaoj`Ss0|U7bU@2ny@KXEImK-Zo8AwIaDVn{K%+`Me4I@4fiN zG|pz%jUN~E9TJXEccsWg7_2Jw-_VZh#?v$c6_GbU>v#~cbr-Fi91o_zbQpDTE@m~o zPOC#fI#R}OQhc^tpEY(1Z_w{e6f7*nzOGFb7ub(BNxQljdi7RJG&P2i>mRF(GELMY z#{~O{ID1dnDAgJDo`>ezf_;vwJ*^8Ip9DIN&(!Oq+`>`wx@sgyg6IA{t?f28U^bTf z(&U`CiZJEE+_!i4XCR8_xX_LrZEo8z=fhV<R+wLdv(=HjZw_s2t-JEyHl{zcZ`vgQ z52zz7tX)k)_i<?K*04}-dGiAeh0M*{ALZ$k>v|i_DqDLU@cI?#s*Q$7qwA7sXp~L~ zme9^d!uy>G{|~toEr`{{<|E1D?IVUF+=V|)$DH$_iOOzj62+#>o@nkf%dx!r@tdqU zs#u?g>yB)^)7~rT9=pTFm~*Ei<|-}L{iyZYsNOVx@B2nZBrUmo?ACO`Jo>6K3NlmY zj=R(MMYcFN-Cjj4FwV|?7q%j22+621EWybbn=@Cp=2y^94dwUNNaj7-vM!11a7S0n zc4!=sniXVWC6S_UgHd2yS9VE6!CVpmmSe9m{`ZY7r8<zjQXk;_R1G?CO<}1!TY|eZ zwEaePAw1>Cz-Gyk9kj3z6`l3DoR9y@thMoIqe>hEH?jNX`yMRq>v==glFNMGSF1D| zW+DpQ)|&%D>BLRLM1Xb(I>~WtCWge-gGG{%ax}^0x^Ftg)vrn0F%jec#4ld$le!|S zY}40dBU;Uhk7R1HdgqJ3lsh!n+ZSe8pnXT`Ls|R+i!~SAM-Vz_{DIGHX#v!0u|l@6 zP*iLvv}DnNR(pLEzVg*W4|(V`-k<=VID9R}qpW-h5>~6yri}TE!4m|j-fFkX{spVg zJsQ{Z1$<7e1&=W`KuAA;#)2B#+yLwMcqoq8Isw_Q{6flbfBMPPE#gU;muk#Pf9ESh z4VQ|@)0LL~I04p1ax2q6lgu>9__$oSFadq_I#zN|X)~C#l<P>5eO0SDZTPb^jb+5& z3%wdcednSdlf<JV-1vHEt%?WKjF6cqSzTmdR(|sIIE6Zja8UJb%#{B<eD{}DW81&c zKJG(ig3d&~1S}zXU8oF3VyMi9BUq)p6375Quk#$|X##JzX@h!y_vodWzwiHdyX%+v zVtFU1-EFpws;OBZJVIKnQxnag6NF)hZL@qZd-nS^o%_f(ZPSO<-(7imgM)*UlapnG zm!Ik0dBGwE{Z4+1<$|ghT{U@=)6EP%%^sqlR}SoaSKt+UgQVUWS0Z`L3$HS)c}Bl- zyf&+2rlsiG9W_=^VE*|udFAD@P`i1ewq|%vn&DHDW)6%|`zf0r@1GP4(MkaruffEN zyTfJ|VLkEhB>8HQ|3)1EFV9>ZyH`##{o3KjIo@_17-s(W)28s>+jyejN92F7*K>R7 z9(EH0xDO{bSvPqxLAPyYIF~fSB1KvT>B<@O<&Vg}Pd3k_vd;4NvTL)jCh&P{Jy<y` zoxo=B^M$*dhl47YR*zrqiSy=_A_R#^;@i|JO85K#DM>`%R&Rr+0w-;TojUKz((IYi zc)-zxhS>E$L3!wF9#R>4z3<XMXqxLG&Fz>_5MmJVaQ?k1L0Bj9C7P<h+c&@7+s(#L zy5UuU2U{`jJWB8)R+nyIzQ?e#9(`#wJ#>F!MydAe*hrc<0nq2l;x`r~h^ME_u>e?y zl-2BkhS{ZyMi8p`Z(?n>B3)HubI9k@KPpXG7vl(@y@(p(*+6W$ue;4KzhSENS|=+c zH>TlsBTCRJr(^8qD(v<yoXkJc^CLc<T0L_ZW6U-?O1{^^`es|mCgp;x;(M@**x_u@ z<2kb4&%RS-)gno3RY56FF~>jB7&7H)l>O2qW{CU0e&cul;yOCSx@8)MGS2#}tl(5X z7cH-E`u;6ThUkfs%G(K=BZEQRxwH`}S7n3km@h4S2bS80+s}To<e=kXW)<yfvcFC2 zv^tppYehw4WSu|gA}&^Mj)XWuqqbQemxRUxW*t-2W?~8v<!?OvA!L%wx@zw*bNQk! zBz;!;C7*oFi4MU7Y#_$F17l5FYOmveR*40+5_fJL?N4Uok~h$v1X*W_3^r_^S@3D) zTIm<O80k&pQATtEu=n-g<2qupW&P^<TVr+2WGQl*D-jJXx48b2E-PuWJ4w@bCw`p4 z*!U=~QgEq3S#?DsHV!_t{psFkukARE1y`;HI_Z!~K!pO<v_PMI2`R<Nef!5kVWWFI zn@U&E9}Y*H7jAcf9G+K&ky<WA14X$g5yDsabVjHLqjR?lQ9(OYxZa~w>g0}(xbJL{ z_ohP*kA-^#d0)IoxnNv$4!TR;?zs9+CV9jvU}P?9XYs4cmGRDFIMUa-`8Nulj;tjJ z601ekDFPGQtT4O(cfO;ITc9fl`Sx0w7Mr3?tktDrIGe2|c6Jya0GZ>HF9@Um+5YI_ zf9|0!+1u1P0^|ddqxhHU$&9%IF?d5>&X|}R)au@Srwu|mH@7AE;e)IWK$&t+m@)x9 zSQc7ic}Pr5s~#<{`08^(@KYc^?`N-U_~mv^jdJxW4@eDose_F)>(2nuwQQK{_GUOV zgWt0RJ!tor1$+Ff==O;ccAT2Hp9mXCUF0)Fx895M9bSUeonE2fck<+;Tb*`)n^2Gp zxIdMCU8MfbijkWF^*X{+C?3UDYRElDV}}8w><e}PX>}n)n4y6{708j*hCs_~6sz;g z-Z6G-LN`FVVEPd9Vg31FUo&#v=Q!yHphsj^H}FM6E1>p62rK`~QgRj^km^x{{uZZg zJIBWtO36F1LnC70UF8c&NB~P{-eab|<l~V?P6Q8KhtCw7dwNUY@?2SE+)s=~R4B8i znOpQ3|H?b;*ck=W+wNC6@NPe60g!T3z&Pd|hd7*94JSRmXRz6@^SAbyX(lripSced zdA!UcBKeqFE`mNlyx4(S4sY1#qa@<g;fVF_w1ATyEnpT(6ZH&px}gA6oy1>-pu&x~ z!q?GgRqtw+h8eYWqC?_M^f>ELkC9~|>B(XVe&<UF_@;J5BnY_%pZz`I=I=yw+1V$O zVKm4DbH+!fC;k(YgvsMlIgmIDo7`}8pTq=Z>9yE~;rO5B@j#Scxah)z?%z1XZ~U}} zOSIbzcJGbSv$V;#TZ8*aq<$SH$8-c7j^4z_2|j<D&lfCd-R9EA#r{>V$f*Rse3!~T zM7$lN$;3xc9YFQ-wrf{K5`J~n<N~RQr2c(<OGJBBu1{Cz9z;nq|4Z30dP0y{Gngb) zc(Zq#8$MJOIzevvyW$enkWVDvcVrncj`mb07!_USTSSP(B%O~RLeh(m70F3;hL8+8 zwEkxD>1Nfd<cA)47@ovs;tSeOrxY%JgEEnMUm4Js`St+br~|fqpvvcw{+qwWY2#JO z1U{v>fy3Pq!xG<xgK@a%+=Mjs)A>w-iO(Hu+s?VdG?Z(-=Cw=nzL42tRRwqRZ9tIQ z+XZD!48Dta`ub`4U~{G}MWUd)d_2T$#(KB_HV!BVZW8n`__4|V=s|@We$XH9&sZdn z?c4`fB_&FDkswZeTht)V+e}hwZJlq99XunJTPA*A+?4(4AA$LsAKZj$0E;3zM28B8 z%GiHDhHCxx0I$MKoj93xNKS2|_;9~?eeLPbc*oTj?F653G&A4Ru6I^GID9v+$vZ&- z!>7R)PiHgJW7M=pO_qnPv#wc=U~4w5>$VOl$JfvQ;Zl(|j_RsfPnM43=E@ljIctN; z*$F#1l6Dv!O|R!|*WAQngFBI<yZPld_*A7pp~&8g`*a;o^O@||npLlHLve?}@aW3L z-)w;wnNp;YB8@yZ4J0&jpS!G)F}cIIl1@MFVvm|g{p+;m_TH>%pcMD4l|-^0Ta}yh zA&L;bh;?Z_`$%{G1VrkV{Zy^yn<R<;X?M57*(5$mvVPy^V)j^m^LVB=_$V2k5DJPw z+x8jniyrs?8}Cdt*w-fWbcubWd!IREW<8)$h>sHZQRO1!qC>Rt1!@H6>|1gL$)e)W zb-~TzgRk6(c@1YUgRXE0Qun`q_UjLQsjz^X-@=l1L-M#u5X~8I&(#~mMrT3b*)Zp~ zpdite+~bvHmn(cK_n#=3lV{9DYtF$G$jtHHw@WNEXu#jW`~=1@onZ1(zL6rbHP^vE z`U;bx2?ky5!iB=n(sW!?kE@wXvzxnn;m<EaF-Qokumyr`CHL&eb<=O;P~fqTb()J6 zWu0<H3Ekz$?SYHBDMaEEK|n!rF~gRQgkso2exuVVls(YXnLQXXcH<%4530DmpZ^(s zl<|h&{E|psb}m%Hi5%C3f(98iFmSb)gTRhTZG{%p)5DY6MrNs!=0<3j5{sN1in~&^ z4;}G0{-LQCibYEJbf^9t!&Jp;qz@{Q8kkuX5=M>Vky9amC23H2Hu~P|f%vo;nFEVr zz$uy3py>nv?9{KH{U-{xz7GMUH;goKf(_j6gj{~2z=fEpK&)5V%tW}>C_+geXqVfq z;X3TWzJxSksG2Ei%$QC~Q597WDS44IRGNIrMR9+@@2U>T(}IV7H2r9r`I(C9uxy*? zyp~M(As2@kog5@RN4^g>EuN4|N{keJcRHElvdS}IJKyN3x)I=J$BX-=x2E&_-%J|s z^PGF9m16@_N3}AqeYh%<o#x@;Ee+b_3~D>QfpJklG5*3gS(Li)b#q9x&a%ELxyCP~ zj3TYaBB29#d4dDanG?N?pXbea*<yVu^|*~u1%9wLAs!#{Q;tt(W2P0ZSIN@R7AO&4 zoS!nmKQRUk`ZVhg0=Rx8)l56R`kqKgSZZ@+8t&~r)R2K*VaS@0|9w@a%R$6TXheZz z%4AY9@#%xOcH?!$U(`m9A^GKWqUN|ie5euNqXwb}ytENVD{^JmF?eNWU8sr=k{pCm zUW>|zpU9Tp$69U7F@UUWzHDH6Xt8z!#Z<sc6}eNVVDfwnELi<Doy(HbwfvZjj)o?^ zvPVR7S2=NvapZ>PppK;;_A02JXa_^5t$UnyL2>b&w7R>WS#u|4E&9=?#a%-aQwAN} zwBxaoW0Wm<xk4P(bag{}9xDrzo$5(OH83r}U!c?9awIUtfT#nuwYA68G$g;{d48W? zk!#5h;JN#j=!w!x0A?FGOwNof-sVovW6W32LmHRoZ6hqHtSRFT3v7H&5TtZyefR_k zcStpq5zv^|f556zhU$MDa2rZCPG;(lD=(FPyEnzi`N8C~80$&#pN<BjIM&cBYo!ws zWo6A^s~2yUk<ou8G(>$;zQNl67Q4?^!k<aZ{4J0q;eGjl7H|Mae@e%Hft6(kr~2KZ zLz2uJc)}o^JbrJq!j!O9Hv;tf$t=Z5abUUW;c_5)$azLRM*2)A+O4F5FHDI!7erX_ z(s81Ocg&g>L_Du7OsR!(kXfs4r2fny_lrdS`UR%5}&FJl)ZhKXwG=1ZA_%}#wK z9GxF_8Q1yaFFV(z^rgO7X<US*rlm^csZLM8Ouohoe9KD@`J%5sIhIqGs+iH$$0yvz zt}X*srhv6jP7Io~#E_A8vWZ#iw@ymp8XHwNbbKkaXYqA931H@S%1y*}((Z06uwp7C z1VTZ1tv1EwNCI(>&TF<CN=Qf`dzNv=iI40IJh7V1q}uIyN9FTL&X~&j-T)z?6YrP} zE6q#%!DVH&BBd@J5Ys?nyL@3XgTCa(@M}}D5x~F00KPnP-ScV9e-2t%x&McjEi+M~ z7$K-}X^P!F(w(v?Vk@JqiYK%&mSW>#{DI0G%E}gkvYTu(_UJ9G)%${qWwPD6t6d0w zl=N1GzxdL=UZqeFio^rd2S}NuGcA27Lozu)KZ`}00jIf{6ANgX583rO@-l)s(9N+L z{`{OJj*)G#M{O$zsK5T!!X*~lZI_Ml#&$Ytw@}qG$N6~OID?a`W}}%X_?3MIDS3V6 zRY51~FGF>kPLWcm$FI6D&1J3+woYf+aGjfVw4<D|poR-N%3#8`HfGeaEAi0u{-sot z#lTN)Kca-CTQxG(N`0lZxhEi1l;gtNmETY|6&o0mc(LD;BRiVM5F}dsAr{5>5FOuk z;G%IzCv5<VY(A(sEmwTr8}(8P+fM7i!GfrjYvG4&E~ZysY^(XWI(e&Yo2C_b?ArtU zGMm_nDdfq2Ji_{QRCn&z#w`aF9(imaK2TuIeV4s4VQu#OBk~wI9*;SBaTJfMTM^x0 zaSvVN!BtR@?7Kg6#|!O0CQmql$H%m^w2X|b_qaDdYy6bq()V68ieKs3zBFeMrGD}- z$+=*%FIgW`XS867D1<!r{AgNp_xt&U|HkVjEa|4x&_(L3%Iy9d^Z1n25?U928a;ai zTIOgRaz#iGVn7T}9Ke{6g6^=hXU}@H@6+UT-*nu(hBoNCt=(Xan#E*Y%R;PQTAy!z z;MCw<4d8nW@Vw8l-sgTAnX;Y4ex>&N28={vS$lD?>J=Ywc-UdbgnrD^DT^OL80uda z9t_F~+T>RkyBgeZ?i4hqvV1NURF;{Wam#N4+jn#OSjL0;`86tMB+8|kq$QNX6{Em! zh`k;N%$~Nq;N`<H(xKtS0e@jRs3IB^Y(2dGs3oZUX<y1r6y+}_hDNNn*tin~Ql@t{ z`8aQo$MfUKe&KPx#g(olUc>HxSvgkp(X~@l`onP~;`sAn6$(}#OPxPI=BqHC&k*T% zU1Z#1o6A!EH*6&|P8CgprzyWaQ5~s;_IOD=@b05Y`c5STUbm;1RhKiBX1qJD2n&97 zD$Orb5zfDM84bm&(D-t1NAI(492C^_STf9^4$x;JFO#{mwRCXbsY>jnOM3jW87x(( z`wr97uq)yG-iw9L+FOL0iZ4?@^Son}yKP@dBhV^CddsQBPA(=`{<KqDy;nQ?I%M>w z-MmL)e;__dI9~q;rNKua<RvPH5kB)6(tK@=UGN*M;qqVql9Q-@t{){^hqIi@{5#ur zN=o@~y?5c(;z{WJN!!A<dDNvga}icrJ?*QzpMBW!$=gHr@B%bBZ`N{}KTh|!z;Ak2 zKHGlm_V4em7AjjFQ|XV+?RjvlE@nM+CYZC&zinIxV?rE791zPeFa3OVc*YHDonT5* z)VjLuLb^J#L8f5ioY;BFZqD+u;EUdvYV^w0-mkyBFeG{24Qc@3jb`2PAuI7O#}_#H z4n(92P({ygjJk;Koe#I6&)9M;#oo{=z7^mA9B{&c>HvpiD7kC<BA%+6V8GMcmATg; zz>QP)MULO(j9t#JUl;$f!|tvUABQ41>OdRN)chhv63$#a3ss0Lsb7qtycvb*fwuNn z@~us<`kylWoYw;H`iTsN&cEh)nWO8;m<|#fNNMY#kS7;0Xc2e^I~EA&#$vLyMryR- z{j%}dQBi;F=YCha3IqqM@E7vE(ju!jN`&TvPdm-cf}RINnHZUzmi!ti<r~*lG_&!B zMo0g2g49i;aX$+c=>VpQW!KBeo3uy%PYA*K*^Id0D0~7WS+GFfC?UdzknLSE^yu8W z2l<s8JwwcZJFx+g8*#Ydub{n}a=`J%REz$tJF5mihv>I1Z~vyRO(yzGUL-!zIuQ_f z$JHpE<lfP;(Z@^q^!X(DMG!c<*wUs-5~Yw!sJ56hQGv7XhTA3ka-`|;?mBGJ#r@WO zK_&g@_^(T)N4w;X@t?$jx!d1@+TpKCFR$1vu8X=vz%m)dR6<^wrs)x>W~z&q*%VL} zFq6&DTaSuHusv7MT^<rl^GCPN%Q-!yY5jvN`%glsvR@^??o9a(z@;wJE$RHaNTi_j zmjtNxJaQeK-#mnDN&Zk1mZ%ZAvj34-){Jqz@ojx5G7~#)Mu<0WLwAPbKPWkxLxx6s zB$SU|=C1YL(ra6Vr5j@)?ta>s1(~&Jxs)`im`oedmcD61xUqZgWC@+sTWde|nOXMm zL><KTmV&d1a{3;Tn^xDtdef2+vOqF*6BDJ`4S(Wd@Y{0Sf>S>KJzp3V_)5B>kYIeI zF0=U>IlnZb2X;F>_bto!x*);u;z&C_pv%6W+B3K4155C3?sl~-#C)6e=@cz1;OxKr zQo<Vr=KK6mW~0I~EL8!~_Sfff!YdU<Tv!>)CD?^pAae}UVvc#0wWKTsB&;}klDWXn zPlEx0_DJp<VUrBX<ncPcw;-<~u(95Q*tAHMRZ;ZR1zx)fM)ymreVLJ<_r}yMXOQVy z#rNRor1%_lF#uQ;ZuB1N<X?SdM@{f~(9-ao>bWs%JIceOdDlR>_CiV11>(G!&~%_t zb$6w!Z^|!fdc%f=RJ!wEh*Aw1r)J^{I{RL?UR1g}seN6CIjZN;B6je7IOmIw=eHd( zm(T{yf2YS%{HI~_*h%ydw(6zts$wfRbCd3)boKo4?n$v9NEu1L;TKaV3&@w_OJn4d zgQgttJ<N4xMPO~(HDNNce=m?Yd*vUa?c4wOpxwZ-9<ikEyheLXD!=>83V=<#n2&y{ zJy8n{^4mK~E;_C)ez6R;+jYKarTXeVga7$65o({Sd1XUtRzga>p+=^n1U@CMrDgb_ z*joq`l)~!sH66C>aj>|we$ui?`s(}f@(1RRyn)3IoYO&5IDhR!k8Lx2cTY0D`=`eN zR#5);DsOa+m6nvvLwTxsi^}v_$S?BMFp&5#bMi3?@yE$C8iagd_hyRM&##6vRqD|> z)1IoMLD<q4m_&b$*zidkysl#6NNDrzDoA%IBwi<}=tylE`(zZ#Wb_>4DtbU&HD?aF z!nrxztLG~PSY`qx?Up8*6Q*5)`A>e~*!bliO)CYeAM7u2`weI=_Kd@Zzsft>_TP=S z+9tg%R-BH1kN<b502$ze5QaqMvdg?sYgum3hnHPSbw({oRt8Wv)J1xBV4Mb;r|^M{ z%soZTaf&`AW{0TAGoRqGQbn9hU*g6@`I_cC3O%zD_Hv@^QWw;A;E@wvlaD`TRAy9+ zlAnAE+-KX<3GO`p{DUo%!<hEXW}dkW7t(NXl^)Z{_86F1J{;qjg<$&DaBKMR9@&xE zAo-abp)w_fciEYIOUU`-F(&grg2w+aHU7IuPo-{;`Ih?d7?b^fwEaitu(Kar@AzlW zEgpC??pnl@5#!C>VVsX9aU6ch9hZiN^Z#kx{qoe@7tUdVfc_5TdP(-iDF4qJpOa(> zQ~2Izy>xcG#i>2`^~Lv1d?gOnF51Zu65>^y4Cmd<xh%LN$8meXVsg4>GEHiz4V-H3 zdfrE2&v0_6OzYz10KZ>z%KN&C^4)p_Z1{?if0_UB_lIk;8<-VohM(tY>`g$fsab0) z;G=c<fXpeZqO`)2*|;Wyxcq4|KQCQ!6u=f>3J2X;@o5j#8llU#SL|1&R~_2U=pN<& zG}}SRolt){{q=?4n{LUSeHa|t*6tt2J==Y9`YpNaKGinoasYu>dHVXE7msZHnVAD9 M$*ar#lr<0eUm~~i&Hw-a literal 0 HcmV?d00001 diff --git a/keep/providers/sumologic_provider/__init__.py b/keep/providers/sumologic_provider/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/keep/providers/sumologic_provider/connection_template.json b/keep/providers/sumologic_provider/connection_template.json new file mode 100644 index 000000000..639d40c08 --- /dev/null +++ b/keep/providers/sumologic_provider/connection_template.json @@ -0,0 +1,20 @@ +{ + "name": "{{Name}}", + "description": "{{Description}}", + "monitorType": "{{MonitorType}}", + "query": "{{Query}}", + "queryURL": "{{QueryURL}}", + "resultsJson": "{{ResultsJson}}", + "numQueryResults": "{{NumQueryResults}}", + "id": "{{Id}}", + "detectionMethod": "{{DetectionMethod}}", + "triggerType": "{{TriggerType}}", + "triggerTimeRange": "{{TriggerTimeRange}}", + "triggerTime": "{{TriggerTime}}", + "triggerCondition": "{{TriggerCondition}}", + "triggerValue": "{{TriggerValue}}", + "triggerTimeStart": "{{TriggerTimeStart}}", + "triggerTimeEnd": "{{TriggerTimeEnd}}", + "sourceURL": "{{SourceURL}}", + "alertResponseUrl": "{{AlertResponseUrl}}" +} diff --git a/keep/providers/sumologic_provider/sumologic_provider.py b/keep/providers/sumologic_provider/sumologic_provider.py new file mode 100644 index 000000000..802c62814 --- /dev/null +++ b/keep/providers/sumologic_provider/sumologic_provider.py @@ -0,0 +1,439 @@ +""" +SumoLogic Provider is a class that allows to install webhooks in SumoLogic. +""" + +import dataclasses +from datetime import datetime +from pathlib import Path +from typing import List, Optional +from urllib.parse import urlencode, urljoin, urlparse + +import pydantic +import requests + +from keep.api.models.alert import AlertDto, AlertSeverity, AlertStatus +from keep.contextmanager.contextmanager import ContextManager +from keep.providers.base.base_provider import BaseProvider +from keep.providers.models.provider_config import ProviderConfig, ProviderScope + + +class ResourceAlreadyExists(Exception): + def __init__(self, *args): + super().__init__(*args) + + +@pydantic.dataclasses.dataclass +class SumologicProviderAuthConfig: + """ + SumoLogic authentication configuration. + """ + + sumoAccessId: str = dataclasses.field( + metadata={ + "required": True, + "description": "SumoLogic Access ID", + "hint": "Your AccessID", + }, + ) + sumoAccessKey: str = dataclasses.field( + metadata={ + "required": True, + "description": "SumoLogic Access Key", + "hint": "SumoLogic Access Key ", + "sensitive": True, + }, + ) + + deployment: str = dataclasses.field( + metadata={ + "required": True, + "description": "Deployment Region", + "hint": "Your deployment Region: AU | CA | DE | EU | FED | IN | JP | KR | US1 | US2", + }, + ) + + +class SumologicProvider(BaseProvider): + """Install Webhooks and receive alerts from SumoLogic.""" + + PROVIDER_DISPLAY_NAME = "SumoLogic" + + PROVIDER_SCOPES = [ + ProviderScope( + name="authenticated", + description="User is Authorized", + mandatory=True, + mandatory_for_webhook=True, + alias="Rules Reader", + ), + ProviderScope( + name="authorized", + description="Required privileges", + mandatory=True, + mandatory_for_webhook=True, + alias="Rules Reader", + ), + ] + + def __init__( + self, context_manager: ContextManager, provider_id: str, config: ProviderConfig + ): + super().__init__(context_manager, provider_id, config) + + def dispose(self): + """ + Dispose the provider. + """ + pass + + def validate_config(self): + """ + Validates required configuration for SumoLogic provider. + + """ + self.authentication_config = SumologicProviderAuthConfig( + **self.config.authentication + ) + + def __get_headers(self): + return { + "Content-Type": "application/json", + "Accept": "application/json", + } + + def __get_url(self, paths: List[str] = [], query_params: dict = None, **kwargs): + """ + Helper method to build the url for SumoLogic api requests. + + Example: + + paths = ["issue", "createmeta"] + query_params = {"projectKeys": "key1"} + url = __get_url("test", paths, query_params) + # url = https://api.sumologic.com/api/v1/issue/createmeta?projectKeys=key1 + """ + if self.authentication_config.deployment.lower() != "us1": + host = f"https://api.{self.authentication_config.deployment.lower()}.sumologic.com/api/v1/" + else: + host = "https://api.sumologic.com/api/v1/" + url = urljoin( + host, + "/".join(str(path) for path in paths), + ) + + # add query params + if query_params: + url = f"{url}?{urlencode(query_params)}" + + return url + + def validate_scopes(self) -> dict[str, bool | str]: + perms = {"manageScheduledViews", "manageConnections", "manageUsersAndRoles"} + self.logger.info("Validating SumoLogic authentication.") + try: + account_owner_response = requests.get( + url=self.__get_url(paths=["account", "accountOwner"]), + auth=self.__get_auth(), + headers=self.__get_headers(), + ) + + if account_owner_response.status_code == 200: + authenticated = True + user_id = account_owner_response.json() + self.logger.info( + "Successfully retrieved user_id", extra={"user_id": user_id} + ) + else: + account_owner_response = account_owner_response.json() + self.logger.error( + "Error while getting UserID", + extra={"error": str(account_owner_response)}, + ) + return { + "authenticated": str(account_owner_response), + "authorized": "Unauthorized", + } + + self.logger.info("Fetching account info...", extra={"user_id": user_id}) + account_info_response = requests.get( + url=self.__get_url(paths=["users", user_id]), + auth=self.__get_auth(), + headers=self.__get_headers(), + ) + + if account_info_response.status_code == 200: + role_ids = account_info_response.json()["roleIds"] + self.logger.info( + "Successfully fetched account info", extra={"roles": role_ids} + ) + else: + account_info_response = account_info_response.json() + self.logger.error( + "Error while getting account info", + extra={"error": str(account_info_response)}, + ) + return { + "authenticated": authenticated, + "authorized": str(account_info_response), + } + + # Checking if the required permissions exists + for role_id in role_ids: + role_info_response = requests.get( + url=self.__get_url(paths=["roles", role_id]), + auth=self.__get_auth(), + headers=self.__get_headers(), + ) + if role_info_response.status_code == 200: + role_info_response = role_info_response.json() + self.logger.info(f"Successfully fetched role: {role_id}") + for capability in role_info_response["capabilities"]: + if capability in perms: + perms.remove(capability) + else: + role_info_response = role_info_response.json() + self.logger.error( + f"Error while getting role: {role_id}", + extra={"error": str(role_info_response)}, + ) + return { + "authenticated": True, + "authorized": str(role_info_response), + } + if len(perms) == 0: + self.logger.info("All required perms found, user is authorized :)") + return {"authenticated": True, "authorized": True} + + except Exception as e: + self.logger.error("Error while getting User ID " + str(e)) + return {"authenticated": str(e), "authorized": str(e)} + + def __get_auth(self) -> tuple[str, str]: + return ( + self.authentication_config.sumoAccessId, + self.authentication_config.sumoAccessKey, + ) + + def __get_connection_id(self, connection_name: str): + params = {"limit": 1000} + while True: + connections_response = requests.get( + url=self.__get_url(paths=["connections"]), + headers=self.__get_headers(), + params=params, + auth=self.__get_auth(), + ) + if connections_response.status_code != 200: + raise Exception(str(connections_response.json())) + connections_response = connections_response.json() + for connection in connections_response["data"]: + if connection["name"] == connection_name: + return connection["id"] + + if connections_response["next"] is None: + break + params["token"] = connections_response["next"] + return None + + def __update_existing_connection(self, connection_id: str, connection_payload): + self.logger.info(f"Updating the connection: {connection_id}") + connection_update_response = requests.put( + url=self.__get_url(paths=["connections", connection_id]), + headers=self.__get_headers(), + auth=self.__get_auth(), + json=connection_payload, + ) + if connection_update_response.status_code == 200: + self.logger.info(f"Successfully updated connection: {connection_id}") + return connection_update_response.json()["id"] + else: + connection_update_response = connection_update_response.json() + self.logger.error( + f"Error while updating connection: {connection_id}", + extra={"error": str(connection_update_response)}, + ) + raise Exception(str(connection_update_response)) + + def __create_connection(self, connection_payload, connection_name: str): + self.logger.info("Creating a Webhook connection with Sumo Logic") + + try: + connection_creation_response = requests.post( + url=self.__get_url(paths=["connections"]), + json=connection_payload, + headers=self.__get_headers(), + auth=self.__get_auth(), + ) + if connection_creation_response.status_code == 200: + self.logger.info("Successfully created Webhook connection") + return connection_creation_response.json()["id"] + if connection_creation_response.status_code == 400: + connection_creation_response = connection_creation_response.json() + if ( + connection_creation_response["errors"][0]["code"] + == "connection:name_already_exists" + ): + self.logger.info( + "Webhook connection already exists, attempting to update it" + ) + connection_id = self.__get_connection_id( + connection_name=connection_name + ) + return self.__update_existing_connection( + connection_payload=connection_payload, + connection_id=connection_id, + ) + + raise Exception(str(connection_creation_response)) + else: + connection_creation_response = connection_creation_response.json() + self.logger.error( + "Error while creating webhook connection", + extra={"error": str(connection_creation_response)}, + ) + raise Exception(connection_creation_response) + except Exception as e: + self.logger.error("Error while creating webhook connection " + str(e)) + raise e + + def __get_monitors_without_keep(self, connection_id: str): + monitors = [] + params = {"query": "type:monitor"} + monitors_response = requests.get( + url=self.__get_url(paths=["monitors", "search"]), + params=params, + headers=self.__get_headers(), + auth=self.__get_auth(), + ) + + if monitors_response.status_code == 200: + self.logger.info("Successfully fetched all monitors") + monitors_response = monitors_response.json() + for monitor in monitors_response: + print(monitor) + for notification in monitor["item"]["notifications"]: + if notification["notification"]["connectionId"] == connection_id: + break + else: + monitors.append(monitor["item"]) + return monitors + else: + monitors_response = monitors_response.json() + self.logger.error( + "Error while getting monitors", extra=str(monitors_response) + ) + raise Exception(str(monitors_response)) + + def __install_connection_in_monitor(self, monitor, connection_id: str): + self.logger.info(f"Installing connection to monitor: {monitor['name']}") + monitor["type"] = "MonitorsLibraryMonitorUpdate" + triggers = [trigger["triggerType"] for trigger in monitor["triggers"]] + keep_notification = { + "notification": { + "connectionType": "Webhook", + "connectionId": connection_id, + "payloadOverride": None, + "resolutionPayloadOverride": None, + }, + "runForTriggerTypes": triggers, + } + monitor["notifications"].append(keep_notification) + monitor_update_response = requests.put( + url=self.__get_url(paths=["monitors", monitor["id"]]), + headers=self.__get_headers(), + auth=self.__get_auth(), + json=monitor, + ) + if monitor_update_response.status_code == 200: + self.logger.info( + f"Successfully installed connection to monitor: {monitor['name']}" + ) + else: + raise Exception(str(monitor_update_response.json())) + + def setup_webhook( + self, tenant_id: str, keep_api_url: str, api_key: str, setup_alerts: bool = True + ): + try: + parsed_url = urlparse(keep_api_url) + + # Extract the query string + query_params = parsed_url.query + + # Find the provider_id in the query parameters + # connection_template.json is the payload that will be sent to keep as an event + provider_id = query_params.split("provider_id=")[-1] + connection_name = f"KeepHQ-{provider_id}" + connection_payload = { + "type": "WebhookDefinition", + "name": connection_name, + "description": "A webhook connection that pushes alerts to KeepHQ", + "url": keep_api_url, + "headers": [], + "customHeaders": [{"name": "X-API-KEY", "value": api_key}], + "defaultPayload": open( + rf"{Path(__file__).parent}/connection_template.json" + ).read(), + "webhookType": "Webhook", + "connectionSubtype": "Event", + "resolutionPayload": open( + rf"{Path(__file__).parent}/connection_template.json" + ).read(), + } + # Creating a sumo logic connection + connection_id = self.__create_connection( + connection_payload=connection_payload, connection_name=connection_name + ) + + # Monitors + monitors = self.__get_monitors_without_keep(connection_id=connection_id) + + # Install connections in monitors that don't have keep + for monitor in monitors: + self.__install_connection_in_monitor( + monitor=monitor, connection_id=connection_id + ) + except Exception as e: + raise e + + @staticmethod + def __extract_severity(severity: str): + if "critical" in severity.lower(): + return AlertSeverity.CRITICAL + elif "warning" in severity.lower(): + return AlertSeverity.WARNING + elif "missing" in severity.lower(): + return AlertSeverity.INFO + + @staticmethod + def __extract_status(status: str): + if "resolved" in status.lower(): + return AlertStatus.RESOLVED + else: + return AlertStatus.FIRING + + @staticmethod + def _format_alert( + event: dict, + provider_instance: Optional["SumologicProvider"] = None, + ) -> AlertDto: + return AlertDto( + id=event["id"], + name=event["name"], + severity=SumologicProvider.__extract_severity( + severity=event["triggerType"] + ), + fingerprint=event["id"], + status=SumologicProvider.__extract_status(status=event["triggerType"]), + lastReceived=datetime.utcfromtimestamp( + int(event["triggerTimeStart"]) / 1000 + ).isoformat() + + "Z", + firingTimeStart=datetime.utcfromtimestamp( + int(event["triggerTimeStart"]) / 1000 + ).isoformat() + + "Z", + description=event["description"], + url=event["alertResponseUrl"], + source=["sumologic"], + ) From 0a3262861da74cf774333b0194ba4cf8c0825651 Mon Sep 17 00:00:00 2001 From: Tal <tal@keephq.dev> Date: Sun, 15 Sep 2024 17:52:13 +0300 Subject: [PATCH 8/9] fix(api): provider that fails to pull may sometime fail other providers (#1930) --- keep/api/routes/preset.py | 82 ++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/keep/api/routes/preset.py b/keep/api/routes/preset.py index bb4cb3c00..5af7768db 100644 --- a/keep/api/routes/preset.py +++ b/keep/api/routes/preset.py @@ -92,47 +92,59 @@ def pull_data_from_providers( provider_config=provider.details, ) - logger.info( - f"Pulling alerts from provider {provider.type} ({provider.id})", - extra=extra, - ) - sorted_provider_alerts_by_fingerprint = ( - provider_class.get_alerts_by_fingerprint(tenant_id=tenant_id) - ) - try: - if isinstance(provider_class, BaseTopologyProvider): - logger.info("Getting topology data", extra=extra) - topology_data = provider_class.pull_topology() - logger.info("Got topology data, processing", extra=extra) - process_topology(tenant_id, topology_data, provider.id, provider.type) - logger.info("Processed topology data", extra=extra) - except NotImplementedError: - logger.warning( - f"Provider {provider.type} ({provider.id}) does not support topology data", + logger.info( + f"Pulling alerts from provider {provider.type} ({provider.id})", extra=extra, ) - except Exception as e: - logger.error( - f"Unknown error pulling topology from provider {provider.type} ({provider.id})", - extra={**extra, "error": str(e)}, + sorted_provider_alerts_by_fingerprint = ( + provider_class.get_alerts_by_fingerprint(tenant_id=tenant_id) + ) + logger.info( + f"Pulling alerts from provider {provider.type} ({provider.id}) completed", + extra=extra, ) - # Even if we failed at processing some event, lets save the last pull time to not iterate this process over and over again. - update_provider_last_pull_time(tenant_id=tenant_id, provider_id=provider.id) - - for fingerprint, alert in sorted_provider_alerts_by_fingerprint.items(): - process_event( - {}, - tenant_id, - provider.type, - provider.id, - fingerprint, - None, - trace_id, - alert, - notify_client=False, + try: + if isinstance(provider_class, BaseTopologyProvider): + logger.info("Getting topology data", extra=extra) + topology_data = provider_class.pull_topology() + logger.info("Got topology data, processing", extra=extra) + process_topology( + tenant_id, topology_data, provider.id, provider.type + ) + logger.info("Processed topology data", extra=extra) + except NotImplementedError: + logger.warning( + f"Provider {provider.type} ({provider.id}) does not support topology data", + extra=extra, + ) + except Exception as e: + logger.error( + f"Unknown error pulling topology from provider {provider.type} ({provider.id})", + extra={**extra, "error": str(e)}, + ) + + for fingerprint, alert in sorted_provider_alerts_by_fingerprint.items(): + process_event( + {}, + tenant_id, + provider.type, + provider.id, + fingerprint, + None, + trace_id, + alert, + notify_client=False, + ) + except Exception: + logger.exception( + f"Unknown error pulling from provider {provider.type} ({provider.id})", + extra=extra, ) + finally: + # Even if we failed at processing some event, lets save the last pull time to not iterate this process over and over again. + update_provider_last_pull_time(tenant_id=tenant_id, provider_id=provider.id) @router.get( From 1f5a4d05b6aadbb92da8a01c165185b0ee6d82f1 Mon Sep 17 00:00:00 2001 From: Shahar Glazner <shaharglazner@gmail.com> Date: Sun, 15 Sep 2024 18:10:25 +0300 Subject: [PATCH 9/9] fix: calculate common path correctly (#1932) --- keep/workflowmanager/workflowstore.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/keep/workflowmanager/workflowstore.py b/keep/workflowmanager/workflowstore.py index 27fd8c898..af1ae746b 100644 --- a/keep/workflowmanager/workflowstore.py +++ b/keep/workflowmanager/workflowstore.py @@ -272,10 +272,10 @@ def provision_workflows_from_directory( # Check for workflows that are no longer in the directory or outside the workflows_dir and delete them for workflow in provisioned_workflows: - if ( - not os.path.exists(workflow.provisioned_file) - or not os.path.commonpath([workflows_dir, workflow.provisioned_file]) - == workflows_dir + if not os.path.exists( + workflow.provisioned_file + ) or not workflows_dir.endswith( + os.path.commonpath([workflows_dir, workflow.provisioned_file]) ): logger.info( f"Deprovisioning workflow {workflow.id} as its file no longer exists or is outside the workflows directory"