From 49946993ecebe5025f5acfbd7722fadeadd02429 Mon Sep 17 00:00:00 2001
From: Jared Burgett <147995946+jaredburgettelastic@users.noreply.github.com>
Date: Fri, 3 Jan 2025 15:10:56 -0600
Subject: [PATCH] [8.x] [Entity Store] [Asset Inventory] Universal entity
 definition (#202888) (#205536)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Entity Store] [Asset Inventory] Universal entity definition
(#202888)](https://github.com/elastic/kibana/pull/202888)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Tiago Vila
Verde","email":"tiago.vilaverde@elastic.co"},"sourceCommit":{"committedDate":"2025-01-03T16:43:16Z","message":"[Entity
Store] [Asset Inventory] Universal entity definition (#202888)\n\n##
Summary\r\n\r\nThis PR adds a universal entity definition.\r\nA
universal entity uses `related.entity` as an identifier field
and\r\nincludes an extra processor step that parses the
field\r\n`entities.keyword` and extracts all the entities in said field
(whose\r\noriginal data comes from `related.entities`).\r\n\r\nSee
this\r\n[doc](https://docs.google.com/document/d/1D8xDtn3HHP65i1Y3eIButacD6ZizyjZZRJB7mxlXzQY/edit?tab=t.0#heading=h.9fz3qtlfzjg7)\r\nfor
more details.\r\n\r\nTo accomplish this, we need to allow describing an
entity along with\r\nextra entity store resources required for that
entity's engine.\r\nThis PR reworks the current entity store by
introducing an `Entity\r\nDescription`, which has all that required
information. From it, we can\r\nbuild an `EntityEngineDescription` which
adds all the needed data that\r\nmust be computed (as opposed to
hardcoded) and is then used to generate\r\nall the resources needed for
that Entity's engine (entity definition,\r\npipeline, enrich policy,
index mappings, etc).\r\n\r\n<img width=\"3776\"
alt=\"EntityDescriptions\"\r\nsrc=\"https://github.com/user-attachments/assets/bdf7915f-1981-47e6-a815-31163f24ad03\">\r\n\r\nThis
required a refactoring of the current `UnitedEntityDefinition`,\r\nwhich
has now been removed in favour of more contextual functions for\r\nall
the different parts.\r\nThe intention is to decouple the Entity
Description schema from the\r\nschemas required for field retention,
entity manager and pipeline. We\r\ncan then freely expand on our Entity
Description as required, and simply\r\nalter the conversion functions
when needed.\r\n\r\n## How to test\r\n\r\n1. On a fresh ES cluster, add
some entity data\r\n* For hosts and user, use the [security
documents\r\ngenerator](https://github.com/elastic/security-documents-generator)\r\n
* For universal, there are a few more steps:\r\n 1. Create the
`entity.keyword` builder pipeline\r\n 2. Add it to a index template\r\n
3. Post some docs to the corresponding index \r\n2. Initialise the
universal entity engine via:
`POST\r\nkbn:/api/entity_store/engines/universal/init {}`\r\n* Note that
using the UI does not work, as we've specifically removed\r\nthe
Universal engine from the normal Entity Store workflow\r\n3. Check the
status of the store is `running` via
`GET\r\nkbn:/api/entity_store/status`\r\n4. Once the transform runs, you
can query `GET entities*/_search` to see\r\nthe created
entities\r\n\r\nNote that universal entities do not show up in the
dashboard Entities\r\nList.\r\n\r\n\r\n### Code to ingest
data\r\n<details>\r\n<summary>Pipeline</summary>\r\n\r\n```js\r\nPUT
_ingest/pipeline/entities-keyword-builder\r\n{\r\n
\"description\":\"Serialize entities.metadata into a keyword
field\",\r\n \"processors\":[\r\n {\r\n \"script\":{\r\n
\"lang\":\"painless\",\r\n \"source\":\"\"\"\r\nString jsonFromMap(Map
map) {\r\n StringBuilder json = new StringBuilder(\"{\");\r\n boolean
first = true;\r\n\r\n for (entry in map.entrySet()) {\r\n if (!first)
{\r\n json.append(\",\");\r\n }\r\n first = false;\r\n\r\n String key =
entry.getKey().replace(\"\\\"\", \"\\\\\\\"\");\r\n Object value =
entry.getValue();\r\n\r\n
json.append(\"\\\"\").append(key).append(\"\\\":\");\r\n\r\n if (value
instanceof String) {\r\n String escapedValue = ((String)
value).replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\", \":\");\r\n
json.append(\"\\\"\").append(escapedValue).append(\"\\\"\");\r\n } else
if (value instanceof Map) {\r\n json.append(jsonFromMap((Map)
value));\r\n } else if (value instanceof List) {\r\n
json.append(jsonFromList((List) value));\r\n } else if (value instanceof
Boolean || value instanceof Number) {\r\n
json.append(value.toString());\r\n } else {\r\n // For other types,
treat as string\r\n String escapedValue =
value.toString().replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\",
\":\");\r\n
json.append(\"\\\"\").append(escapedValue).append(\"\\\"\");\r\n }\r\n
}\r\n\r\n json.append(\"}\");\r\n return
json.toString();\r\n}\r\n\r\nString jsonFromList(List list) {\r\n\r\n
StringBuilder json = new StringBuilder(\"[\");\r\n boolean first =
true;\r\n\r\n for (item in list) {\r\n if (!first) {\r\n
json.append(\",\");\r\n }\r\n first = false;\r\n\r\n if (item instanceof
String) {\r\n String escapedItem = ((String) item).replace(\"\\\"\",
\"\\\\\\\"\").replace(\"=\", \":\");\r\n
json.append(\"\\\"\").append(escapedItem).append(\"\\\"\");\r\n } else
if (item instanceof Map) {\r\n json.append(jsonFromMap((Map) item));\r\n
} else if (item instanceof List) {\r\n json.append(jsonFromList((List)
item));\r\n } else if (item instanceof Boolean || item instanceof
Number) {\r\n json.append(item.toString());\r\n } else {\r\n // For
other types, treat as string\r\n String escapedItem =
item.toString().replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\",
\":\");\r\n
json.append(\"\\\"\").append(escapedItem).append(\"\\\"\");\r\n }\r\n
}\r\n\r\n json.append(\"]\");\r\n return
json.toString();\r\n}\r\n\r\ndef metadata =
jsonFromMap(ctx['entities']['metadata']);\r\nctx['entities']['keyword']
= metadata;\r\n\"\"\"\r\n\r\n }\r\n }\r\n
]\r\n}\r\n```\r\n</details>\r\n\r\n<details>\r\n<summary>Index
template</summary>\r\n\r\n```js\r\nPUT
/_index_template/entity_store_index_template\r\n{\r\n
\"index_patterns\":[\r\n \"logs-store\"\r\n ],\r\n \"template\":{\r\n
\"settings\":{\r\n \"index\":{\r\n
\"default_pipeline\":\"entities-keyword-builder\"\r\n }\r\n },\r\n
\"mappings\":{\r\n \"properties\":{\r\n \"@timestamp\":{\r\n
\"type\":\"date\"\r\n },\r\n \"message\":{\r\n \"type\":\"text\"\r\n
},\r\n \"event\":{\r\n \"properties\":{\r\n \"action\":{\r\n
\"type\":\"keyword\"\r\n },\r\n \"category\":{\r\n
\"type\":\"keyword\"\r\n },\r\n \"type\":{\r\n \"type\":\"keyword\"\r\n
},\r\n \"outcome\":{\r\n \"type\":\"keyword\"\r\n },\r\n
\"provider\":{\r\n \"type\":\"keyword\"\r\n },\r\n \"ingested\":{\r\n
\"type\": \"date\"\r\n }\r\n }\r\n },\r\n \"related\":{\r\n
\"properties\":{\r\n \"entity\":{\r\n \"type\":\"keyword\"\r\n }\r\n
}\r\n },\r\n \"entities\":{\r\n \"properties\":{\r\n \"metadata\":{\r\n
\"type\":\"flattened\"\r\n },\r\n \"keyword\":{\r\n
\"type\":\"keyword\"\r\n }\r\n }\r\n }\r\n }\r\n }\r\n
}\r\n}\r\n```\r\n</details>\r\n\r\n<details>\r\n<summary>Example source
doc</summary>\r\n\r\n```js\r\nPOST /logs-store/_doc/\r\n{\r\n
\"@timestamp\":\"2024-11-29T10:01:00Z\",\r\n \"message\":\"Eddie\",\r\n
\"event\": {\r\n \"type\":[\r\n \"creation\"\r\n ],\r\n \"ingested\":
\"2024-12-03T10:01:00Z\"\r\n },\r\n \"related\":{\r\n \"entity\":[\r\n
\"AKIAI44QH8DHBEXAMPLE\"\r\n ]\r\n },\r\n \"entities\":{\r\n
\"metadata\":{\r\n \"AKIAI44QH8DHBEXAMPLE\":{\r\n \"entity\":{\r\n
\"id\":\"AKIAI44QH8DHBEXAMPLE\",\r\n \"category\":\"Access
Management\",\r\n \"type\":\"AWS IAM Access Key\"\r\n },\r\n
\"cloud\":{\r\n \"account\":{\r\n \"id\":\"444455556666\"\r\n }\r\n
}\r\n }\r\n }\r\n }\r\n}\r\n```\r\n</details>\r\n\r\n### To do\r\n\r\n-
[x] Add/Improve [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\n-
[x] Feature flag\r\n\r\n\r\n----\r\n#### Update:\r\n\r\nAdded
`assetInventoryStoreEnabled` Feature Flag. It is disabled by\r\ndefault
and even when enabled, the `/api/entity_store/enable` route does\r\nnot
initialize the Universal Entity
Engine.\r\n`/api/entity_store/engines/universal/init` needs to be
manually called\r\nto initialize
it\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Rômulo Farias <romulodefarias@gmail.com>\r\nCo-authored-by:
jaredburgettelastic <jared.burgett@elastic.co>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"c6b0a31d8ec6d423a8071f50a22e55acedd0dee0","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Cloud
Security","ci:cloud-deploy","Theme: entity_analytics","Team:Entity
Analytics","backport:version","v8.18.0"],"number":202888,"url":"https://github.com/elastic/kibana/pull/202888","mergeCommit":{"message":"[Entity
Store] [Asset Inventory] Universal entity definition (#202888)\n\n##
Summary\r\n\r\nThis PR adds a universal entity definition.\r\nA
universal entity uses `related.entity` as an identifier field
and\r\nincludes an extra processor step that parses the
field\r\n`entities.keyword` and extracts all the entities in said field
(whose\r\noriginal data comes from `related.entities`).\r\n\r\nSee
this\r\n[doc](https://docs.google.com/document/d/1D8xDtn3HHP65i1Y3eIButacD6ZizyjZZRJB7mxlXzQY/edit?tab=t.0#heading=h.9fz3qtlfzjg7)\r\nfor
more details.\r\n\r\nTo accomplish this, we need to allow describing an
entity along with\r\nextra entity store resources required for that
entity's engine.\r\nThis PR reworks the current entity store by
introducing an `Entity\r\nDescription`, which has all that required
information. From it, we can\r\nbuild an `EntityEngineDescription` which
adds all the needed data that\r\nmust be computed (as opposed to
hardcoded) and is then used to generate\r\nall the resources needed for
that Entity's engine (entity definition,\r\npipeline, enrich policy,
index mappings, etc).\r\n\r\n<img width=\"3776\"
alt=\"EntityDescriptions\"\r\nsrc=\"https://github.com/user-attachments/assets/bdf7915f-1981-47e6-a815-31163f24ad03\">\r\n\r\nThis
required a refactoring of the current `UnitedEntityDefinition`,\r\nwhich
has now been removed in favour of more contextual functions for\r\nall
the different parts.\r\nThe intention is to decouple the Entity
Description schema from the\r\nschemas required for field retention,
entity manager and pipeline. We\r\ncan then freely expand on our Entity
Description as required, and simply\r\nalter the conversion functions
when needed.\r\n\r\n## How to test\r\n\r\n1. On a fresh ES cluster, add
some entity data\r\n* For hosts and user, use the [security
documents\r\ngenerator](https://github.com/elastic/security-documents-generator)\r\n
* For universal, there are a few more steps:\r\n 1. Create the
`entity.keyword` builder pipeline\r\n 2. Add it to a index template\r\n
3. Post some docs to the corresponding index \r\n2. Initialise the
universal entity engine via:
`POST\r\nkbn:/api/entity_store/engines/universal/init {}`\r\n* Note that
using the UI does not work, as we've specifically removed\r\nthe
Universal engine from the normal Entity Store workflow\r\n3. Check the
status of the store is `running` via
`GET\r\nkbn:/api/entity_store/status`\r\n4. Once the transform runs, you
can query `GET entities*/_search` to see\r\nthe created
entities\r\n\r\nNote that universal entities do not show up in the
dashboard Entities\r\nList.\r\n\r\n\r\n### Code to ingest
data\r\n<details>\r\n<summary>Pipeline</summary>\r\n\r\n```js\r\nPUT
_ingest/pipeline/entities-keyword-builder\r\n{\r\n
\"description\":\"Serialize entities.metadata into a keyword
field\",\r\n \"processors\":[\r\n {\r\n \"script\":{\r\n
\"lang\":\"painless\",\r\n \"source\":\"\"\"\r\nString jsonFromMap(Map
map) {\r\n StringBuilder json = new StringBuilder(\"{\");\r\n boolean
first = true;\r\n\r\n for (entry in map.entrySet()) {\r\n if (!first)
{\r\n json.append(\",\");\r\n }\r\n first = false;\r\n\r\n String key =
entry.getKey().replace(\"\\\"\", \"\\\\\\\"\");\r\n Object value =
entry.getValue();\r\n\r\n
json.append(\"\\\"\").append(key).append(\"\\\":\");\r\n\r\n if (value
instanceof String) {\r\n String escapedValue = ((String)
value).replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\", \":\");\r\n
json.append(\"\\\"\").append(escapedValue).append(\"\\\"\");\r\n } else
if (value instanceof Map) {\r\n json.append(jsonFromMap((Map)
value));\r\n } else if (value instanceof List) {\r\n
json.append(jsonFromList((List) value));\r\n } else if (value instanceof
Boolean || value instanceof Number) {\r\n
json.append(value.toString());\r\n } else {\r\n // For other types,
treat as string\r\n String escapedValue =
value.toString().replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\",
\":\");\r\n
json.append(\"\\\"\").append(escapedValue).append(\"\\\"\");\r\n }\r\n
}\r\n\r\n json.append(\"}\");\r\n return
json.toString();\r\n}\r\n\r\nString jsonFromList(List list) {\r\n\r\n
StringBuilder json = new StringBuilder(\"[\");\r\n boolean first =
true;\r\n\r\n for (item in list) {\r\n if (!first) {\r\n
json.append(\",\");\r\n }\r\n first = false;\r\n\r\n if (item instanceof
String) {\r\n String escapedItem = ((String) item).replace(\"\\\"\",
\"\\\\\\\"\").replace(\"=\", \":\");\r\n
json.append(\"\\\"\").append(escapedItem).append(\"\\\"\");\r\n } else
if (item instanceof Map) {\r\n json.append(jsonFromMap((Map) item));\r\n
} else if (item instanceof List) {\r\n json.append(jsonFromList((List)
item));\r\n } else if (item instanceof Boolean || item instanceof
Number) {\r\n json.append(item.toString());\r\n } else {\r\n // For
other types, treat as string\r\n String escapedItem =
item.toString().replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\",
\":\");\r\n
json.append(\"\\\"\").append(escapedItem).append(\"\\\"\");\r\n }\r\n
}\r\n\r\n json.append(\"]\");\r\n return
json.toString();\r\n}\r\n\r\ndef metadata =
jsonFromMap(ctx['entities']['metadata']);\r\nctx['entities']['keyword']
= metadata;\r\n\"\"\"\r\n\r\n }\r\n }\r\n
]\r\n}\r\n```\r\n</details>\r\n\r\n<details>\r\n<summary>Index
template</summary>\r\n\r\n```js\r\nPUT
/_index_template/entity_store_index_template\r\n{\r\n
\"index_patterns\":[\r\n \"logs-store\"\r\n ],\r\n \"template\":{\r\n
\"settings\":{\r\n \"index\":{\r\n
\"default_pipeline\":\"entities-keyword-builder\"\r\n }\r\n },\r\n
\"mappings\":{\r\n \"properties\":{\r\n \"@timestamp\":{\r\n
\"type\":\"date\"\r\n },\r\n \"message\":{\r\n \"type\":\"text\"\r\n
},\r\n \"event\":{\r\n \"properties\":{\r\n \"action\":{\r\n
\"type\":\"keyword\"\r\n },\r\n \"category\":{\r\n
\"type\":\"keyword\"\r\n },\r\n \"type\":{\r\n \"type\":\"keyword\"\r\n
},\r\n \"outcome\":{\r\n \"type\":\"keyword\"\r\n },\r\n
\"provider\":{\r\n \"type\":\"keyword\"\r\n },\r\n \"ingested\":{\r\n
\"type\": \"date\"\r\n }\r\n }\r\n },\r\n \"related\":{\r\n
\"properties\":{\r\n \"entity\":{\r\n \"type\":\"keyword\"\r\n }\r\n
}\r\n },\r\n \"entities\":{\r\n \"properties\":{\r\n \"metadata\":{\r\n
\"type\":\"flattened\"\r\n },\r\n \"keyword\":{\r\n
\"type\":\"keyword\"\r\n }\r\n }\r\n }\r\n }\r\n }\r\n
}\r\n}\r\n```\r\n</details>\r\n\r\n<details>\r\n<summary>Example source
doc</summary>\r\n\r\n```js\r\nPOST /logs-store/_doc/\r\n{\r\n
\"@timestamp\":\"2024-11-29T10:01:00Z\",\r\n \"message\":\"Eddie\",\r\n
\"event\": {\r\n \"type\":[\r\n \"creation\"\r\n ],\r\n \"ingested\":
\"2024-12-03T10:01:00Z\"\r\n },\r\n \"related\":{\r\n \"entity\":[\r\n
\"AKIAI44QH8DHBEXAMPLE\"\r\n ]\r\n },\r\n \"entities\":{\r\n
\"metadata\":{\r\n \"AKIAI44QH8DHBEXAMPLE\":{\r\n \"entity\":{\r\n
\"id\":\"AKIAI44QH8DHBEXAMPLE\",\r\n \"category\":\"Access
Management\",\r\n \"type\":\"AWS IAM Access Key\"\r\n },\r\n
\"cloud\":{\r\n \"account\":{\r\n \"id\":\"444455556666\"\r\n }\r\n
}\r\n }\r\n }\r\n }\r\n}\r\n```\r\n</details>\r\n\r\n### To do\r\n\r\n-
[x] Add/Improve [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\n-
[x] Feature flag\r\n\r\n\r\n----\r\n#### Update:\r\n\r\nAdded
`assetInventoryStoreEnabled` Feature Flag. It is disabled by\r\ndefault
and even when enabled, the `/api/entity_store/enable` route does\r\nnot
initialize the Universal Entity
Engine.\r\n`/api/entity_store/engines/universal/init` needs to be
manually called\r\nto initialize
it\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Rômulo Farias <romulodefarias@gmail.com>\r\nCo-authored-by:
jaredburgettelastic <jared.burgett@elastic.co>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"c6b0a31d8ec6d423a8071f50a22e55acedd0dee0"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/202888","number":202888,"mergeCommit":{"message":"[Entity
Store] [Asset Inventory] Universal entity definition (#202888)\n\n##
Summary\r\n\r\nThis PR adds a universal entity definition.\r\nA
universal entity uses `related.entity` as an identifier field
and\r\nincludes an extra processor step that parses the
field\r\n`entities.keyword` and extracts all the entities in said field
(whose\r\noriginal data comes from `related.entities`).\r\n\r\nSee
this\r\n[doc](https://docs.google.com/document/d/1D8xDtn3HHP65i1Y3eIButacD6ZizyjZZRJB7mxlXzQY/edit?tab=t.0#heading=h.9fz3qtlfzjg7)\r\nfor
more details.\r\n\r\nTo accomplish this, we need to allow describing an
entity along with\r\nextra entity store resources required for that
entity's engine.\r\nThis PR reworks the current entity store by
introducing an `Entity\r\nDescription`, which has all that required
information. From it, we can\r\nbuild an `EntityEngineDescription` which
adds all the needed data that\r\nmust be computed (as opposed to
hardcoded) and is then used to generate\r\nall the resources needed for
that Entity's engine (entity definition,\r\npipeline, enrich policy,
index mappings, etc).\r\n\r\n<img width=\"3776\"
alt=\"EntityDescriptions\"\r\nsrc=\"https://github.com/user-attachments/assets/bdf7915f-1981-47e6-a815-31163f24ad03\">\r\n\r\nThis
required a refactoring of the current `UnitedEntityDefinition`,\r\nwhich
has now been removed in favour of more contextual functions for\r\nall
the different parts.\r\nThe intention is to decouple the Entity
Description schema from the\r\nschemas required for field retention,
entity manager and pipeline. We\r\ncan then freely expand on our Entity
Description as required, and simply\r\nalter the conversion functions
when needed.\r\n\r\n## How to test\r\n\r\n1. On a fresh ES cluster, add
some entity data\r\n* For hosts and user, use the [security
documents\r\ngenerator](https://github.com/elastic/security-documents-generator)\r\n
* For universal, there are a few more steps:\r\n 1. Create the
`entity.keyword` builder pipeline\r\n 2. Add it to a index template\r\n
3. Post some docs to the corresponding index \r\n2. Initialise the
universal entity engine via:
`POST\r\nkbn:/api/entity_store/engines/universal/init {}`\r\n* Note that
using the UI does not work, as we've specifically removed\r\nthe
Universal engine from the normal Entity Store workflow\r\n3. Check the
status of the store is `running` via
`GET\r\nkbn:/api/entity_store/status`\r\n4. Once the transform runs, you
can query `GET entities*/_search` to see\r\nthe created
entities\r\n\r\nNote that universal entities do not show up in the
dashboard Entities\r\nList.\r\n\r\n\r\n### Code to ingest
data\r\n<details>\r\n<summary>Pipeline</summary>\r\n\r\n```js\r\nPUT
_ingest/pipeline/entities-keyword-builder\r\n{\r\n
\"description\":\"Serialize entities.metadata into a keyword
field\",\r\n \"processors\":[\r\n {\r\n \"script\":{\r\n
\"lang\":\"painless\",\r\n \"source\":\"\"\"\r\nString jsonFromMap(Map
map) {\r\n StringBuilder json = new StringBuilder(\"{\");\r\n boolean
first = true;\r\n\r\n for (entry in map.entrySet()) {\r\n if (!first)
{\r\n json.append(\",\");\r\n }\r\n first = false;\r\n\r\n String key =
entry.getKey().replace(\"\\\"\", \"\\\\\\\"\");\r\n Object value =
entry.getValue();\r\n\r\n
json.append(\"\\\"\").append(key).append(\"\\\":\");\r\n\r\n if (value
instanceof String) {\r\n String escapedValue = ((String)
value).replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\", \":\");\r\n
json.append(\"\\\"\").append(escapedValue).append(\"\\\"\");\r\n } else
if (value instanceof Map) {\r\n json.append(jsonFromMap((Map)
value));\r\n } else if (value instanceof List) {\r\n
json.append(jsonFromList((List) value));\r\n } else if (value instanceof
Boolean || value instanceof Number) {\r\n
json.append(value.toString());\r\n } else {\r\n // For other types,
treat as string\r\n String escapedValue =
value.toString().replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\",
\":\");\r\n
json.append(\"\\\"\").append(escapedValue).append(\"\\\"\");\r\n }\r\n
}\r\n\r\n json.append(\"}\");\r\n return
json.toString();\r\n}\r\n\r\nString jsonFromList(List list) {\r\n\r\n
StringBuilder json = new StringBuilder(\"[\");\r\n boolean first =
true;\r\n\r\n for (item in list) {\r\n if (!first) {\r\n
json.append(\",\");\r\n }\r\n first = false;\r\n\r\n if (item instanceof
String) {\r\n String escapedItem = ((String) item).replace(\"\\\"\",
\"\\\\\\\"\").replace(\"=\", \":\");\r\n
json.append(\"\\\"\").append(escapedItem).append(\"\\\"\");\r\n } else
if (item instanceof Map) {\r\n json.append(jsonFromMap((Map) item));\r\n
} else if (item instanceof List) {\r\n json.append(jsonFromList((List)
item));\r\n } else if (item instanceof Boolean || item instanceof
Number) {\r\n json.append(item.toString());\r\n } else {\r\n // For
other types, treat as string\r\n String escapedItem =
item.toString().replace(\"\\\"\", \"\\\\\\\"\").replace(\"=\",
\":\");\r\n
json.append(\"\\\"\").append(escapedItem).append(\"\\\"\");\r\n }\r\n
}\r\n\r\n json.append(\"]\");\r\n return
json.toString();\r\n}\r\n\r\ndef metadata =
jsonFromMap(ctx['entities']['metadata']);\r\nctx['entities']['keyword']
= metadata;\r\n\"\"\"\r\n\r\n }\r\n }\r\n
]\r\n}\r\n```\r\n</details>\r\n\r\n<details>\r\n<summary>Index
template</summary>\r\n\r\n```js\r\nPUT
/_index_template/entity_store_index_template\r\n{\r\n
\"index_patterns\":[\r\n \"logs-store\"\r\n ],\r\n \"template\":{\r\n
\"settings\":{\r\n \"index\":{\r\n
\"default_pipeline\":\"entities-keyword-builder\"\r\n }\r\n },\r\n
\"mappings\":{\r\n \"properties\":{\r\n \"@timestamp\":{\r\n
\"type\":\"date\"\r\n },\r\n \"message\":{\r\n \"type\":\"text\"\r\n
},\r\n \"event\":{\r\n \"properties\":{\r\n \"action\":{\r\n
\"type\":\"keyword\"\r\n },\r\n \"category\":{\r\n
\"type\":\"keyword\"\r\n },\r\n \"type\":{\r\n \"type\":\"keyword\"\r\n
},\r\n \"outcome\":{\r\n \"type\":\"keyword\"\r\n },\r\n
\"provider\":{\r\n \"type\":\"keyword\"\r\n },\r\n \"ingested\":{\r\n
\"type\": \"date\"\r\n }\r\n }\r\n },\r\n \"related\":{\r\n
\"properties\":{\r\n \"entity\":{\r\n \"type\":\"keyword\"\r\n }\r\n
}\r\n },\r\n \"entities\":{\r\n \"properties\":{\r\n \"metadata\":{\r\n
\"type\":\"flattened\"\r\n },\r\n \"keyword\":{\r\n
\"type\":\"keyword\"\r\n }\r\n }\r\n }\r\n }\r\n }\r\n
}\r\n}\r\n```\r\n</details>\r\n\r\n<details>\r\n<summary>Example source
doc</summary>\r\n\r\n```js\r\nPOST /logs-store/_doc/\r\n{\r\n
\"@timestamp\":\"2024-11-29T10:01:00Z\",\r\n \"message\":\"Eddie\",\r\n
\"event\": {\r\n \"type\":[\r\n \"creation\"\r\n ],\r\n \"ingested\":
\"2024-12-03T10:01:00Z\"\r\n },\r\n \"related\":{\r\n \"entity\":[\r\n
\"AKIAI44QH8DHBEXAMPLE\"\r\n ]\r\n },\r\n \"entities\":{\r\n
\"metadata\":{\r\n \"AKIAI44QH8DHBEXAMPLE\":{\r\n \"entity\":{\r\n
\"id\":\"AKIAI44QH8DHBEXAMPLE\",\r\n \"category\":\"Access
Management\",\r\n \"type\":\"AWS IAM Access Key\"\r\n },\r\n
\"cloud\":{\r\n \"account\":{\r\n \"id\":\"444455556666\"\r\n }\r\n
}\r\n }\r\n }\r\n }\r\n}\r\n```\r\n</details>\r\n\r\n### To do\r\n\r\n-
[x] Add/Improve [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\n-
[x] Feature flag\r\n\r\n\r\n----\r\n#### Update:\r\n\r\nAdded
`assetInventoryStoreEnabled` Feature Flag. It is disabled by\r\ndefault
and even when enabled, the `/api/entity_store/enable` route does\r\nnot
initialize the Universal Entity
Engine.\r\n`/api/entity_store/engines/universal/init` needs to be
manually called\r\nto initialize
it\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Rômulo Farias <romulodefarias@gmail.com>\r\nCo-authored-by:
jaredburgettelastic <jared.burgett@elastic.co>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"c6b0a31d8ec6d423a8071f50a22e55acedd0dee0"}},{"branch":"8.x","label":"v8.18.0","labelRegex":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Tiago Vila Verde <tiago.vilaverde@elastic.co>
---
 oas_docs/output/kibana.serverless.yaml        |   5 +-
 oas_docs/output/kibana.yaml                   |   2 +
 .../asset_criticality/common.gen.ts           |   2 +-
 .../asset_criticality/common.schema.yaml      |   1 +
 .../entity_store/common.gen.ts                |   2 +-
 .../entity_store/common.schema.yaml           |   1 +
 .../parse_asset_criticality_csv_row.test.ts   |   4 +-
 .../entity_store/constants.ts                 |   1 +
 .../common/experimental_features.ts           |   7 +-
 ...alytics_api_2023_10_31.bundled.schema.yaml |   2 +
 ...alytics_api_2023_10_31.bundled.schema.yaml |   2 +
 .../asset_criticality/helpers.ts              |   1 +
 .../component_template.ts                     |  21 +-
 .../elasticsearch_assets/enrich_policy.ts     |  72 ++--
 .../elasticsearch_assets/entity_index.ts      |  51 +++
 .../elasticsearch_assets/ingest_pipeline.ts   |  84 ++--
 .../ingest_processor_steps/index.ts           |   1 -
 ...on_definition_to_ingest_processor_steps.ts |  26 --
 .../entity_definitions/constants.ts           |  27 ++
 .../entity_descriptions/common.ts             |  36 ++
 .../entity_descriptions/field_utils.ts        |  79 ++++
 .../entity_descriptions/host.ts               |  48 +++
 .../entity_descriptions}/index.ts             |   3 +-
 .../entity_descriptions/service.ts            |  32 ++
 .../entity_descriptions/universal.ts          |  60 +++
 .../entity_descriptions/user.ts               |  39 ++
 .../entity_manager_conversion.ts              |  40 ++
 .../entity_store/entity_definitions/types.ts  |  27 ++
 .../entity_store_data_client.test.ts          |  32 +-
 .../entity_store/entity_store_data_client.ts  | 103 ++---
 .../collect_values.ts                         |  27 +-
 .../index.ts                                  |   0
 .../operator_to_ingest_processor.ts           |  22 +-
 .../prefer_newest_value.ts                    |  16 +-
 .../prefer_oldest_value.ts                    |  14 +-
 .../entity_store/field_retention/types.ts     |  18 +
 .../field_retention_definition/types.ts       |  34 --
 .../engine_description.test.ts}               | 363 +++---------------
 .../installation/engine_description.ts        |  95 +++++
 .../entity_store/installation/types.ts        |  73 ++++
 .../task/field_retention_enrichment_task.ts   |  13 +-
 .../united_entity_definitions/constants.ts    |  28 --
 .../definition_utils.ts                       |  86 -----
 .../entity_types/common.ts                    |  46 ---
 .../entity_types/host.ts                      |  44 ---
 .../entity_types/service.ts                   |  29 --
 .../entity_types/user.ts                      |  35 --
 .../get_united_definition.ts                  |  69 ----
 .../united_entity_definitions/index.ts        |  10 -
 .../united_entity_definitions/types.ts        |  30 --
 .../united_entity_definition.ts               | 126 ------
 .../entity_store/utils/entity_utils.ts        |   5 -
 .../field_retention_operators.ts              | 264 +++++++++----
 .../asset_criticality_csv_upload.ts           |   2 +-
 54 files changed, 1130 insertions(+), 1130 deletions(-)
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/retention_definition_to_ingest_processor_steps.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/field_utils.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/host.ts
 rename x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/{united_entity_definitions/entity_types => entity_definitions/entity_descriptions}/index.ts (79%)
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/service.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/user.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_manager_conversion.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/types.ts
 rename x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/{field_retention_definition => field_retention}/collect_values.ts (77%)
 rename x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/{field_retention_definition => field_retention}/index.ts (100%)
 rename x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/{field_retention_definition => field_retention}/operator_to_ingest_processor.ts (54%)
 rename x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/{field_retention_definition => field_retention}/prefer_newest_value.ts (68%)
 rename x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/{field_retention_definition => field_retention}/prefer_oldest_value.ts (69%)
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/types.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/types.ts
 rename x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/{united_entity_definitions/get_united_definition.test.ts => installation/engine_description.test.ts} (64%)
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts
 create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/types.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/constants.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/definition_utils.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/common.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/host.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/user.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/index.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/types.ts
 delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts

diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml
index a91f43493af42..9a44e72c7a33e 100644
--- a/oas_docs/output/kibana.serverless.yaml
+++ b/oas_docs/output/kibana.serverless.yaml
@@ -45909,6 +45909,8 @@ components:
       enum:
         - user
         - host
+        - service
+        - universal
       type: string
     Security_Entity_Analytics_API_HostEntity:
       type: object
@@ -45979,6 +45981,7 @@ components:
         - host.name
         - user.name
         - service.name
+        - related.entity
       type: string
     Security_Entity_Analytics_API_IndexPattern:
       type: string
@@ -50085,7 +50088,7 @@ components:
           items:
             type: string
           description: |
-            A list of "carbon copy" email addresses. Addresses can be specified in `user@host-name` format or in name `<user@host-name>` format 
+            A list of "carbon copy" email addresses. Addresses can be specified in `user@host-name` format or in name `<user@host-name>` format
         message:
           type: string
           description: The email message text. Markdown format is supported.
diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml
index 3b937f164cba2..ad802ce8cdd19 100644
--- a/oas_docs/output/kibana.yaml
+++ b/oas_docs/output/kibana.yaml
@@ -35059,6 +35059,7 @@ components:
         - user
         - host
         - service
+        - universal
       type: string
     Security_Entity_Analytics_API_HostEntity:
       type: object
@@ -35129,6 +35130,7 @@ components:
         - host.name
         - user.name
         - service.name
+        - related.entity
       type: string
     Security_Entity_Analytics_API_IndexPattern:
       type: string
diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts
index acae48846c82c..2c9cc0760f210 100644
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.gen.ts
@@ -17,7 +17,7 @@
 import { z } from '@kbn/zod';
 
 export type IdField = z.infer<typeof IdField>;
-export const IdField = z.enum(['host.name', 'user.name', 'service.name']);
+export const IdField = z.enum(['host.name', 'user.name', 'service.name', 'related.entity']);
 export type IdFieldEnum = typeof IdField.enum;
 export const IdFieldEnum = IdField.enum;
 
diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml
index 2dd4f3f635599..1bd73eea6cef4 100644
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/asset_criticality/common.schema.yaml
@@ -29,6 +29,7 @@ components:
         - 'host.name'
         - 'user.name'
         - 'service.name'
+        - 'related.entity'
     AssetCriticalityRecordIdParts:
       type: object
       properties:
diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
index 3511e1d166ede..8fd0b17161154 100644
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
@@ -17,7 +17,7 @@
 import { z } from '@kbn/zod';
 
 export type EntityType = z.infer<typeof EntityType>;
-export const EntityType = z.enum(['user', 'host', 'service']);
+export const EntityType = z.enum(['user', 'host', 'service', 'universal']);
 export type EntityTypeEnum = typeof EntityType.enum;
 export const EntityTypeEnum = EntityType.enum;
 
diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
index 0ddbf1c9b3fd0..68b6e6612735c 100644
--- a/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
@@ -12,6 +12,7 @@ components:
         - user
         - host
         - service
+        - universal
 
     EngineDescriptor:
       type: object
diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts
index 21c0785db5c28..f4633ba87d385 100644
--- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts
@@ -46,7 +46,7 @@ describe('parseAssetCriticalityCsvRow', () => {
 
     // @ts-ignore result can now only be InvalidRecord
     expect(result.error).toMatchInlineSnapshot(
-      `"Invalid entity type \\"invalid\\", expected to be one of: user, host, service"`
+      `"Invalid entity type \\"invalid\\", expected to be one of: user, host, service, universal"`
     );
   });
 
@@ -57,7 +57,7 @@ describe('parseAssetCriticalityCsvRow', () => {
 
     // @ts-ignore result can now only be InvalidRecord
     expect(result.error).toMatchInlineSnapshot(
-      `"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host, service"`
+      `"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host, service, universal"`
     );
   });
 
diff --git a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts
index d74c0fdd3735d..4c050dc7a83ce 100644
--- a/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/entity_analytics/entity_store/constants.ts
@@ -31,6 +31,7 @@ export const IDENTITY_FIELD_MAP: Record<EntityType, IdField> = {
   [EntityTypeEnum.host]: 'host.name',
   [EntityTypeEnum.user]: 'user.name',
   [EntityTypeEnum.service]: 'service.name',
+  [EntityTypeEnum.universal]: 'related.entity',
 };
 
 export const getAvailableEntityTypes = (): EntityType[] =>
diff --git a/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts b/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts
index d7f004e3f9430..e7476ab2b1a5c 100644
--- a/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts
+++ b/x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts
@@ -266,8 +266,13 @@ export const allowedExperimentalValues = Object.freeze({
   /**
    * Enables CrowdStrike's RunScript RTR command
    */
-
   crowdstrikeRunScriptEnabled: false,
+
+  /**
+   * Enables the Asset Inventory Entity Store feature.
+   * Allows initializing the Universal Entity Store via the API.
+   */
+  assetInventoryStoreEnabled: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;
diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
index 90e76de4a6137..7f753171f567a 100644
--- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -1002,6 +1002,7 @@ components:
         - user
         - host
         - service
+        - universal
       type: string
     HostEntity:
       type: object
@@ -1072,6 +1073,7 @@ components:
         - host.name
         - user.name
         - service.name
+        - related.entity
       type: string
     IndexPattern:
       type: string
diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
index 736139e14a52b..07160dd6a44dc 100644
--- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
+++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -1002,6 +1002,7 @@ components:
         - user
         - host
         - service
+        - universal
       type: string
     HostEntity:
       type: object
@@ -1072,6 +1073,7 @@ components:
         - host.name
         - user.name
         - service.name
+        - related.entity
       type: string
     IndexPattern:
       type: string
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts
index 5ae034fe5d86d..f3aa9fa5b0d09 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/asset_criticality/helpers.ts
@@ -82,6 +82,7 @@ const entityTypeByIdField = {
   'host.name': 'host',
   'user.name': 'user',
   'service.name': 'service',
+  'related.entity': 'universal',
 } as const;
 
 export const getImplicitEntityFields = (record: AssetCriticalityUpsertWithDeleted) => {
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/component_template.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/component_template.ts
index a1352594ff47d..4a7365c729d95 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/component_template.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/component_template.ts
@@ -10,18 +10,24 @@ import {
   EngineComponentResourceEnum,
   type EngineComponentStatus,
 } from '../../../../../common/api/entity_analytics';
-import type { UnitedEntityDefinition } from '../united_entity_definitions';
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
 
 const getComponentTemplateName = (definitionId: string) => `${definitionId}-latest@platform`;
 
 interface Options {
-  unitedDefinition: UnitedEntityDefinition;
+  /**
+   * The entity engine description id
+   **/
+  id: string;
   esClient: ElasticsearchClient;
 }
 
-export const createEntityIndexComponentTemplate = ({ unitedDefinition, esClient }: Options) => {
-  const { entityManagerDefinition, indexMappings } = unitedDefinition;
-  const name = getComponentTemplateName(entityManagerDefinition.id);
+export const createEntityIndexComponentTemplate = (
+  description: EntityEngineInstallationDescriptor,
+  esClient: ElasticsearchClient
+) => {
+  const { id, indexMappings } = description;
+  const name = getComponentTemplateName(id);
   return esClient.cluster.putComponentTemplate({
     name,
     body: {
@@ -35,9 +41,8 @@ export const createEntityIndexComponentTemplate = ({ unitedDefinition, esClient
   });
 };
 
-export const deleteEntityIndexComponentTemplate = ({ unitedDefinition, esClient }: Options) => {
-  const { entityManagerDefinition } = unitedDefinition;
-  const name = getComponentTemplateName(entityManagerDefinition.id);
+export const deleteEntityIndexComponentTemplate = ({ id, esClient }: Options) => {
+  const name = getComponentTemplateName(id);
   return esClient.cluster.deleteComponentTemplate(
     { name },
     {
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts
index 7064a4dd98851..35a7fc40fc58d 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/enrich_policy.ts
@@ -6,12 +6,14 @@
  */
 
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
-import type { EnrichPutPolicyRequest } from '@elastic/elasticsearch/lib/api/types';
+import type { EntityType } from '../../../../../common/api/entity_analytics';
 import { EngineComponentResourceEnum } from '../../../../../common/api/entity_analytics';
 import { getEntitiesIndexName } from '../utils';
-import type { UnitedEntityDefinition } from '../united_entity_definitions';
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
 
-type DefinitionMetadata = Pick<UnitedEntityDefinition, 'namespace' | 'entityType' | 'version'>;
+type DefinitionMetadata = Pick<EntityEngineInstallationDescriptor, 'entityType' | 'version'> & {
+  namespace: string;
+};
 
 export const getFieldRetentionEnrichPolicyName = ({
   namespace,
@@ -21,41 +23,47 @@ export const getFieldRetentionEnrichPolicyName = ({
   return `entity_store_field_retention_${entityType}_${namespace}_v${version}`;
 };
 
-const getFieldRetentionEnrichPolicy = (
-  unitedDefinition: UnitedEntityDefinition
-): EnrichPutPolicyRequest => {
-  const { namespace, entityType, fieldRetentionDefinition } = unitedDefinition;
-  return {
-    name: getFieldRetentionEnrichPolicyName(unitedDefinition),
-    match: {
-      indices: getEntitiesIndexName(entityType, namespace),
-      match_field: fieldRetentionDefinition.matchField,
-      enrich_fields: fieldRetentionDefinition.fields.map(({ field }) => field),
-    },
-  };
-};
-
 export const createFieldRetentionEnrichPolicy = async ({
   esClient,
-  unitedDefinition,
+  description,
+  options,
 }: {
   esClient: ElasticsearchClient;
-  unitedDefinition: UnitedEntityDefinition;
+  description: EntityEngineInstallationDescriptor;
+  options: { namespace: string };
 }) => {
-  const policy = getFieldRetentionEnrichPolicy(unitedDefinition);
-  return esClient.enrich.putPolicy(policy);
+  return esClient.enrich.putPolicy({
+    name: getFieldRetentionEnrichPolicyName({
+      namespace: options.namespace,
+      entityType: description.entityType,
+      version: description.version,
+    }),
+    match: {
+      indices: getEntitiesIndexName(description.entityType, options.namespace),
+      match_field: description.identityField,
+      enrich_fields: description.fields.map(({ destination }) => destination),
+    },
+  });
 };
 
 export const executeFieldRetentionEnrichPolicy = async ({
   esClient,
-  unitedDefinition,
+  entityType,
+  version,
   logger,
+  options,
 }: {
-  unitedDefinition: DefinitionMetadata;
+  entityType: EntityType;
+  version: string;
   esClient: ElasticsearchClient;
   logger: Logger;
+  options: { namespace: string };
 }): Promise<{ executed: boolean }> => {
-  const name = getFieldRetentionEnrichPolicyName(unitedDefinition);
+  const name = getFieldRetentionEnrichPolicyName({
+    namespace: options.namespace,
+    entityType,
+    version,
+  });
   try {
     await esClient.enrich.executePolicy({ name });
     return { executed: true };
@@ -63,27 +71,31 @@ export const executeFieldRetentionEnrichPolicy = async ({
     if (e.statusCode === 404) {
       return { executed: false };
     }
-    logger.error(
-      `Error executing field retention enrich policy for ${unitedDefinition.entityType}: ${e.message}`
-    );
+    logger.error(`Error executing field retention enrich policy for ${entityType}: ${e.message}`);
     throw e;
   }
 };
 
 export const deleteFieldRetentionEnrichPolicy = async ({
-  unitedDefinition,
+  description,
+  options,
   esClient,
   logger,
   attempts = 5,
   delayMs = 2000,
 }: {
-  unitedDefinition: DefinitionMetadata;
+  description: EntityEngineInstallationDescriptor;
+  options: { namespace: string };
   esClient: ElasticsearchClient;
   logger: Logger;
   attempts?: number;
   delayMs?: number;
 }) => {
-  const name = getFieldRetentionEnrichPolicyName(unitedDefinition);
+  const name = getFieldRetentionEnrichPolicyName({
+    namespace: options.namespace,
+    entityType: description.entityType,
+    version: description.version,
+  });
   let currentAttempt = 1;
   while (currentAttempt <= attempts) {
     try {
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/entity_index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/entity_index.ts
index 0de8fe20050ce..719d2df7a9521 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/entity_index.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/entity_index.ts
@@ -6,6 +6,7 @@
  */
 
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
 import {
   EngineComponentResourceEnum,
   type EngineComponentStatus,
@@ -14,6 +15,8 @@ import {
 import { getEntitiesIndexName } from '../utils';
 import { createOrUpdateIndex } from '../../utils/create_or_update_index';
 
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
+
 interface Options {
   entityType: EntityType;
   esClient: ElasticsearchClient;
@@ -58,3 +61,51 @@ export const getEntityIndexStatus = async ({
 
   return { id: index, installed: exists, resource: EngineComponentResourceEnum.index };
 };
+
+export type MappingProperties = NonNullable<MappingTypeMapping['properties']>;
+
+export const generateIndexMappings = (
+  description: EntityEngineInstallationDescriptor
+): MappingTypeMapping => {
+  const identityFieldMappings: MappingProperties = {
+    [description.identityField]: {
+      type: 'keyword',
+      fields: {
+        text: {
+          type: 'match_only_text',
+        },
+      },
+    },
+  };
+
+  const otherFieldMappings = description.fields
+    .filter(({ mapping }) => mapping)
+    .reduce((acc, { destination, mapping }) => {
+      acc[destination] = mapping;
+      return acc;
+    }, {} as MappingProperties);
+
+  return {
+    properties: { ...BASE_ENTITY_INDEX_MAPPING, ...identityFieldMappings, ...otherFieldMappings },
+  };
+};
+
+export const BASE_ENTITY_INDEX_MAPPING: MappingProperties = {
+  '@timestamp': {
+    type: 'date',
+  },
+  'asset.criticality': {
+    type: 'keyword',
+  },
+  'entity.name': {
+    type: 'keyword',
+    fields: {
+      text: {
+        type: 'match_only_text',
+      },
+    },
+  },
+  'entity.source': {
+    type: 'keyword',
+  },
+};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_pipeline.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_pipeline.ts
index f57102fd13f14..c6b2a5902ec7f 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_pipeline.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_pipeline.ts
@@ -7,22 +7,22 @@
 
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
 import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
-import type { EntityDefinition } from '@kbn/entities-schema';
 import { EngineComponentResourceEnum } from '../../../../../common/api/entity_analytics';
-import { type FieldRetentionDefinition } from '../field_retention_definition';
+
 import {
   debugDeepCopyContextStep,
   getDotExpanderSteps,
   getRemoveEmptyFieldSteps,
   removeEntityDefinitionFieldsStep,
-  retentionDefinitionToIngestProcessorSteps,
 } from './ingest_processor_steps';
-import { getIdentityFieldForEntityType } from '../utils';
+
 import { getFieldRetentionEnrichPolicyName } from './enrich_policy';
-import type { UnitedEntityDefinition } from '../united_entity_definitions';
 
-const getPlatformPipelineId = (definition: EntityDefinition) => {
-  return `${definition.id}-latest@platform`;
+import { fieldOperatorToIngestProcessor } from '../field_retention';
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
+
+const getPlatformPipelineId = (descriptionId: string) => {
+  return `${descriptionId}-latest@platform`;
 };
 
 // the field that the enrich processor writes to
@@ -38,30 +38,28 @@ export const ENRICH_FIELD = 'historical';
  * and the context field in the document to help with debugging.
  */
 const buildIngestPipeline = ({
-  version,
-  fieldRetentionDefinition,
   allEntityFields,
   debugMode,
   namespace,
+  description,
 }: {
-  fieldRetentionDefinition: FieldRetentionDefinition;
   allEntityFields: string[];
   debugMode?: boolean;
   namespace: string;
   version: string;
+  description: EntityEngineInstallationDescriptor;
 }): IngestProcessorContainer[] => {
-  const { entityType, matchField } = fieldRetentionDefinition;
   const enrichPolicyName = getFieldRetentionEnrichPolicyName({
     namespace,
-    entityType,
-    version,
+    entityType: description.entityType,
+    version: description.version,
   });
-  return [
-    ...(debugMode ? [debugDeepCopyContextStep()] : []),
+
+  const processors = [
     {
       enrich: {
         policy_name: enrichPolicyName,
-        field: matchField,
+        field: description.identityField,
         target_field: ENRICH_FIELD,
       },
     },
@@ -74,14 +72,14 @@ const buildIngestPipeline = ({
     {
       set: {
         field: 'entity.name',
-        value: `{{${getIdentityFieldForEntityType(entityType)}}}`,
+        value: `{{${description.identityField}}}`,
       },
     },
     ...getDotExpanderSteps(allEntityFields),
-    ...retentionDefinitionToIngestProcessorSteps(fieldRetentionDefinition, {
-      enrichField: ENRICH_FIELD,
-    }),
-    ...getRemoveEmptyFieldSteps([...allEntityFields, 'asset', `${entityType}.risk`]),
+    ...description.fields.map((field) =>
+      fieldOperatorToIngestProcessor(field, { enrichField: ENRICH_FIELD })
+    ),
+    ...getRemoveEmptyFieldSteps([...allEntityFields, 'asset', `${description.entityType}.risk`]),
     removeEntityDefinitionFieldsStep(),
     ...(!debugMode
       ? [
@@ -94,43 +92,45 @@ const buildIngestPipeline = ({
         ]
       : []),
   ];
+
+  const extraSteps =
+    (typeof description.pipeline === 'function'
+      ? description.pipeline(processors)
+      : description.pipeline) ?? [];
+
+  return [...(debugMode ? [debugDeepCopyContextStep()] : []), ...processors, ...extraSteps];
 };
 
 // developing the pipeline is a bit tricky, so we have a debug mode
 // set  xpack.securitySolution.entityAnalytics.entityStore.developer.pipelineDebugMode
 // to true to keep the enrich field and the context field in the document to help with debugging.
 export const createPlatformPipeline = async ({
-  unitedDefinition,
   logger,
   esClient,
   debugMode,
+  description,
+  options,
 }: {
-  unitedDefinition: UnitedEntityDefinition;
+  description: EntityEngineInstallationDescriptor;
+  options: { namespace: string };
   logger: Logger;
   esClient: ElasticsearchClient;
   debugMode?: boolean;
 }) => {
-  const { fieldRetentionDefinition, entityManagerDefinition } = unitedDefinition;
-  const allEntityFields: string[] = (entityManagerDefinition?.metadata || []).map((m) => {
-    if (typeof m === 'string') {
-      return m;
-    }
-
-    return m.destination;
-  });
+  const allEntityFields = description.fields.map(({ destination }) => destination);
 
   const pipeline = {
-    id: getPlatformPipelineId(entityManagerDefinition),
+    id: getPlatformPipelineId(description.id),
     body: {
       _meta: {
         managed_by: 'entity_store',
         managed: true,
       },
-      description: `Ingest pipeline for entity definition ${entityManagerDefinition.id}`,
+      description: `Ingest pipeline for entity definition ${description.id}`,
       processors: buildIngestPipeline({
-        namespace: unitedDefinition.namespace,
-        version: unitedDefinition.version,
-        fieldRetentionDefinition,
+        namespace: options.namespace,
+        description,
+        version: description.version,
         allEntityFields,
         debugMode,
       }),
@@ -143,15 +143,15 @@ export const createPlatformPipeline = async ({
 };
 
 export const deletePlatformPipeline = ({
-  unitedDefinition,
+  description,
   logger,
   esClient,
 }: {
-  unitedDefinition: UnitedEntityDefinition;
+  description: EntityEngineInstallationDescriptor;
   logger: Logger;
   esClient: ElasticsearchClient;
 }) => {
-  const pipelineId = getPlatformPipelineId(unitedDefinition.entityManagerDefinition);
+  const pipelineId = getPlatformPipelineId(description.id);
   logger.debug(`Attempting to delete pipeline: ${pipelineId}`);
   return esClient.ingest.deletePipeline(
     {
@@ -164,13 +164,13 @@ export const deletePlatformPipeline = ({
 };
 
 export const getPlatformPipelineStatus = async ({
-  definition,
+  engineId,
   esClient,
 }: {
-  definition: EntityDefinition;
+  engineId: string;
   esClient: ElasticsearchClient;
 }) => {
-  const pipelineId = getPlatformPipelineId(definition);
+  const pipelineId = getPlatformPipelineId(engineId);
   const pipeline = await esClient.ingest.getPipeline(
     {
       id: pipelineId,
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/index.ts
index e8b715783b0a3..11fac0f9663ec 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/index.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/index.ts
@@ -9,4 +9,3 @@ export { debugDeepCopyContextStep } from './debug_deep_copy_context_step';
 export { getDotExpanderSteps } from './get_dot_expander_steps';
 export { getRemoveEmptyFieldSteps } from './get_remove_empty_field_steps';
 export { removeEntityDefinitionFieldsStep } from './remove_entity_definition_fields_step';
-export { retentionDefinitionToIngestProcessorSteps } from './retention_definition_to_ingest_processor_steps';
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/retention_definition_to_ingest_processor_steps.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/retention_definition_to_ingest_processor_steps.ts
deleted file mode 100644
index 2b6c5f555eb35..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/retention_definition_to_ingest_processor_steps.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
-import type {
-  FieldRetentionDefinition,
-  FieldRetentionOperatorBuilderOptions,
-} from '../../field_retention_definition';
-import { fieldOperatorToIngestProcessor } from '../../field_retention_definition';
-
-/**
- * Converts a field retention definition to the ingest processor steps
- * required to apply the field retention policy.
- */
-export const retentionDefinitionToIngestProcessorSteps = (
-  fieldRetentionDefinition: FieldRetentionDefinition,
-  options: FieldRetentionOperatorBuilderOptions
-): IngestProcessorContainer[] => {
-  return fieldRetentionDefinition.fields.map((field) =>
-    fieldOperatorToIngestProcessor(field, options)
-  );
-};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts
new file mode 100644
index 0000000000000..f24b6f6a0c4c1
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/constants.ts
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store';
+import {
+  HOST_DEFINITION_VERSION,
+  UNIVERSAL_DEFINITION_VERSION,
+  USER_DEFINITION_VERSION,
+  SERVICE_DEFINITION_VERSION,
+} from './entity_descriptions';
+
+export const VERSIONS_BY_ENTITY_TYPE: Record<EntityType, string> = {
+  host: HOST_DEFINITION_VERSION,
+  user: USER_DEFINITION_VERSION,
+  universal: UNIVERSAL_DEFINITION_VERSION,
+  service: SERVICE_DEFINITION_VERSION,
+};
+
+export const DEFAULT_FIELD_HISTORY_LENGTH = 10;
+export const DEFAULT_SYNC_DELAY = '1m';
+export const DEFAULT_FREQUENCY = '1m';
+export const DEFAULT_LOOKBACK_PERIOD = '1d';
+export const DEFAULT_TIMESTAMP_FIELD = '@timestamp';
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts
new file mode 100644
index 0000000000000..d7c966093ff87
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { EntityType } from '../../../../../../common/api/entity_analytics/entity_store';
+import type { FieldDescription } from '../../installation/types';
+
+import { oldestValue, newestValue } from './field_utils';
+
+export const getCommonFieldDescriptions = (entityType: EntityType): FieldDescription[] => {
+  return [
+    oldestValue({
+      source: '_index',
+      destination: 'entity.source',
+    }),
+    newestValue({ source: 'asset.criticality' }),
+    newestValue({
+      source: `${entityType}.risk.calculated_level`,
+    }),
+    newestValue({
+      source: `${entityType}.risk.calculated_score`,
+      mapping: {
+        type: 'float',
+      },
+    }),
+    newestValue({
+      source: `${entityType}.risk.calculated_score_norm`,
+      mapping: {
+        type: 'float',
+      },
+    }),
+  ];
+};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/field_utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/field_utils.ts
new file mode 100644
index 0000000000000..d19c56e4f7b2a
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/field_utils.ts
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
+import type { FieldDescription } from '../../installation/types';
+
+export const collectValues = ({
+  destination,
+  source,
+  fieldHistoryLength = 10,
+  mapping = { type: 'keyword' },
+}: {
+  source: string;
+  destination?: string;
+  mapping?: MappingProperty;
+  fieldHistoryLength?: number;
+}): FieldDescription => ({
+  destination: destination ?? source,
+  source,
+  retention: {
+    operation: 'collect_values',
+    maxLength: fieldHistoryLength,
+  },
+  aggregation: {
+    type: 'terms',
+    limit: fieldHistoryLength,
+  },
+  mapping,
+});
+
+export const newestValue = ({
+  destination,
+  mapping = { type: 'keyword' },
+  source,
+  sort,
+}: {
+  source: string;
+  destination?: string;
+  mapping?: MappingProperty;
+  sort?: Record<string, 'asc' | 'desc'>;
+}): FieldDescription => ({
+  destination: destination ?? source,
+  source,
+  retention: { operation: 'prefer_newest_value' },
+  aggregation: {
+    type: 'top_value',
+    sort: sort ?? {
+      '@timestamp': 'desc',
+    },
+  },
+  mapping,
+});
+
+export const oldestValue = ({
+  source,
+  destination,
+  mapping = { type: 'keyword' },
+  sort,
+}: {
+  source: string;
+  destination?: string;
+  mapping?: MappingProperty;
+  sort?: Record<string, 'asc' | 'desc'>;
+}): FieldDescription => ({
+  destination: destination ?? source,
+  source,
+  retention: { operation: 'prefer_oldest_value' },
+  aggregation: {
+    type: 'top_value',
+    sort: sort ?? {
+      '@timestamp': 'asc',
+    },
+  },
+  mapping,
+});
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/host.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/host.ts
new file mode 100644
index 0000000000000..7446f9a00911a
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/host.ts
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { collectValues as collect } from './field_utils';
+import type { EntityDescription } from '../types';
+import { getCommonFieldDescriptions } from './common';
+
+export const HOST_DEFINITION_VERSION = '1.0.0';
+export const HOST_IDENTITY_FIELD = 'host.name';
+export const hostEntityEngineDescription: EntityDescription = {
+  entityType: 'host',
+  version: HOST_DEFINITION_VERSION,
+  identityField: HOST_IDENTITY_FIELD,
+  settings: {
+    timestampField: '@timestamp',
+  },
+  fields: [
+    collect({ source: 'host.domain' }),
+    collect({ source: 'host.hostname' }),
+    collect({ source: 'host.id' }),
+    collect({
+      source: 'host.os.name',
+      mapping: {
+        type: 'keyword',
+        fields: {
+          text: {
+            type: 'match_only_text',
+          },
+        },
+      },
+    }),
+    collect({ source: 'host.os.type' }),
+    collect({
+      source: 'host.ip',
+      mapping: {
+        type: 'ip',
+      },
+    }),
+    collect({ source: 'host.mac' }),
+    collect({ source: 'host.type' }),
+    collect({ source: 'host.architecture' }),
+    ...getCommonFieldDescriptions('host'),
+  ],
+};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/index.ts
similarity index 79%
rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/index.ts
rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/index.ts
index cf78a4b0e363b..8984c4aac0eca 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/index.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/index.ts
@@ -8,4 +8,5 @@
 export * from './host';
 export * from './user';
 export * from './service';
-export { getCommonUnitedFieldDefinitions } from './common';
+export * from './universal';
+export { getCommonFieldDescriptions } from './common';
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/service.ts
new file mode 100644
index 0000000000000..86c5091d2844a
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/service.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { EntityDescription } from '../types';
+import { collectValues as collect, newestValue } from './field_utils';
+
+export const SERVICE_DEFINITION_VERSION = '1.0.0';
+export const SERVICE_IDENTITY_FIELD = 'service.name';
+
+export const serviceEntityEngineDescription: EntityDescription = {
+  entityType: 'service',
+  version: SERVICE_DEFINITION_VERSION,
+  identityField: SERVICE_IDENTITY_FIELD,
+  settings: {
+    timestampField: '@timestamp',
+  },
+  fields: [
+    collect({ source: 'service.address' }),
+    collect({ source: 'service.environment' }),
+    collect({ source: 'service.ephemeral_id' }),
+    collect({ source: 'service.id' }),
+    collect({ source: 'service.node.name' }),
+    collect({ source: 'service.node.roles' }),
+    newestValue({ source: 'service.state' }),
+    collect({ source: 'service.type' }),
+    newestValue({ source: 'service.version' }),
+  ],
+};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal.ts
new file mode 100644
index 0000000000000..d4e0997ee4f93
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/universal.ts
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { EntityDescription } from '../types';
+import { collectValues as collect } from './field_utils';
+
+export const UNIVERSAL_DEFINITION_VERSION = '1.0.0';
+export const UNIVERSAL_IDENTITY_FIELD = 'related.entity';
+
+const entityMetadataExtractorProcessor = {
+  script: {
+    tag: 'entity_metadata_extractor',
+    on_failure: [
+      {
+        set: {
+          field: 'error.message',
+          value:
+            'Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}',
+        },
+      },
+    ],
+    lang: 'painless',
+    source: /* java */ `
+      Map merged = ctx;
+      def id = ctx.entity.id;
+
+      for (meta in ctx.collected.metadata) {
+        Object json = Processors.json(meta);
+
+        if (((Map)json)[id] == null) {
+          continue;
+        }
+
+        for (entry in ((Map)json)[id].entrySet()) {
+          String key = entry.getKey();
+          Object value = entry.getValue();
+          merged.put(key, value);
+        }
+      }
+
+      merged.entity.id = id;
+      ctx = merged;
+    `,
+  },
+};
+
+export const universalEntityEngineDescription: EntityDescription = {
+  version: UNIVERSAL_DEFINITION_VERSION,
+  entityType: 'universal',
+  identityField: UNIVERSAL_IDENTITY_FIELD,
+  fields: [collect({ source: 'entities.keyword', destination: 'collected.metadata' })],
+  settings: {
+    timestampField: 'event.ingested',
+  },
+  pipeline: [entityMetadataExtractorProcessor],
+};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/user.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/user.ts
new file mode 100644
index 0000000000000..b79bdf9766d96
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/user.ts
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { EntityDescription } from '../types';
+import { getCommonFieldDescriptions } from './common';
+import { collectValues as collect } from './field_utils';
+
+export const USER_DEFINITION_VERSION = '1.0.0';
+export const USER_IDENTITY_FIELD = 'user.name';
+export const userEntityEngineDescription: EntityDescription = {
+  entityType: 'user',
+  version: USER_DEFINITION_VERSION,
+  identityField: USER_IDENTITY_FIELD,
+  settings: {
+    timestampField: '@timestamp',
+  },
+  fields: [
+    collect({ source: 'user.domain' }),
+    collect({ source: 'user.email' }),
+    collect({
+      source: 'user.full_name',
+      mapping: {
+        type: 'keyword',
+        fields: {
+          text: {
+            type: 'match_only_text',
+          },
+        },
+      },
+    }),
+    collect({ source: 'user.hash' }),
+    collect({ source: 'user.id' }),
+    collect({ source: 'user.roles' }),
+    ...getCommonFieldDescriptions('user'),
+  ],
+};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_manager_conversion.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_manager_conversion.ts
new file mode 100644
index 0000000000000..acc69c9bf9d33
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_manager_conversion.ts
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { pick } from 'lodash/fp';
+import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
+import { buildEntityDefinitionId } from '../utils';
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
+
+export const convertToEntityManagerDefinition = (
+  description: EntityEngineInstallationDescriptor,
+  options: { namespace: string; filter: string }
+): EntityDefinition => {
+  const metadata = description.fields.map(pick(['source', 'destination', 'aggregation']));
+
+  const definition = {
+    id: buildEntityDefinitionId(description.entityType, options.namespace),
+    name: `Security '${description.entityType}' Entity Store Definition`,
+    type: description.entityType,
+    indexPatterns: description.indexPatterns,
+    identityFields: [description.identityField],
+    displayNameTemplate: `{{${description.identityField}}}`,
+    metadata,
+    latest: {
+      timestampField: description.settings.timestampField,
+      lookbackPeriod: description.settings.lookbackPeriod,
+      settings: {
+        syncDelay: description.settings.syncDelay,
+        frequency: description.settings.frequency,
+      },
+    },
+    version: description.version,
+    managed: true,
+  };
+
+  return entityDefinitionSchema.parse(definition);
+};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/types.ts
new file mode 100644
index 0000000000000..e271c29e1d186
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/types.ts
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { EntityEngineInstallationDescriptor } from '../installation/types';
+
+type PickPartial<T, K extends keyof T, Optional extends K = never> = {
+  [P in K as P extends Optional ? never : P]: T[P];
+} & {
+  [P in K as P extends Optional ? P : never]?: Partial<T[P]>;
+};
+
+export type EntityDescription = PickPartial<
+  EntityEngineInstallationDescriptor,
+  | 'version'
+  | 'entityType'
+  | 'fields'
+  | 'identityField'
+  | 'indexPatterns'
+  | 'indexMappings'
+  | 'settings'
+  | 'pipeline',
+  'indexPatterns' | 'indexMappings' | 'settings' | 'pipeline'
+>;
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts
index b7cacea3c3757..8a1eaf9b5cc0c 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts
@@ -18,17 +18,27 @@ import type { AppClient } from '../../..';
 import type { EntityStoreConfig } from './types';
 import { mockGlobalState } from '../../../../public/common/mock';
 import type { EntityDefinition } from '@kbn/entities-schema';
-import { getUnitedEntityDefinition } from './united_entity_definitions';
-
-const unitedDefinition = getUnitedEntityDefinition({
-  entityType: 'host',
-  namespace: 'test',
-  fieldHistoryLength: 10,
-  indexPatterns: [],
-  syncDelay: '1m',
-  frequency: '1m',
-});
-const definition: EntityDefinition = unitedDefinition.entityManagerDefinition;
+import { convertToEntityManagerDefinition } from './entity_definitions/entity_manager_conversion';
+
+const definition: EntityDefinition = convertToEntityManagerDefinition(
+  {
+    id: 'host_engine',
+    entityType: 'host',
+    pipeline: [],
+    version: '0.0.1',
+    fields: [],
+    identityField: 'host.name',
+    indexMappings: {},
+    indexPatterns: [],
+    settings: {
+      syncDelay: '1m',
+      frequency: '1m',
+      timestampField: '@timestamp',
+      lookbackPeriod: '24h',
+    },
+  },
+  { namespace: 'test', filter: '' }
+);
 
 describe('EntityStoreDataClient', () => {
   const mockSavedObjectClient = savedObjectsClientMock.create();
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
index 8c4448558c86c..2c08d2e0f7f4c 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
@@ -51,7 +51,6 @@ import type {
 import { EngineDescriptorClient } from './saved_object/engine_descriptor';
 import { ENGINE_STATUS, ENTITY_STORE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
 import { AssetCriticalityMigrationClient } from '../asset_criticality/asset_criticality_migration_client';
-import { getUnitedEntityDefinition } from './united_entity_definitions';
 import {
   startEntityStoreFieldRetentionEnrichTask,
   removeEntityStoreFieldRetentionEnrichTask,
@@ -89,6 +88,8 @@ import {
   ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT,
 } from '../../telemetry/event_based/events';
 import { CRITICALITY_VALUES } from '../asset_criticality/constants';
+import { createEngineDescription } from './installation/engine_description';
+import { convertToEntityManagerDefinition } from './entity_definitions/entity_manager_conversion';
 
 // Workaround. TransformState type is wrong. The health type should be: TransformHealth from '@kbn/transform-plugin/common/types/transform_stats'
 export interface TransformHealth extends estypes.TransformGetTransformStatsTransformStatsHealth {
@@ -176,7 +177,7 @@ export class EntityStoreDataClient {
             ? [getEntityStoreFieldRetentionEnrichTaskStatus({ namespace, taskManager })]
             : []),
           getPlatformPipelineStatus({
-            definition,
+            engineId: definition.id,
             esClient: this.esClient,
           }),
           getFieldRetentionEnrichPolicyStatus({
@@ -213,9 +214,12 @@ export class EntityStoreDataClient {
       new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
 
     const { experimentalFeatures } = this.options;
-    const enginesTypes = experimentalFeatures.serviceEntityStoreEnabled
-      ? [EntityTypeEnum.host, EntityTypeEnum.user, EntityTypeEnum.service]
-      : [EntityTypeEnum.host, EntityTypeEnum.user];
+
+    const enginesTypes: EntityType[] = [EntityTypeEnum.host, EntityTypeEnum.user];
+    if (experimentalFeatures.serviceEntityStoreEnabled) {
+      enginesTypes.push(EntityTypeEnum.service);
+    }
+    // NOTE: Whilst the Universal Entity Store is also behind a feature flag, we do not want to enable it as part of this flow as of 8.18
 
     const promises = enginesTypes.map((entity) =>
       run(() =>
@@ -247,18 +251,15 @@ export class EntityStoreDataClient {
     if (withComponents) {
       const enginesWithComponents = await Promise.all(
         engines.map(async (engine) => {
-          const entityDefinitionId = buildEntityDefinitionId(engine.type, namespace);
+          const id = buildEntityDefinitionId(engine.type, namespace);
           const {
             definitions: [definition],
           } = await this.entityClient.getEntityDefinitions({
-            id: entityDefinitionId,
+            id,
             includeState: withComponents,
           });
 
-          const definitionComponents = this.getComponentFromEntityDefinition(
-            entityDefinitionId,
-            definition
-          );
+          const definitionComponents = this.getComponentFromEntityDefinition(id, definition);
 
           const entityStoreComponents = await this.getEngineComponentsState(
             engine.type,
@@ -283,6 +284,19 @@ export class EntityStoreDataClient {
     { indexPattern = '', filter = '', fieldHistoryLength = 10 }: InitEntityEngineRequestBody,
     { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
   ): Promise<InitEntityEngineResponse> {
+    const { experimentalFeatures } = this.options;
+
+    if (
+      entityType === EntityTypeEnum.universal &&
+      !experimentalFeatures.assetInventoryStoreEnabled
+    ) {
+      throw new Error('Universal entity store is not enabled');
+    }
+
+    if (entityType === EntityTypeEnum.service && !experimentalFeatures.serviceEntityStoreEnabled) {
+      throw new Error('Service entity store is not enabled');
+    }
+
     if (!this.options.taskManager) {
       throw new Error('Task Manager is not available');
     }
@@ -350,40 +364,34 @@ export class EntityStoreDataClient {
     const setupStartTime = moment().utc().toISOString();
     const { logger, namespace, appClient, dataViewsService } = this.options;
     try {
-      const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
+      const defaultIndexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
 
-      const unitedDefinition = getUnitedEntityDefinition({
-        indexPatterns,
+      const description = createEngineDescription({
         entityType,
         namespace,
-        fieldHistoryLength,
-        syncDelay: `${config.syncDelay.asSeconds()}s`,
-        frequency: `${config.frequency.asSeconds()}s`,
+        requestParams: { indexPattern, fieldHistoryLength },
+        defaultIndexPatterns,
+        config,
       });
-      const { entityManagerDefinition } = unitedDefinition;
 
       // clean up any existing entity store
       await this.delete(entityType, taskManager, { deleteData: false, deleteEngine: false });
 
       // set up the entity manager definition
+      const definition = convertToEntityManagerDefinition(description, {
+        namespace,
+        filter,
+      });
+
       await this.entityClient.createEntityDefinition({
-        definition: {
-          ...entityManagerDefinition,
-          filter,
-          indexPatterns: indexPattern
-            ? [...entityManagerDefinition.indexPatterns, ...indexPattern.split(',')]
-            : entityManagerDefinition.indexPatterns,
-        },
+        definition,
         installOnly: true,
       });
       this.log(`debug`, entityType, `Created entity definition`);
 
       // the index must be in place with the correct mapping before the enrich policy is created
       // this is because the enrich policy will fail if the index does not exist with the correct fields
-      await createEntityIndexComponentTemplate({
-        unitedDefinition,
-        esClient: this.esClient,
-      });
+      await createEntityIndexComponentTemplate(description, this.esClient);
       this.log(`debug`, entityType, `Created entity index component template`);
       await createEntityIndex({
         entityType,
@@ -396,20 +404,24 @@ export class EntityStoreDataClient {
       // we must create and execute the enrich policy before the pipeline is created
       // this is because the pipeline will fail if the enrich index does not exist
       await createFieldRetentionEnrichPolicy({
-        unitedDefinition,
+        description,
+        options: { namespace },
         esClient: this.esClient,
       });
       this.log(`debug`, entityType, `Created field retention enrich policy`);
 
       await executeFieldRetentionEnrichPolicy({
-        unitedDefinition,
+        entityType,
+        version: description.version,
+        options: { namespace },
         esClient: this.esClient,
         logger,
       });
       this.log(`debug`, entityType, `Executed field retention enrich policy`);
       await createPlatformPipeline({
+        description,
+        options: { namespace },
         debugMode: pipelineDebugMode,
-        unitedDefinition,
         logger,
         esClient: this.esClient,
       });
@@ -598,18 +610,18 @@ export class EntityStoreDataClient {
     const { deleteData, deleteEngine } = options;
 
     const descriptor = await this.engineClient.maybeGet(entityType);
-    const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
+    const defaultIndexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
 
-    // TODO delete unitedDefinition from this method. we only need the id for deletion
-    const unitedDefinition = getUnitedEntityDefinition({
-      indexPatterns,
+    const description = createEngineDescription({
       entityType,
-      namespace: this.options.namespace,
-      fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10,
-      syncDelay: `${config.syncDelay.asSeconds()}s`,
-      frequency: `${config.frequency.asSeconds()}s`,
+      namespace,
+      defaultIndexPatterns,
+      config,
     });
-    const { entityManagerDefinition } = unitedDefinition;
+
+    if (!description.id) {
+      throw new Error(`Unable to find entity definition for ${entityType}`);
+    }
 
     this.log('info', entityType, `Deleting entity store`);
     this.audit(
@@ -621,7 +633,7 @@ export class EntityStoreDataClient {
     try {
       await this.entityClient
         .deleteEntityDefinition({
-          id: entityManagerDefinition.id,
+          id: description.id,
           deleteData,
         })
         // Swallowing the error as it is expected to fail if no entity definition exists
@@ -631,20 +643,21 @@ export class EntityStoreDataClient {
       this.log('debug', entityType, `Deleted entity definition`);
 
       await deleteEntityIndexComponentTemplate({
-        unitedDefinition,
+        id: description.id,
         esClient: this.esClient,
       });
       this.log('debug', entityType, `Deleted entity index component template`);
 
       await deletePlatformPipeline({
-        unitedDefinition,
+        description,
         logger,
         esClient: this.esClient,
       });
       this.log('debug', entityType, `Deleted platform pipeline`);
 
       await deleteFieldRetentionEnrichPolicy({
-        unitedDefinition,
+        description,
+        options: { namespace },
         esClient: this.esClient,
         logger,
       });
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/collect_values.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/collect_values.ts
similarity index 77%
rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/collect_values.ts
rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/collect_values.ts
index ee26c0dbd64c8..1857e4430b18c 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/collect_values.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/collect_values.ts
@@ -6,29 +6,28 @@
  */
 
 import { isFieldMissingOrEmpty } from '../painless';
-import type { BaseFieldRetentionOperator, FieldRetentionOperatorBuilder } from './types';
-
-export interface CollectValues extends BaseFieldRetentionOperator {
-  operation: 'collect_values';
-  maxLength: number;
-}
+import type { FieldRetentionOperatorBuilder } from './types';
 
 /**
  * A field retention operator that collects up to `maxLength` values of the specified field.
  * Values are first collected from the field, then from the enrich field if the field is not present or empty.
  */
-export const collectValuesProcessor: FieldRetentionOperatorBuilder<CollectValues> = (
-  { field, maxLength },
+export const collectValuesProcessor: FieldRetentionOperatorBuilder = (
+  { destination, retention },
   { enrichField }
 ) => {
-  const ctxField = `ctx.${field}`;
-  const ctxEnrichField = `ctx.${enrichField}.${field}`;
+  if (retention?.operation !== 'collect_values') {
+    throw new Error('Wrong operation for collectValuesProcessor');
+  }
+
+  const ctxField = `ctx.${destination}`;
+  const ctxEnrichField = `ctx.${enrichField}.${destination}`;
   return {
     script: {
       lang: 'painless',
       source: `
   Set uniqueVals = new HashSet();
-  
+
   if (!(${isFieldMissingOrEmpty(ctxField)})) {
     if(${ctxField} instanceof Collection) {
       uniqueVals.addAll(${ctxField});
@@ -36,17 +35,17 @@ export const collectValuesProcessor: FieldRetentionOperatorBuilder<CollectValues
       uniqueVals.add(${ctxField});
     }
   }
-  
+
   if (uniqueVals.size() < params.max_length && !(${isFieldMissingOrEmpty(ctxEnrichField)})) {
     int remaining = params.max_length - uniqueVals.size();
     List historicalVals = ${ctxEnrichField}.subList(0, (int) Math.min(remaining, ${ctxEnrichField}.size()));
     uniqueVals.addAll(historicalVals);
   }
-  
+
   ${ctxField} = new ArrayList(uniqueVals).subList(0, (int) Math.min(params.max_length, uniqueVals.size()));
   `,
       params: {
-        max_length: maxLength,
+        max_length: retention.maxLength,
       },
     },
   };
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/index.ts
similarity index 100%
rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/index.ts
rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/index.ts
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/operator_to_ingest_processor.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/operator_to_ingest_processor.ts
similarity index 54%
rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/operator_to_ingest_processor.ts
rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/operator_to_ingest_processor.ts
index ee78f3e435830..37827c908ab0f 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/operator_to_ingest_processor.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/operator_to_ingest_processor.ts
@@ -5,24 +5,30 @@
  * 2.0.
  */
 
-import type { FieldRetentionOperator, FieldRetentionOperatorBuilder } from './types';
+import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
 import { preferNewestValueProcessor } from './prefer_newest_value';
 import { preferOldestValueProcessor } from './prefer_oldest_value';
 import { collectValuesProcessor } from './collect_values';
+import type { FieldDescription } from '../installation/types';
 
 /**
  * Converts a field retention operator to an ingest processor.
  * An ingest processor is a step that can be added to an ingest pipeline.
  */
-export const fieldOperatorToIngestProcessor: FieldRetentionOperatorBuilder<
-  FieldRetentionOperator
-> = (fieldOperator, options) => {
-  switch (fieldOperator.operation) {
+export const fieldOperatorToIngestProcessor = (
+  field: FieldDescription,
+  options: { enrichField: string }
+): IngestProcessorContainer => {
+  if (!field.retention) {
+    throw new Error('Field retention operator is required');
+  }
+
+  switch (field.retention.operation) {
     case 'prefer_newest_value':
-      return preferNewestValueProcessor(fieldOperator, options);
+      return preferNewestValueProcessor(field, options);
     case 'prefer_oldest_value':
-      return preferOldestValueProcessor(fieldOperator, options);
+      return preferOldestValueProcessor(field, options);
     case 'collect_values':
-      return collectValuesProcessor(fieldOperator, options);
+      return collectValuesProcessor(field, options);
   }
 };
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/prefer_newest_value.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/prefer_newest_value.ts
similarity index 68%
rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/prefer_newest_value.ts
rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/prefer_newest_value.ts
index 2378c67e3010d..fa156b976d062 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/prefer_newest_value.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/prefer_newest_value.ts
@@ -5,29 +5,25 @@
  * 2.0.
  */
 
-import type { BaseFieldRetentionOperator, FieldRetentionOperatorBuilder } from './types';
+import type { FieldRetentionOperatorBuilder } from './types';
 import { isFieldMissingOrEmpty } from '../painless';
 
-export interface PreferNewestValue extends BaseFieldRetentionOperator {
-  operation: 'prefer_newest_value';
-}
-
 /**
  * A field retention operator that prefers the newest value of the specified field.
  * If the field is missing or empty, the value from the enrich field is used.
  */
-export const preferNewestValueProcessor: FieldRetentionOperatorBuilder<PreferNewestValue> = (
-  { field },
+export const preferNewestValueProcessor: FieldRetentionOperatorBuilder = (
+  { destination },
   { enrichField }
 ) => {
-  const latestField = `ctx.${field}`;
-  const historicalField = `${enrichField}.${field}`;
+  const latestField = `ctx.${destination}`;
+  const historicalField = `${enrichField}.${destination}`;
   return {
     set: {
       if: `(${isFieldMissingOrEmpty(latestField)}) && !(${isFieldMissingOrEmpty(
         `ctx.${historicalField}`
       )})`,
-      field,
+      field: destination,
       value: `{{${historicalField}}}`,
     },
   };
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/prefer_oldest_value.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/prefer_oldest_value.ts
similarity index 69%
rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/prefer_oldest_value.ts
rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/prefer_oldest_value.ts
index a8a101d213ba9..c308fd2d50642 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/prefer_oldest_value.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/prefer_oldest_value.ts
@@ -5,26 +5,22 @@
  * 2.0.
  */
 
-import type { BaseFieldRetentionOperator, FieldRetentionOperatorBuilder } from './types';
+import type { FieldRetentionOperatorBuilder } from './types';
 import { isFieldMissingOrEmpty } from '../painless';
 
-export interface PreferOldestValue extends BaseFieldRetentionOperator {
-  operation: 'prefer_oldest_value';
-}
-
 /**
  * A field retention operator that prefers the oldest value of the specified field.
  * If the historical field is missing or empty, the latest value is used.
  */
-export const preferOldestValueProcessor: FieldRetentionOperatorBuilder<PreferOldestValue> = (
-  { field },
+export const preferOldestValueProcessor: FieldRetentionOperatorBuilder = (
+  { destination },
   { enrichField }
 ) => {
-  const historicalField = `${enrichField}.${field}`;
+  const historicalField = `${enrichField}.${destination}`;
   return {
     set: {
       if: `!(${isFieldMissingOrEmpty(`ctx.${historicalField}`)})`,
-      field,
+      field: destination,
       value: `{{${historicalField}}}`,
     },
   };
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/types.ts
new file mode 100644
index 0000000000000..088d943844d8c
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention/types.ts
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
+import type { FieldDescription } from '../installation/types';
+
+interface Options {
+  enrichField: string;
+}
+
+export type FieldRetentionOperatorBuilder = (
+  field: FieldDescription,
+  options: Options
+) => IngestProcessorContainer;
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/types.ts
deleted file mode 100644
index e07bd99525600..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/field_retention_definition/types.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
-import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store';
-import type { CollectValues } from './collect_values';
-import type { PreferNewestValue } from './prefer_newest_value';
-import type { PreferOldestValue } from './prefer_oldest_value';
-
-export interface FieldRetentionDefinition {
-  entityType: EntityType;
-  matchField: string;
-  fields: FieldRetentionOperator[];
-}
-
-export interface BaseFieldRetentionOperator {
-  field: string;
-  operation: string;
-}
-
-export interface FieldRetentionOperatorBuilderOptions {
-  enrichField: string;
-}
-
-export type FieldRetentionOperator = PreferNewestValue | PreferOldestValue | CollectValues;
-
-export type FieldRetentionOperatorBuilder<O extends BaseFieldRetentionOperator> = (
-  operator: O,
-  options: FieldRetentionOperatorBuilderOptions
-) => IngestProcessorContainer;
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.test.ts
similarity index 64%
rename from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts
rename to x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.test.ts
index 16e5e06aea8f2..81290e832978c 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.test.ts
@@ -5,22 +5,29 @@
  * 2.0.
  */
 
-import { getUnitedEntityDefinition } from './get_united_definition';
+import { duration } from 'moment';
+import { createEngineDescription } from './engine_description';
+import { convertToEntityManagerDefinition } from '../entity_definitions/entity_manager_conversion';
 
 describe('getUnitedEntityDefinition', () => {
-  const indexPatterns = ['test*'];
+  const defaultIndexPatterns = ['test*'];
   describe('host', () => {
-    const unitedDefinition = getUnitedEntityDefinition({
+    const description = createEngineDescription({
       entityType: 'host',
       namespace: 'test',
-      fieldHistoryLength: 10,
-      indexPatterns,
-      syncDelay: '1m',
-      frequency: '1m',
+      requestParams: {
+        fieldHistoryLength: 10,
+      },
+      defaultIndexPatterns,
+      config: {
+        syncDelay: duration(60, 'seconds'),
+        frequency: duration(60, 'seconds'),
+        developer: { pipelineDebugMode: false },
+      },
     });
 
     it('mapping', () => {
-      expect(unitedDefinition.indexMappings).toMatchInlineSnapshot(`
+      expect(description.indexMappings).toMatchInlineSnapshot(`
         Object {
           "properties": Object {
             "@timestamp": Object {
@@ -93,83 +100,14 @@ describe('getUnitedEntityDefinition', () => {
         }
       `);
     });
-    it('fieldRetentionDefinition', () => {
-      expect(unitedDefinition.fieldRetentionDefinition).toMatchInlineSnapshot(`
-        Object {
-          "entityType": "host",
-          "fields": Array [
-            Object {
-              "field": "host.domain",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "host.hostname",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "host.id",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "host.os.name",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "host.os.type",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "host.ip",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "host.mac",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "host.type",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "host.architecture",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "entity.source",
-              "operation": "prefer_oldest_value",
-            },
-            Object {
-              "field": "asset.criticality",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "host.risk.calculated_level",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "host.risk.calculated_score",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "host.risk.calculated_score_norm",
-              "operation": "prefer_newest_value",
-            },
-          ],
-          "matchField": "host.name",
-        }
-      `);
-    });
+
     it('entityManagerDefinition', () => {
-      expect(unitedDefinition.entityManagerDefinition).toMatchInlineSnapshot(`
+      const entityManagerDefinition = convertToEntityManagerDefinition(description, {
+        namespace: 'test',
+        filter: '',
+      });
+
+      expect(entityManagerDefinition).toMatchInlineSnapshot(`
         Object {
           "displayNameTemplate": "{{host.name}}",
           "id": "security_host_test",
@@ -183,10 +121,10 @@ describe('getUnitedEntityDefinition', () => {
             "test*",
           ],
           "latest": Object {
-            "lookbackPeriod": "24h",
+            "lookbackPeriod": "1d",
             "settings": Object {
-              "frequency": "1m",
-              "syncDelay": "1m",
+              "frequency": "60s",
+              "syncDelay": "60s",
             },
             "timestampField": "@timestamp",
           },
@@ -323,17 +261,22 @@ describe('getUnitedEntityDefinition', () => {
     });
   });
   describe('user', () => {
-    const unitedDefinition = getUnitedEntityDefinition({
+    const description = createEngineDescription({
       entityType: 'user',
       namespace: 'test',
-      fieldHistoryLength: 10,
-      indexPatterns,
-      syncDelay: '1m',
-      frequency: '1m',
+      requestParams: {
+        fieldHistoryLength: 10,
+      },
+      defaultIndexPatterns,
+      config: {
+        syncDelay: duration(60, 'seconds'),
+        frequency: duration(60, 'seconds'),
+        developer: { pipelineDebugMode: false },
+      },
     });
 
     it('mapping', () => {
-      expect(unitedDefinition.indexMappings).toMatchInlineSnapshot(`
+      expect(description.indexMappings).toMatchInlineSnapshot(`
         Object {
           "properties": Object {
             "@timestamp": Object {
@@ -397,68 +340,12 @@ describe('getUnitedEntityDefinition', () => {
         }
       `);
     });
-    it('fieldRetentionDefinition', () => {
-      expect(unitedDefinition.fieldRetentionDefinition).toMatchInlineSnapshot(`
-        Object {
-          "entityType": "user",
-          "fields": Array [
-            Object {
-              "field": "user.domain",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "user.email",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "user.full_name",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "user.hash",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "user.id",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "user.roles",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "entity.source",
-              "operation": "prefer_oldest_value",
-            },
-            Object {
-              "field": "asset.criticality",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "user.risk.calculated_level",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "user.risk.calculated_score",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "user.risk.calculated_score_norm",
-              "operation": "prefer_newest_value",
-            },
-          ],
-          "matchField": "user.name",
-        }
-      `);
-    });
     it('entityManagerDefinition', () => {
-      expect(unitedDefinition.entityManagerDefinition).toMatchInlineSnapshot(`
+      const entityManagerDefinition = convertToEntityManagerDefinition(description, {
+        namespace: 'test',
+        filter: '',
+      });
+      expect(entityManagerDefinition).toMatchInlineSnapshot(`
         Object {
           "displayNameTemplate": "{{user.name}}",
           "id": "security_user_test",
@@ -472,10 +359,10 @@ describe('getUnitedEntityDefinition', () => {
             "test*",
           ],
           "latest": Object {
-            "lookbackPeriod": "24h",
+            "lookbackPeriod": "1d",
             "settings": Object {
-              "frequency": "1m",
-              "syncDelay": "1m",
+              "frequency": "60s",
+              "syncDelay": "60s",
             },
             "timestampField": "@timestamp",
           },
@@ -589,17 +476,22 @@ describe('getUnitedEntityDefinition', () => {
   });
 
   describe('service', () => {
-    const unitedDefinition = getUnitedEntityDefinition({
+    const description = createEngineDescription({
       entityType: 'service',
       namespace: 'test',
-      fieldHistoryLength: 10,
-      indexPatterns,
-      syncDelay: '1m',
-      frequency: '1m',
+      requestParams: {
+        fieldHistoryLength: 10,
+      },
+      defaultIndexPatterns,
+      config: {
+        syncDelay: duration(60, 'seconds'),
+        frequency: duration(60, 'seconds'),
+        developer: { pipelineDebugMode: false },
+      },
     });
 
     it('mapping', () => {
-      expect(unitedDefinition.indexMappings).toMatchInlineSnapshot(`
+      expect(description.indexMappings).toMatchInlineSnapshot(`
         Object {
           "properties": Object {
             "@timestamp": Object {
@@ -645,15 +537,6 @@ describe('getUnitedEntityDefinition', () => {
             "service.node.roles": Object {
               "type": "keyword",
             },
-            "service.risk.calculated_level": Object {
-              "type": "keyword",
-            },
-            "service.risk.calculated_score": Object {
-              "type": "float",
-            },
-            "service.risk.calculated_score_norm": Object {
-              "type": "float",
-            },
             "service.state": Object {
               "type": "keyword",
             },
@@ -667,81 +550,13 @@ describe('getUnitedEntityDefinition', () => {
         }
       `);
     });
-    it('fieldRetentionDefinition', () => {
-      expect(unitedDefinition.fieldRetentionDefinition).toMatchInlineSnapshot(`
-        Object {
-          "entityType": "service",
-          "fields": Array [
-            Object {
-              "field": "service.address",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "service.environment",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "service.ephemeral_id",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "service.id",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "service.node.name",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "service.node.roles",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "service.state",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "service.type",
-              "maxLength": 10,
-              "operation": "collect_values",
-            },
-            Object {
-              "field": "service.version",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "entity.source",
-              "operation": "prefer_oldest_value",
-            },
-            Object {
-              "field": "asset.criticality",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "service.risk.calculated_level",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "service.risk.calculated_score",
-              "operation": "prefer_newest_value",
-            },
-            Object {
-              "field": "service.risk.calculated_score_norm",
-              "operation": "prefer_newest_value",
-            },
-          ],
-          "matchField": "service.name",
-        }
-      `);
-    });
+
     it('entityManagerDefinition', () => {
-      expect(unitedDefinition.entityManagerDefinition).toMatchInlineSnapshot(`
+      const entityManagerDefinition = convertToEntityManagerDefinition(description, {
+        namespace: 'test',
+        filter: '',
+      });
+      expect(entityManagerDefinition).toMatchInlineSnapshot(`
         Object {
           "displayNameTemplate": "{{service.name}}",
           "id": "security_service_test",
@@ -755,10 +570,10 @@ describe('getUnitedEntityDefinition', () => {
             "test*",
           ],
           "latest": Object {
-            "lookbackPeriod": "24h",
+            "lookbackPeriod": "1d",
             "settings": Object {
-              "frequency": "1m",
-              "syncDelay": "1m",
+              "frequency": "60s",
+              "syncDelay": "60s",
             },
             "timestampField": "@timestamp",
           },
@@ -840,56 +655,6 @@ describe('getUnitedEntityDefinition', () => {
               "destination": "service.version",
               "source": "service.version",
             },
-            Object {
-              "aggregation": Object {
-                "sort": Object {
-                  "@timestamp": "asc",
-                },
-                "type": "top_value",
-              },
-              "destination": "entity.source",
-              "source": "_index",
-            },
-            Object {
-              "aggregation": Object {
-                "sort": Object {
-                  "@timestamp": "desc",
-                },
-                "type": "top_value",
-              },
-              "destination": "asset.criticality",
-              "source": "asset.criticality",
-            },
-            Object {
-              "aggregation": Object {
-                "sort": Object {
-                  "@timestamp": "desc",
-                },
-                "type": "top_value",
-              },
-              "destination": "service.risk.calculated_level",
-              "source": "service.risk.calculated_level",
-            },
-            Object {
-              "aggregation": Object {
-                "sort": Object {
-                  "@timestamp": "desc",
-                },
-                "type": "top_value",
-              },
-              "destination": "service.risk.calculated_score",
-              "source": "service.risk.calculated_score",
-            },
-            Object {
-              "aggregation": Object {
-                "sort": Object {
-                  "@timestamp": "desc",
-                },
-                "type": "top_value",
-              },
-              "destination": "service.risk.calculated_score_norm",
-              "source": "service.risk.calculated_score_norm",
-            },
           ],
           "name": "Security 'service' Entity Store Definition",
           "type": "service",
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts
new file mode 100644
index 0000000000000..ccc4ba5f477a3
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { pipe } from 'fp-ts/lib/function';
+
+import { assign, concat, map, merge, update } from 'lodash/fp';
+import { set } from '@kbn/safer-lodash-set/fp';
+
+import type { EntityType } from '../../../../../common/api/entity_analytics';
+import {
+  DEFAULT_FIELD_HISTORY_LENGTH,
+  DEFAULT_LOOKBACK_PERIOD,
+  DEFAULT_TIMESTAMP_FIELD,
+} from '../entity_definitions/constants';
+import { generateIndexMappings } from '../elasticsearch_assets';
+import {
+  hostEntityEngineDescription,
+  userEntityEngineDescription,
+  universalEntityEngineDescription,
+  serviceEntityEngineDescription,
+} from '../entity_definitions/entity_descriptions';
+
+import type { EntityStoreConfig } from '../types';
+import { buildEntityDefinitionId } from '../utils';
+import type { EntityDescription } from '../entity_definitions/types';
+import type { EntityEngineInstallationDescriptor } from './types';
+
+const engineDescriptionRegistry: Record<EntityType, EntityDescription> = {
+  host: hostEntityEngineDescription,
+  user: userEntityEngineDescription,
+  universal: universalEntityEngineDescription,
+  service: serviceEntityEngineDescription,
+};
+export const getAvailableEntityDescriptions = () =>
+  Object.keys(engineDescriptionRegistry) as EntityType[];
+
+interface EngineDescriptionParams {
+  entityType: EntityType;
+  namespace: string;
+  config: EntityStoreConfig;
+  requestParams?: {
+    indexPattern?: string;
+    fieldHistoryLength?: number;
+  };
+  defaultIndexPatterns: string[];
+}
+
+export const createEngineDescription = (options: EngineDescriptionParams) => {
+  const { entityType, namespace, config, requestParams, defaultIndexPatterns } = options;
+  const fieldHistoryLength = requestParams?.fieldHistoryLength || DEFAULT_FIELD_HISTORY_LENGTH;
+
+  const indexPatterns = requestParams?.indexPattern
+    ? defaultIndexPatterns.concat(requestParams?.indexPattern.split(','))
+    : defaultIndexPatterns;
+
+  const description = engineDescriptionRegistry[entityType];
+
+  const settings: EntityEngineInstallationDescriptor['settings'] = {
+    syncDelay: `${config.syncDelay.asSeconds()}s`,
+    frequency: `${config.frequency.asSeconds()}s`,
+    lookbackPeriod: description.settings?.lookbackPeriod || DEFAULT_LOOKBACK_PERIOD,
+    timestampField: description.settings?.timestampField || DEFAULT_TIMESTAMP_FIELD,
+  };
+
+  const updatedDescription = pipe(
+    description,
+    set('id', buildEntityDefinitionId(entityType, namespace)),
+    update('settings', assign(settings)),
+    updateIndexPatterns(indexPatterns),
+    updateRetentionFields(fieldHistoryLength),
+    addIndexMappings
+  ) as EntityEngineInstallationDescriptor;
+
+  return updatedDescription;
+};
+
+const updateIndexPatterns = (indexPatterns: string[]) =>
+  update('indexPatterns', (prev = []) => concat(indexPatterns, prev));
+
+const updateRetentionFields = (fieldHistoryLength: number) =>
+  update(
+    'fields',
+    map(
+      merge({
+        retention: { maxLength: fieldHistoryLength },
+        aggregation: { limit: fieldHistoryLength },
+      })
+    )
+  );
+
+const addIndexMappings = (description: EntityEngineInstallationDescriptor) =>
+  set('indexMappings', generateIndexMappings(description), description);
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/types.ts
new file mode 100644
index 0000000000000..b68ce5faa1ac4
--- /dev/null
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/types.ts
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  MappingTypeMapping,
+  IngestProcessorContainer,
+  MappingProperty,
+} from '@elastic/elasticsearch/lib/api/types';
+
+import type { EntityDefinition } from '@kbn/entities-schema';
+import type { EntityType } from '../../../../../common/api/entity_analytics';
+
+export type EntityDefinitionMetadataElement = NonNullable<EntityDefinition['metadata']>[number];
+
+export interface EntityEngineInstallationDescriptor {
+  id: string;
+  version: string;
+  entityType: EntityType;
+
+  identityField: string;
+
+  /**
+   * Default static index patterns to use as the source of entity data.
+   * By default, the Security Data View patterns will be added to this list.
+   * API parameters can be used to add additional patterns.
+   **/
+  indexPatterns: string[];
+
+  /**
+   * The mappings for the entity store index.
+   */
+  indexMappings: MappingTypeMapping;
+
+  /**
+   * Field descriptions for the entity.
+   * Identity fields are not included here as they are treated separately.
+   */
+  fields: FieldDescription[];
+
+  /**
+   * Entity manager default pivot transform settings.
+   * Any kibana.yml configuration will override these settings.
+   */
+  settings: {
+    syncDelay: string;
+    frequency: string;
+    lookbackPeriod: string;
+    timestampField: string;
+  };
+
+  /**
+   * The ingest pipeline to apply to the entity data.
+   * This can be an array of processors which get appended to the default pipeline,
+   * or a function that takes the default processors and returns an array of processors.
+   **/
+  pipeline:
+    | IngestProcessorContainer[]
+    | ((defaultProcessors: IngestProcessorContainer[]) => IngestProcessorContainer[]);
+}
+
+export type FieldDescription = EntityDefinitionMetadataElement & {
+  mapping: MappingProperty;
+  retention: FieldRetentionOp;
+};
+
+export type FieldRetentionOp =
+  | { operation: 'collect_values'; maxLength: number }
+  | { operation: 'prefer_newest_value' }
+  | { operation: 'prefer_oldest_value' };
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/task/field_retention_enrichment_task.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/task/field_retention_enrichment_task.ts
index d60f994488fca..3b802fc081e1a 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/task/field_retention_enrichment_task.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/task/field_retention_enrichment_task.ts
@@ -13,7 +13,6 @@ import type {
   TaskManagerSetupContract,
   TaskManagerStartContract,
 } from '@kbn/task-manager-plugin/server';
-import { getAvailableEntityTypes } from '../../../../../common/entity_analytics/entity_store/constants';
 import {
   EngineComponentResourceEnum,
   type EntityType,
@@ -25,7 +24,7 @@ import {
 } from './state';
 import { INTERVAL, SCOPE, TIMEOUT, TYPE, VERSION } from './constants';
 import type { EntityAnalyticsRoutesDeps } from '../../types';
-import { getUnitedEntityDefinitionVersion } from '../united_entity_definitions';
+
 import { executeFieldRetentionEnrichPolicy } from '../elasticsearch_assets';
 
 import { getEntitiesIndexName } from '../utils';
@@ -33,6 +32,8 @@ import {
   FIELD_RETENTION_ENRICH_POLICY_EXECUTION_EVENT,
   ENTITY_STORE_USAGE_EVENT,
 } from '../../../telemetry/event_based/events';
+import { VERSIONS_BY_ENTITY_TYPE } from '../entity_definitions/constants';
+import { getAvailableEntityDescriptions } from '../installation/engine_description';
 
 const logFactory =
   (logger: Logger, taskId: string) =>
@@ -79,10 +80,10 @@ export const registerEntityStoreFieldRetentionEnrichTask = ({
     const [coreStart, _] = await getStartServices();
     const esClient = coreStart.elasticsearch.client.asInternalUser;
 
-    const unitedDefinitionVersion = getUnitedEntityDefinitionVersion(entityType);
-
     return executeFieldRetentionEnrichPolicy({
-      unitedDefinition: { namespace, entityType, version: unitedDefinitionVersion },
+      entityType,
+      version: VERSIONS_BY_ENTITY_TYPE[entityType],
+      options: { namespace },
       esClient,
       logger,
     });
@@ -202,7 +203,7 @@ export const runTask = async ({
       return { state: updatedState };
     }
 
-    const entityTypes = getAvailableEntityTypes();
+    const entityTypes = getAvailableEntityDescriptions();
 
     for (const entityType of entityTypes) {
       const start = Date.now();
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/constants.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/constants.ts
deleted file mode 100644
index 43730fa06357a..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/constants.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { MappingProperties } from './types';
-
-export const BASE_ENTITY_INDEX_MAPPING: MappingProperties = {
-  '@timestamp': {
-    type: 'date',
-  },
-  'asset.criticality': {
-    type: 'keyword',
-  },
-  'entity.name': {
-    type: 'keyword',
-    fields: {
-      text: {
-        type: 'match_only_text',
-      },
-    },
-  },
-  'entity.source': {
-    type: 'keyword',
-  },
-};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/definition_utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/definition_utils.ts
deleted file mode 100644
index a56dcaa253857..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/definition_utils.ts
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
-import type { UnitedDefinitionField } from './types';
-
-export const collectValues = ({
-  field,
-  mapping = { type: 'keyword' },
-  sourceField,
-  fieldHistoryLength,
-}: {
-  field: string;
-  mapping?: MappingProperty;
-  sourceField?: string;
-  fieldHistoryLength: number;
-}): UnitedDefinitionField => ({
-  field,
-  definition: {
-    source: sourceField ?? field,
-    destination: field,
-    aggregation: {
-      type: 'terms',
-      limit: fieldHistoryLength,
-    },
-  },
-  retention_operator: { operation: 'collect_values', field, maxLength: fieldHistoryLength },
-  mapping,
-});
-
-export const collectValuesWithLength =
-  (fieldHistoryLength: number) =>
-  (opts: { field: string; mapping?: MappingProperty; sourceField?: string }) =>
-    collectValues({ ...opts, fieldHistoryLength });
-
-export const newestValue = ({
-  field,
-  mapping = { type: 'keyword' },
-  sourceField,
-}: {
-  field: string;
-  mapping?: MappingProperty;
-  sourceField?: string;
-}): UnitedDefinitionField => ({
-  field,
-  definition: {
-    source: sourceField ?? field,
-    destination: field,
-    aggregation: {
-      type: 'top_value',
-      sort: {
-        '@timestamp': 'desc',
-      },
-    },
-  },
-  retention_operator: { operation: 'prefer_newest_value', field },
-  mapping,
-});
-
-export const oldestValue = ({
-  field,
-  mapping = { type: 'keyword' },
-  sourceField,
-}: {
-  field: string;
-  mapping?: MappingProperty;
-  sourceField?: string;
-}): UnitedDefinitionField => ({
-  field,
-  definition: {
-    source: sourceField ?? field,
-    destination: field,
-    aggregation: {
-      type: 'top_value',
-      sort: {
-        '@timestamp': 'asc',
-      },
-    },
-  },
-  retention_operator: { operation: 'prefer_oldest_value', field },
-  mapping,
-});
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/common.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/common.ts
deleted file mode 100644
index ac974bf119d4a..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/common.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { EntityType } from '../../../../../../common/api/entity_analytics/entity_store';
-import { getIdentityFieldForEntityType } from '../../utils';
-import { oldestValue, newestValue } from '../definition_utils';
-import type { UnitedDefinitionField } from '../types';
-
-export const getCommonUnitedFieldDefinitions = ({
-  entityType,
-  fieldHistoryLength,
-}: {
-  entityType: EntityType;
-  fieldHistoryLength: number;
-}): UnitedDefinitionField[] => {
-  const identityField = getIdentityFieldForEntityType(entityType);
-  return [
-    oldestValue({
-      sourceField: '_index',
-      field: 'entity.source',
-    }),
-    newestValue({ field: 'asset.criticality' }),
-    newestValue({
-      field: `${entityType}.risk.calculated_level`,
-    }),
-    newestValue({
-      field: `${entityType}.risk.calculated_score`,
-      mapping: {
-        type: 'float',
-      },
-    }),
-    newestValue({
-      field: `${entityType}.risk.calculated_score_norm`,
-      mapping: {
-        type: 'float',
-      },
-    }),
-    {
-      field: identityField,
-    },
-  ];
-};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/host.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/host.ts
deleted file mode 100644
index 5abeffd05b1f1..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/host.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { collectValuesWithLength } from '../definition_utils';
-import type { UnitedDefinitionBuilder } from '../types';
-
-export const HOST_DEFINITION_VERSION = '1.0.0';
-export const getHostUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
-  const collect = collectValuesWithLength(fieldHistoryLength);
-  return {
-    entityType: 'host',
-    version: HOST_DEFINITION_VERSION,
-    fields: [
-      collect({ field: 'host.domain' }),
-      collect({ field: 'host.hostname' }),
-      collect({ field: 'host.id' }),
-      collect({
-        field: 'host.os.name',
-        mapping: {
-          type: 'keyword',
-          fields: {
-            text: {
-              type: 'match_only_text',
-            },
-          },
-        },
-      }),
-      collect({ field: 'host.os.type' }),
-      collect({
-        field: 'host.ip',
-        mapping: {
-          type: 'ip',
-        },
-      }),
-      collect({ field: 'host.mac' }),
-      collect({ field: 'host.type' }),
-      collect({ field: 'host.architecture' }),
-    ],
-  };
-};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts
deleted file mode 100644
index 26d10046b26ae..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { collectValuesWithLength, newestValue } from '../definition_utils';
-import type { UnitedDefinitionBuilder } from '../types';
-
-export const SERVICE_DEFINITION_VERSION = '1.0.0';
-export const getServiceUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
-  const collect = collectValuesWithLength(fieldHistoryLength);
-  return {
-    entityType: 'service',
-    version: SERVICE_DEFINITION_VERSION,
-    fields: [
-      collect({ field: 'service.address' }),
-      collect({ field: 'service.environment' }),
-      collect({ field: 'service.ephemeral_id' }),
-      collect({ field: 'service.id' }),
-      collect({ field: 'service.node.name' }),
-      collect({ field: 'service.node.roles' }),
-      newestValue({ field: 'service.state' }),
-      collect({ field: 'service.type' }),
-      newestValue({ field: 'service.version' }),
-    ],
-  };
-};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/user.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/user.ts
deleted file mode 100644
index 3881425df77ce..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/user.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-import { collectValuesWithLength } from '../definition_utils';
-import type { UnitedDefinitionBuilder } from '../types';
-
-export const USER_DEFINITION_VERSION = '1.0.0';
-export const getUserUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
-  const collect = collectValuesWithLength(fieldHistoryLength);
-  return {
-    entityType: 'user',
-    version: USER_DEFINITION_VERSION,
-    fields: [
-      collect({ field: 'user.domain' }),
-      collect({ field: 'user.email' }),
-      collect({
-        field: 'user.full_name',
-        mapping: {
-          type: 'keyword',
-          fields: {
-            text: {
-              type: 'match_only_text',
-            },
-          },
-        },
-      }),
-      collect({ field: 'user.hash' }),
-      collect({ field: 'user.id' }),
-      collect({ field: 'user.roles' }),
-    ],
-  };
-};
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts
deleted file mode 100644
index 28685172dc7d4..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-import { memoize } from 'lodash';
-import type { EntityType } from '../../../../../common/api/entity_analytics';
-import {
-  getHostUnitedDefinition,
-  getUserUnitedDefinition,
-  getCommonUnitedFieldDefinitions,
-  USER_DEFINITION_VERSION,
-  HOST_DEFINITION_VERSION,
-  getServiceUnitedDefinition,
-} from './entity_types';
-import type { UnitedDefinitionBuilder } from './types';
-import { UnitedEntityDefinition } from './united_entity_definition';
-const unitedDefinitionBuilders: Record<EntityType, UnitedDefinitionBuilder> = {
-  host: getHostUnitedDefinition,
-  user: getUserUnitedDefinition,
-  service: getServiceUnitedDefinition,
-};
-
-interface Options {
-  entityType: EntityType;
-  namespace: string;
-  fieldHistoryLength: number;
-  indexPatterns: string[];
-  syncDelay: string;
-  frequency: string;
-}
-
-export const getUnitedEntityDefinition = memoize(
-  ({
-    entityType,
-    namespace,
-    fieldHistoryLength,
-    indexPatterns,
-    syncDelay,
-    frequency,
-  }: Options): UnitedEntityDefinition => {
-    const unitedDefinition = unitedDefinitionBuilders[entityType](fieldHistoryLength);
-
-    unitedDefinition.fields.push(
-      ...getCommonUnitedFieldDefinitions({
-        entityType,
-        fieldHistoryLength,
-      })
-    );
-
-    return new UnitedEntityDefinition({
-      ...unitedDefinition,
-      namespace,
-      indexPatterns,
-      syncDelay,
-      frequency,
-    });
-  }
-);
-
-const versionByEntityType: Record<EntityType, string> = {
-  host: HOST_DEFINITION_VERSION,
-  user: USER_DEFINITION_VERSION,
-  service: USER_DEFINITION_VERSION,
-};
-
-export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string =>
-  versionByEntityType[entityType];
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/index.ts
deleted file mode 100644
index 5280e066d5b81..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/index.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-export * from './get_united_definition';
-
-export { UnitedEntityDefinition } from './united_entity_definition';
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/types.ts
deleted file mode 100644
index c48c6b09a8e56..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/types.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { MappingProperty, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
-import type { EntityDefinition } from '@kbn/entities-schema';
-import type { EntityType } from '../../../../../common/api/entity_analytics';
-import type { FieldRetentionOperator } from '../field_retention_definition';
-
-export type MappingProperties = NonNullable<MappingTypeMapping['properties']>;
-
-type EntityDefinitionMetadataElement = NonNullable<EntityDefinition['metadata']>[number];
-
-export interface UnitedDefinitionField {
-  field: string;
-  retention_operator?: FieldRetentionOperator;
-  mapping?: MappingProperty;
-  definition?: EntityDefinitionMetadataElement;
-}
-
-export interface UnitedEntityDefinitionConfig {
-  version: string;
-  entityType: EntityType;
-  fields: UnitedDefinitionField[];
-}
-
-export type UnitedDefinitionBuilder = (fieldHistoryLength: number) => UnitedEntityDefinitionConfig;
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts
deleted file mode 100644
index fc7430ebb1806..0000000000000
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
-import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
-import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
-import { DEFAULT_LOOKBACK_PERIOD } from '../constants';
-import { buildEntityDefinitionId, getIdentityFieldForEntityType } from '../utils';
-import type {
-  FieldRetentionDefinition,
-  FieldRetentionOperator,
-} from '../field_retention_definition';
-import type { MappingProperties, UnitedDefinitionField } from './types';
-import { BASE_ENTITY_INDEX_MAPPING } from './constants';
-
-export class UnitedEntityDefinition {
-  version: string;
-  entityType: EntityType;
-  indexPatterns: string[];
-  fields: UnitedDefinitionField[];
-  namespace: string;
-  entityManagerDefinition: EntityDefinition;
-  fieldRetentionDefinition: FieldRetentionDefinition;
-  indexMappings: MappingTypeMapping;
-  syncDelay: string;
-  frequency: string;
-
-  constructor(opts: {
-    version: string;
-    entityType: EntityType;
-    indexPatterns: string[];
-    fields: UnitedDefinitionField[];
-    namespace: string;
-    syncDelay: string;
-    frequency: string;
-  }) {
-    this.version = opts.version;
-    this.entityType = opts.entityType;
-    this.indexPatterns = opts.indexPatterns;
-    this.fields = opts.fields;
-    this.frequency = opts.frequency;
-    this.syncDelay = opts.syncDelay;
-    this.namespace = opts.namespace;
-    this.entityManagerDefinition = this.toEntityManagerDefinition();
-    this.fieldRetentionDefinition = this.toFieldRetentionDefinition();
-    this.indexMappings = this.toIndexMappings();
-  }
-
-  private toEntityManagerDefinition(): EntityDefinition {
-    const { entityType, namespace, indexPatterns, syncDelay, frequency } = this;
-    const identityField = getIdentityFieldForEntityType(this.entityType);
-    const metadata = this.fields
-      .filter((field) => field.definition)
-      .map((field) => field.definition!); // eslint-disable-line @typescript-eslint/no-non-null-assertion
-
-    return entityDefinitionSchema.parse({
-      id: buildEntityDefinitionId(entityType, namespace),
-      name: `Security '${entityType}' Entity Store Definition`,
-      type: entityType,
-      indexPatterns,
-      identityFields: [identityField],
-      displayNameTemplate: `{{${identityField}}}`,
-      metadata,
-      latest: {
-        timestampField: '@timestamp',
-        lookbackPeriod: DEFAULT_LOOKBACK_PERIOD,
-        settings: {
-          syncDelay,
-          frequency,
-        },
-      },
-      version: this.version,
-      managed: true,
-    });
-  }
-
-  private toFieldRetentionDefinition(): FieldRetentionDefinition {
-    return {
-      entityType: this.entityType,
-      matchField: getIdentityFieldForEntityType(this.entityType),
-      fields: this.fields
-        .filter((field) => field.retention_operator !== undefined)
-        .map((field) => field.retention_operator as FieldRetentionOperator),
-    };
-  }
-
-  private toIndexMappings(): MappingTypeMapping {
-    const identityField = getIdentityFieldForEntityType(this.entityType);
-
-    const initialMappings: MappingProperties = {
-      ...BASE_ENTITY_INDEX_MAPPING,
-      [identityField]: {
-        type: 'keyword',
-        fields: {
-          text: {
-            type: 'match_only_text',
-          },
-        },
-      },
-    };
-
-    const properties = this.fields.reduce((acc, { field, mapping }) => {
-      if (!mapping) {
-        return acc;
-      }
-      acc[field] = mapping;
-      return acc;
-    }, initialMappings);
-
-    properties[identityField] = {
-      type: 'keyword',
-      fields: {
-        text: {
-          type: 'match_only_text',
-        },
-      },
-    };
-
-    return {
-      properties,
-    };
-  }
-}
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts
index c889d3a8ce34b..5844f539bcad3 100644
--- a/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts
+++ b/x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts
@@ -11,17 +11,12 @@ import {
   entitiesIndexPattern,
 } from '@kbn/entities-schema';
 import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common';
-import { IDENTITY_FIELD_MAP } from '../../../../../common/entity_analytics/entity_store/constants';
 import type { AppClient } from '../../../../types';
 import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
 import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
 import { type EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
 import { entityEngineDescriptorTypeName } from '../saved_object';
 
-export const getIdentityFieldForEntityType = (entityType: EntityType) => {
-  return IDENTITY_FIELD_MAP[entityType];
-};
-
 export const buildIndexPatterns = async (
   space: string,
   appClient: AppClient,
diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts
index 6e8f7dc3f6578..1bb0205a75c81 100644
--- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/field_retention_operators.ts
@@ -6,10 +6,8 @@
  */
 
 import expect from '@kbn/expect';
-import {
-  FieldRetentionOperator,
-  fieldOperatorToIngestProcessor,
-} from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/field_retention_definition';
+import { fieldOperatorToIngestProcessor } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/field_retention';
+import { FieldDescription } from '@kbn/security-solution-plugin/server/lib/entity_analytics/entity_store/installation/types';
 import { FtrProviderContext } from '../../../../ftr_provider_context';
 import { applyIngestProcessorToDoc } from '../utils/ingest';
 export default ({ getService }: FtrProviderContext) => {
@@ -22,10 +20,7 @@ export default ({ getService }: FtrProviderContext) => {
     expect(aSorted).to.eql(bSorted);
   };
 
-  const applyOperatorToDoc = async (
-    operator: FieldRetentionOperator,
-    docSource: any
-  ): Promise<any> => {
+  const applyOperatorToDoc = async (operator: FieldDescription, docSource: any): Promise<any> => {
     const step = fieldOperatorToIngestProcessor(operator, { enrichField: 'historical' });
 
     return applyIngestProcessorToDoc([step], docSource, es, log);
@@ -34,10 +29,16 @@ export default ({ getService }: FtrProviderContext) => {
   describe('@ess @serverless @skipInServerlessMKI Entity store - Field Retention Pipeline Steps', () => {
     describe('collect_values operator', () => {
       it('should return value if no history', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'collect_values',
-          field: 'test_field',
-          maxLength: 10,
+        const op: FieldDescription = {
+          retention: { operation: 'collect_values', maxLength: 10 },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -50,10 +51,16 @@ export default ({ getService }: FtrProviderContext) => {
       });
 
       it('should not take from history if latest field has maxLength values', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'collect_values',
-          field: 'test_field',
-          maxLength: 1,
+        const op: FieldDescription = {
+          retention: { operation: 'collect_values', maxLength: 1 },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -69,10 +76,16 @@ export default ({ getService }: FtrProviderContext) => {
       });
 
       it("should take from history if latest field doesn't have maxLength values", async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'collect_values',
-          field: 'test_field',
-          maxLength: 10,
+        const op: FieldDescription = {
+          retention: { operation: 'collect_values', maxLength: 10 },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -88,10 +101,16 @@ export default ({ getService }: FtrProviderContext) => {
       });
 
       it('should only take from history up to maxLength values', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'collect_values',
-          field: 'test_field',
-          maxLength: 2,
+        const op: FieldDescription = {
+          retention: { operation: 'collect_values', maxLength: 2 },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -107,10 +126,16 @@ export default ({ getService }: FtrProviderContext) => {
       });
 
       it('should handle value not being an array', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'collect_values',
-          field: 'test_field',
-          maxLength: 2,
+        const op: FieldDescription = {
+          retention: { operation: 'collect_values', maxLength: 2 },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
         const doc = {
           test_field: 'foo',
@@ -125,10 +150,16 @@ export default ({ getService }: FtrProviderContext) => {
       });
 
       it('should handle missing values', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'collect_values',
-          field: 'test_field',
-          maxLength: 2,
+        const op: FieldDescription = {
+          retention: { operation: 'collect_values', maxLength: 2 },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
         const doc = {};
 
@@ -139,9 +170,16 @@ export default ({ getService }: FtrProviderContext) => {
     });
     describe('prefer_newest_value operator', () => {
       it('should return latest value if no history value', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_newest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_newest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -153,9 +191,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('latest');
       });
       it('should return history value if no latest value (undefined)', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_newest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_newest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -169,9 +214,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('historical');
       });
       it('should return history value if no latest value (empty string)', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_newest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_newest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -186,9 +238,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('historical');
       });
       it('should return history value if no latest value (empty array)', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_newest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_newest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -203,9 +262,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('historical');
       });
       it('should return history value if no latest value (empty object)', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_newest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_newest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -220,9 +286,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('historical');
       });
       it('should return latest value if both latest and history values', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_newest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_newest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -238,9 +311,16 @@ export default ({ getService }: FtrProviderContext) => {
       });
 
       it('should handle missing values', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_newest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_newest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
         const doc = {};
 
@@ -251,9 +331,16 @@ export default ({ getService }: FtrProviderContext) => {
     });
     describe('prefer_oldest_value operator', () => {
       it('should return history value if no latest value', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_oldest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_oldest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -267,9 +354,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('historical');
       });
       it('should return latest value if no history value (undefined)', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_oldest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_oldest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -284,9 +378,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('latest');
       });
       it('should return latest value if no history value (empty string)', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_oldest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_oldest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -301,9 +402,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('latest');
       });
       it('should return latest value if no history value (empty array)', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_oldest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_oldest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -318,9 +426,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('latest');
       });
       it('should return latest value if no history value (empty object)', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_oldest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_oldest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
@@ -335,9 +450,16 @@ export default ({ getService }: FtrProviderContext) => {
         expect(resultDoc.test_field).to.eql('latest');
       });
       it('should return history value if both latest and history values', async () => {
-        const op: FieldRetentionOperator = {
-          operation: 'prefer_oldest_value',
-          field: 'test_field',
+        const op: FieldDescription = {
+          retention: { operation: 'prefer_oldest_value' },
+          destination: 'test_field',
+          source: 'test_field',
+          aggregation: {
+            type: 'terms',
+            limit: 10,
+            lookbackPeriod: undefined,
+          },
+          mapping: { type: 'keyword' },
         };
 
         const doc = {
diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts
index 3dda5f462afdd..f57c2692c7253 100644
--- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts
@@ -109,7 +109,7 @@ export default ({ getService }: FtrProviderContext) => {
         {
           index: 2,
           message:
-            'Invalid entity type "invalid_entity", expected to be one of: user, host, service',
+            'Invalid entity type "invalid_entity", expected to be one of: user, host, service, universal',
         },
         {
           index: 3,