From 00d965151c3ba255dd376e7e9dfe1e542cdb31e9 Mon Sep 17 00:00:00 2001 From: Guillaume ERETEO Date: Tue, 12 Mar 2024 17:07:12 +0100 Subject: [PATCH 1/3] POC to preindex relatinships types --- src/ctia/stores/es/mapping.clj | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/ctia/stores/es/mapping.clj b/src/ctia/stores/es/mapping.clj index bce3750041..8692482019 100644 --- a/src/ctia/stores/es/mapping.clj +++ b/src/ctia/stores/es/mapping.clj @@ -1,4 +1,5 @@ (ns ctia.stores.es.mapping + (:require [clojure.string :as string]) (:refer-clojure :exclude [identity])) ;; This provides a reasonable default mapping for all of our entities. @@ -260,6 +261,36 @@ {:properties {:type token :text text}}) +(def type-simple-pattern + (->> [:actor + :asset + :asset-mapping + :asset-properties + :attack-pattern + :campaign + :casebook + :coa + :data-table + :event + :feed + :feedback + :identity + :identity-assertion + :incident + :indicator + :investigation + :judgement + :malware + :note + :relationship + :sighting + :target-record + :tool + :vulnerability + :weakness] + (map name) + (string/join "|"))) + (def store-settings {:number_of_replicas 1 :number_of_shards 1 @@ -284,6 +315,10 @@ :english_stemmer {:type "stemmer" :language "english"}} ;; when applying filters, order matters + :tokenizer + {:type_tokenizer + {:type "simple_pattern", + :pattern type-simple-pattern}} :analyzer {:default ;; same as text_analyzer {:type "custom" @@ -298,6 +333,10 @@ :filter ["lowercase" "ctia_stemmer" "english_stemmer"]} + :type_analyzer { + :tokenizer "type_tokenizer" + :filter [ "fingerprint"] + } :search_analyzer {:type "custom" :tokenizer "standard" From b333934bd406b8d8e4db7085deb720d34551ea1c Mon Sep 17 00:00:00 2001 From: Guillaume ERETEO Date: Fri, 26 Apr 2024 15:58:40 +0200 Subject: [PATCH 2/3] mapping + test --- src/ctia/entity/relationship.clj | 9 ++++-- src/ctia/stores/es/init.clj | 13 +++----- test/ctia/entity/relationship_test.clj | 44 +++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/ctia/entity/relationship.clj b/src/ctia/entity/relationship.clj index 9c692bf2ab..647220f1f7 100644 --- a/src/ctia/entity/relationship.clj +++ b/src/ctia/entity/relationship.clj @@ -18,6 +18,11 @@ [schema-tools.core :as st] [schema.core :as s])) +(def node-type + (assoc em/token + :fields {:type {:type "text" + :analyzer "type_analyzer"}})) + (def relationship-mapping {"relationship" {:dynamic false @@ -28,8 +33,8 @@ em/sourcable-entity-mapping em/stored-entity-mapping {:relationship_type em/token - :source_ref em/token - :target_ref em/token})}}) + :source_ref node-type + :target_ref node-type})}}) (def-es-store RelationshipStore :relationship diff --git a/src/ctia/stores/es/init.clj b/src/ctia/stores/es/init.clj index fa4cfaecf9..5605b6b7de 100644 --- a/src/ctia/stores/es/init.clj +++ b/src/ctia/stores/es/init.clj @@ -47,8 +47,9 @@ settings {:refresh_interval refresh_interval :number_of_shards shards :number_of_replicas replicas} - mappings (cond-> (get-in entity-fields [entity :es-mapping] mappings) - (< 5 version) (some-> first val)) + mappings (some-> (get-in entity-fields [entity :es-mapping] mappings) + first + val) searchable-fields (get-in entity-fields [entity :searchable-fields])] {:index indexname :props (assoc props :write-index write-index) @@ -89,21 +90,17 @@ (s/defn update-mappings! [{:keys [conn index] {:keys [mappings]} :config} :- ESConnState] - (let [[entity-type type-mappings] (when (= (:version conn) 5) - (first mappings)) - update-body (or type-mappings mappings)] (try (log/info "updating mapping: " index) (index/update-mappings! conn index - entity-type - update-body) + mappings) (catch clojure.lang.ExceptionInfo e (log/error "cannot update mapping. You probably tried to update the mapping of an existing field. It's only possible to add new field to existing mappings. If you need to modify the type of a field in an existing index, you must perform a migration" (assoc (ex-data e) :conn conn :mappings mappings)) - (system-exit-error))))) + (system-exit-error)))) (s/defn refresh-mappings! [{:keys [conn index] diff --git a/test/ctia/entity/relationship_test.clj b/test/ctia/entity/relationship_test.clj index 7e3ec0c722..3021a01df8 100644 --- a/test/ctia/entity/relationship_test.clj +++ b/test/ctia/entity/relationship_test.clj @@ -6,7 +6,7 @@ [ctia.test-helpers [access-control :refer [access-control-test]] [auth :refer [all-capabilities]] - [core :as helpers :refer [POST]] + [core :as helpers :refer [POST GET]] [crud :refer [entity-crud-test]] [aggregate :refer [test-metric-routes]] [fake-whoami-service :as whoami-helpers] @@ -43,6 +43,48 @@ "http://ex.tld/ctia/relationship/relationship-456"]) (dissoc :id))) +(deftest test-relationship-source-target-type-search + (test-for-each-store-with-app + (fn [app] + (establish-user! app) + (testing "POST /ctia/relationship" + (let [new-relationship-1 new-relationship + new-relationship-2 (update new-relationship-1 + :source_ref + str/replace + "judgement" + "sighting") + {status-1 :status + rel1 :parsed-body} + (POST app + "ctia/relationship" + :body new-relationship-1 + :headers {"Authorization" "45c1f5e3f05d0"}) + {status-2 :status + rel2 :parsed-body} + (POST app + "ctia/relationship" + :body new-relationship-2 + :headers {"Authorization" "45c1f5e3f05d0"}) + test-plan [{:expected [rel1 rel2] + :query "target_ref.type=indicator"} + {:expected [rel1] + :query "source_ref.type=judgement"} + {:expected [rel2] + :query "source_ref.type=sighting"} + {:expected [] + :query "source_ref.type=incident"}]] + (assert (= 201 status-1)) + (assert (= 201 status-2)) + (doseq [{:keys [expected query]} test-plan] + (is (= (set expected) + (-> (GET app + "ctia/relationship/search" + :query-params {:query query} + :headers {"Authorization" "45c1f5e3f05d0"}) + :parsed-body + set))))))))) + (deftest test-relationship-routes-bad-reference (test-for-each-store-with-app (fn [app] From 535ac8b652b08504b9f4f442cdfd3d52e26b502b Mon Sep 17 00:00:00 2001 From: Guillaume ERETEO Date: Wed, 26 Jun 2024 15:23:20 +0200 Subject: [PATCH 3/3] wip --- src/ctia/stores/es/mapping.clj | 8 ++-- test/ctia/entity/relationship_test.clj | 53 +++++++++++++++++++------- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/src/ctia/stores/es/mapping.clj b/src/ctia/stores/es/mapping.clj index 8692482019..b31877587f 100644 --- a/src/ctia/stores/es/mapping.clj +++ b/src/ctia/stores/es/mapping.clj @@ -263,9 +263,7 @@ (def type-simple-pattern (->> [:actor - :asset - :asset-mapping - :asset-properties + "asset([-](mapping|properties))?" :attack-pattern :campaign :casebook @@ -274,8 +272,8 @@ :event :feed :feedback - :identity :identity-assertion + :identity :incident :indicator :investigation @@ -288,7 +286,7 @@ :tool :vulnerability :weakness] - (map name) + (map (comp #(string/replace % "-" "\\-") name)) (string/join "|"))) (def store-settings diff --git a/test/ctia/entity/relationship_test.clj b/test/ctia/entity/relationship_test.clj index 3021a01df8..063a8fc8b7 100644 --- a/test/ctia/entity/relationship_test.clj +++ b/test/ctia/entity/relationship_test.clj @@ -48,42 +48,69 @@ (fn [app] (establish-user! app) (testing "POST /ctia/relationship" - (let [new-relationship-1 new-relationship + (let [new-relationship-1 + (-> new-relationship-minimal + (assoc + :source_ref (str "http://example.com/ctia/judgement/judgement-" + "f9832ac2-ee90-4e18-9ce6-0c4e4ff61a7a") + :target_ref (str "http://example.com/ctia/indicator/indicator-" + "8c94ca8d-fb2b-4556-8517-8e6923d8d3c7")) + (dissoc :id)) new-relationship-2 (update new-relationship-1 :source_ref str/replace "judgement" "sighting") - {status-1 :status - rel1 :parsed-body} + new-relationship-3 (-> new-relationship-1 + (update :source_ref + str/replace + "judgement" + "asset") + (update :target_ref + str/replace + "indicator" + "asset-properties")) + {status-1 :status rel1 :parsed-body} (POST app "ctia/relationship" :body new-relationship-1 :headers {"Authorization" "45c1f5e3f05d0"}) - {status-2 :status - rel2 :parsed-body} + {status-2 :status rel2 :parsed-body} (POST app "ctia/relationship" :body new-relationship-2 :headers {"Authorization" "45c1f5e3f05d0"}) + {status-3 :status rel3 :parsed-body} + (POST app + "ctia/relationship" + :body new-relationship-3 + :headers {"Authorization" "45c1f5e3f05d0"}) test-plan [{:expected [rel1 rel2] :query "target_ref.type=indicator"} {:expected [rel1] :query "source_ref.type=judgement"} {:expected [rel2] :query "source_ref.type=sighting"} + {:expected [rel3] + :query "source_ref.type=asset"} + {:expected [rel3] + :query "target_ref.type=\"asset-properties\""} + {:expected [] + :query "target_ref.type=asset"} {:expected [] - :query "source_ref.type=incident"}]] + :query "source_ref.type=malware"}]] (assert (= 201 status-1)) (assert (= 201 status-2)) + (assert (= 201 status-3)) (doseq [{:keys [expected query]} test-plan] - (is (= (set expected) - (-> (GET app - "ctia/relationship/search" - :query-params {:query query} - :headers {"Authorization" "45c1f5e3f05d0"}) - :parsed-body - set))))))))) + (testing query + (is (= (set expected) + (-> (GET app + "ctia/relationship/search" + :query-params {:query query} + :headers {"Authorization" "45c1f5e3f05d0"}) + :parsed-body + set)))))))))) (deftest test-relationship-routes-bad-reference (test-for-each-store-with-app