From dd0da3bdb461ab6a62189531dfdb64006c5b5c63 Mon Sep 17 00:00:00 2001 From: Mariya Ivanina Date: Thu, 29 Jul 2021 13:47:13 +0300 Subject: [PATCH] fixed no defect analysis for small logs --- ..._items_including_no_defect_small_logs.json | 43 +++++++++++++++++++ service/auto_analyzer_service.py | 20 ++++++--- test/test_esquery.py | 35 +++++++++++++++ 3 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 fixtures/query_analyze_items_including_no_defect_small_logs.json diff --git a/fixtures/query_analyze_items_including_no_defect_small_logs.json b/fixtures/query_analyze_items_including_no_defect_small_logs.json new file mode 100644 index 00000000..a71f23ab --- /dev/null +++ b/fixtures/query_analyze_items_including_no_defect_small_logs.json @@ -0,0 +1,43 @@ +{ + "size": 10, + "sort": ["_score", + {"start_time": "desc"}], + "query": { + "bool": { + "filter": [ + {"range": {"log_level": {"gte": 40000}}}, + {"exists": {"field": "issue_type"}}, + {"term": {"is_merged": true}} + ], + "must_not": [ + {"wildcard": {"issue_type": "TI*"}}, + {"wildcard": {"issue_type": "ti*"}}, + {"term": {"test_item": "123"}}, + {"wildcard": {"message": "*"}} + ], + "must": [ + {"term": { + "unique_id": "unique" + }}, + {"term": { + "test_case_hash": 1 + }}, + {"term": { + "launch_name": { + "value": "Launch name" + } + }}, + {"more_like_this": { + "fields": ["merged_small_logs"], + "like": "hello world", + "min_doc_freq": 1, + "min_term_freq": 1, + "minimum_should_match": "5<80%", + "max_query_terms": 50, + "boost": 1.0 + }} + ], + "should": [] + } + } +} \ No newline at end of file diff --git a/service/auto_analyzer_service.py b/service/auto_analyzer_service.py index b99bacfb..e5ba3f61 100644 --- a/service/auto_analyzer_service.py +++ b/service/auto_analyzer_service.py @@ -174,8 +174,7 @@ def build_query_with_no_defect(self, launch, log, size=10): "bool": { "filter": [ {"range": {"log_level": {"gte": utils.ERROR_LOGGING_LEVEL}}}, - {"exists": {"field": "issue_type"}}, - {"term": {"is_merged": False}} + {"exists": {"field": "issue_type"}} ], "must_not": [ {"wildcard": {"issue_type": "TI*"}}, @@ -189,10 +188,19 @@ def build_query_with_no_defect(self, launch, log, size=10): "should": [] }}} query = self.add_constraints_for_launches_into_query(query, launch) - query["query"]["bool"]["must"].append( - self.build_more_like_this_query(min_should_match, - log["_source"]["message"], - field_name="message")) + if log["_source"]["message"].strip(): + query["query"]["bool"]["filter"].append({"term": {"is_merged": False}}) + query["query"]["bool"]["must"].append( + self.build_more_like_this_query(min_should_match, + log["_source"]["message"], + field_name="message")) + else: + query["query"]["bool"]["filter"].append({"term": {"is_merged": True}}) + query["query"]["bool"]["must_not"].append({"wildcard": {"message": "*"}}) + query["query"]["bool"]["must"].append( + self.build_more_like_this_query(min_should_match, + log["_source"]["merged_small_logs"], + field_name="merged_small_logs")) return query def find_relevant_with_no_defect(self, candidates_with_no_defect, boosting_config): diff --git a/test/test_esquery.py b/test/test_esquery.py index 061d0535..b2f83a45 100644 --- a/test/test_esquery.py +++ b/test/test_esquery.py @@ -47,6 +47,8 @@ def setUp(self): "suggest_query_all_logs_nonempty_stacktrace_launches_with_the_same_name.json" self.suggest_query_merged_small_logs_search = "suggest_query_merged_small_logs_search.json" self.query_analyze_items_including_no_defect = "query_analyze_items_including_no_defect.json" + self.query_analyze_items_including_no_defect_small_logs =\ + "query_analyze_items_including_no_defect_small_logs.json" self.app_config = { "esHost": "http://localhost:9200", "esUser": "", @@ -602,6 +604,39 @@ def test_build_query_with_no_defect(self): query_from_service.should.equal(demo_query) + @utils.ignore_warnings + def test_build_query_with_no_defect_small_logs(self): + """Tests building analyze query with finding No defect for small logs""" + search_cfg = TestEsQuery.get_default_search_config() + + launch = launch_objects.Launch(**{ + "analyzerConfig": {"analyzerMode": "LAUNCH_NAME", "numberOfLogLines": -1}, + "launchId": 12, + "launchName": "Launch name", + "project": 1}) + log = { + "_id": 1, + "_index": 1, + "_source": { + "unique_id": "unique", + "test_case_hash": 1, + "test_item": "123", + "message": "", + "merged_small_logs": "hello world", + "detected_message": "", + "detected_message_with_numbers": "", + "stacktrace": "", + "only_numbers": "1", + "found_exceptions": "", + "potential_status_codes": "300 401", + "found_tests_and_methods": ""}} + query_from_service = AutoAnalyzerService( + self.app_config, search_cfg).build_query_with_no_defect(launch, log) + demo_query = utils.get_fixture( + self.query_analyze_items_including_no_defect_small_logs, to_json=True) + + query_from_service.should.equal(demo_query) + if __name__ == '__main__': unittest.main()