improvement(perf): add validation rules for latency decorator

Added validation rules for results sent by
`latency_calculator_decorator` to Argus.
Each workload and result name (nemesis, predefined step) may set own
rules.

Current rules were created based on existing results - to pass typical
good results.

closes: scylladb#9237

configurations/performance/latency-decorator-error-thresholds-nemesis-ent-tablets.yaml

@@ -0,0 +1,51 @@
+latency_decorator_error_thresholds:
+  write:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 10800
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 2500
+    decommission_nodes:
+      duration:
+        fixed_limit: 1800
+    replace_node:
+      duration:
+        fixed_limit: 3600
+
+  read:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 3600
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 3000
+    decommission_nodes:
+      duration:
+        fixed_limit: 1800
+    replace_node:
+      duration:
+        fixed_limit: 3000
+
+  mixed:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 5000
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 2500
+    decommission_nodes:
+      duration:
+        fixed_limit: 1800
+    replace_node:
+      duration:
+        fixed_limit: 3200

configurations/performance/latency-decorator-error-thresholds-nemesis-oss-tablets.yaml

@@ -0,0 +1,51 @@
+latency_decorator_error_thresholds:
+  write:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 10800  # 3h this one typically will fail
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 7200
+    decommission_nodes:
+      duration:
+        fixed_limit: 8000
+    replace_node:
+      duration:
+        fixed_limit: 3600
+
+  read:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 10800  # 3h this one typically will fail
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 2600
+    decommission_nodes:
+      duration:
+        fixed_limit: 2600
+    replace_node:
+      duration:
+        fixed_limit: 2600
+
+  mixed:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 10800  # 3h this one typically will fail
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 2800
+    decommission_nodes:
+      duration:
+        fixed_limit: 2600
+    replace_node:
+      duration:
+        fixed_limit: 2600

configurations/performance/latency-decorator-error-thresholds-nemesis-oss-vnodes.yaml

@@ -0,0 +1,51 @@
+latency_decorator_error_thresholds:
+  write:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 28000
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 10000
+    decommission_nodes:
+      duration:
+        fixed_limit: 10000
+    replace_node:
+      duration:
+        fixed_limit: 2500
+
+  read:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 3000
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 2500
+    decommission_nodes:
+      duration:
+        fixed_limit: 2800
+    replace_node:
+      duration:
+        fixed_limit: 1500
+
+  mixed:
+    _mgmt_repair_cli:
+      duration:
+        fixed_limit: 5000
+    _terminate_and_wait:
+      duration:
+        fixed_limit: 500
+    add_new_nodes:
+      duration:
+        fixed_limit: 3000
+    decommission_nodes:
+      duration:
+        fixed_limit: 3600
+    replace_node:
+      duration:
+        fixed_limit: 1700

configurations/performance/latency-decorator-error-thresholds-steps-ent-tablets.yaml

@@ -0,0 +1,98 @@
+latency_decorator_error_thresholds:
+  write:
+    "300000":
+      P99 write:
+        fixed_limit: 1000
+    "400000":
+      P99 write:
+        fixed_limit: 1000
+    unthrottled:
+      P90 write:
+        fixed_limit: null
+      P99 write:
+        fixed_limit: null
+      Throughput write:
+        best_pct: 10
+
+  read:
+    "150000":
+      P90 read:
+        fixed_limit: 1
+      P99 read:
+        fixed_limit: 1
+    "300000":
+      P90 read:
+        fixed_limit: 1
+      P99 read:
+        fixed_limit: 1
+    "450000":
+      P90 read:
+        fixed_limit: 1
+      P99 read:
+        fixed_limit: 3
+    "600000":
+      P90 read:
+        fixed_limit: 1.5
+      P99 read:
+        fixed_limit: 50
+    "700000":
+      P90 read:
+        fixed_limit: 3
+      P99 read:
+        fixed_limit: 50
+    unthrottled:
+      P90 read:
+        fixed_limit: null
+      P99 read:
+        fixed_limit: null
+      Throughput read:
+        best_pct: 10
+
+  mixed:
+    "50000":
+      P90 write:
+          fixed_limit: 1
+      P90 read:
+          fixed_limit: 1
+      P99 write:
+        fixed_limit: 2.5
+      P99 read:
+        fixed_limit: 2.5
+    "150000":
+      P90 write:
+        fixed_limit: 1
+      P90 read:
+        fixed_limit: 1.7
+      P99 write:
+        fixed_limit: 3
+      P99 read:
+        fixed_limit: 3
+    "300000":
+      P90 write:
+        fixed_limit: 3
+      P90 read:
+        fixed_limit: 3
+      P99 write:
+        fixed_limit: 5
+      P99 read:
+        fixed_limit: 5
+    "450000":
+      P90 write:
+        fixed_limit: 3
+      P90 read:
+        fixed_limit: 4
+      P99 write:
+        fixed_limit: 15
+      P99 read:
+        fixed_limit: 15
+    unthrottled:
+      P90 write:
+        fixed_limit: null
+      P90 read:
+        fixed_limit: null
+      P99 write:
+        fixed_limit: null
+      P99 read:
+        fixed_limit: null
+      Throughput write:
+        best_pct: 10

configurations/performance/latency-decorator-error-thresholds-steps-ent-vnodes.yaml

@@ -0,0 +1,98 @@
+latency_decorator_error_thresholds:
+  write:
+    "300000":
+      P99 write:
+        fixed_limit: 1000
+    "400000":
+      P99 write:
+        fixed_limit: 1000
+    unthrottled:
+      P90 write:
+        fixed_limit: null
+      P99 write:
+        fixed_limit: null
+      Throughput write:
+        best_pct: 10
+
+  read:
+    "150000":
+      P90 read:
+        fixed_limit: 1
+      P99 read:
+        fixed_limit: 1
+    "300000":
+      P90 read:
+        fixed_limit: 1
+      P99 read:
+        fixed_limit: 1
+    "450000":
+      P90 read:
+        fixed_limit: 1
+      P99 read:
+        fixed_limit: 3
+    "600000":
+      P90 read:
+        fixed_limit: 1.5
+      P99 read:
+        fixed_limit: 50
+    "700000":
+      P90 read:
+        fixed_limit: 3
+      P99 read:
+        fixed_limit: 50
+    unthrottled:
+      P90 read:
+        fixed_limit: null
+      P99 read:
+        fixed_limit: null
+      Throughput read:
+        best_pct: 10
+
+  mixed:
+    "50000":
+      P90 write:
+          fixed_limit: 1
+      P90 read:
+          fixed_limit: 1
+      P99 write:
+        fixed_limit: 2.5
+      P99 read:
+        fixed_limit: 2.5
+    "150000":
+      P90 write:
+        fixed_limit: 1
+      P90 read:
+        fixed_limit: 1.7
+      P99 write:
+        fixed_limit: 3
+      P99 read:
+        fixed_limit: 3
+    "300000":
+      P90 write:
+        fixed_limit: 3
+      P90 read:
+        fixed_limit: 3
+      P99 write:
+        fixed_limit: 5
+      P99 read:
+        fixed_limit: 5
+    "450000":
+      P90 write:
+        fixed_limit: 3
+      P90 read:
+        fixed_limit: 4
+      P99 write:
+        fixed_limit: 15
+      P99 read:
+        fixed_limit: 15
+    unthrottled:
+      P90 write:
+        fixed_limit: null
+      P90 read:
+        fixed_limit: null
+      P99 write:
+        fixed_limit: null
+      P99 read:
+        fixed_limit: null
+      Throughput write:
+        best_pct: 10

defaults/test_default.yaml

@@ -264,3 +264,27 @@ skip_test_stages: {}
 n_db_zero_token_nodes: 0
 zero_token_instance_type_db: 'i4i.large'
 use_zero_nodes: false
+
+latency_decorator_error_thresholds:
+  write:
+    default:
+      P90 write:
+        fixed_limit: 5
+      P99 write:
+        fixed_limit: 10
+  read:
+    default:
+      P90 read:
+        fixed_limit: 5
+      P99 read:
+        fixed_limit: 10
+  mixed:
+    default:
+      P90 write:
+        fixed_limit: 5
+      P90 read:
+          fixed_limit: 5
+      P99 write:
+          fixed_limit: 10
+      P99 read:
+          fixed_limit: 10

jenkins-pipelines/performance/branch-perf-v15/scylla-enterprise/perf-regression/scylla-enterprise-perf-regression-latency-650gb-with-nemesis-tablets.jenkinsfile

@@ -6,7 +6,7 @@ def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm)
 perfRegressionParallelPipeline(
     backend: "aws",
     test_name: "performance_regression_test.PerformanceRegressionTest",
-    test_config: """["test-cases/performance/perf-regression-latency-650gb-with-nemesis.yaml", "configurations/disable_kms.yaml"]""",
+    test_config: """["test-cases/performance/perf-regression-latency-650gb-with-nemesis.yaml", "configurations/disable_kms.yaml", "configurations/performance/latency-decorator-error-thresholds-nemesis-ent.yaml", "configurations/performance/latency-decorator-error-thresholds-nemesis-ent-tablets.yaml"]""",
     sub_tests: ["test_latency_write_with_nemesis", "test_latency_read_with_nemesis", "test_latency_mixed_with_nemesis"],
     test_email_title: "latency during operations / tablets",
     perf_extra_jobs_to_compare: "scylla-master/perf-regression/scylla-master-perf-regression-latency-650gb-with-nemesis-tablets",

jenkins-pipelines/performance/branch-perf-v15/scylla-enterprise/perf-regression/scylla-enterprise-perf-regression-latency-650gb-with-nemesis.jenkinsfile

@@ -6,7 +6,7 @@ def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm)
 perfRegressionParallelPipeline(
     backend: "aws",
     test_name: "performance_regression_test.PerformanceRegressionTest",
-    test_config: """["test-cases/performance/perf-regression-latency-650gb-with-nemesis.yaml", "configurations/tablets_disabled.yaml", "configurations/disable_kms.yaml"]""",
+    test_config: """["test-cases/performance/perf-regression-latency-650gb-with-nemesis.yaml", "configurations/tablets_disabled.yaml", "configurations/disable_kms.yaml", "configurations/performance/latency-decorator-error-thresholds-nemesis-ent-vnodes.yaml"]""",
     sub_tests: ["test_latency_write_with_nemesis", "test_latency_read_with_nemesis", "test_latency_mixed_with_nemesis"],
     perf_extra_jobs_to_compare: """["scylla-enterprise/scylla-enterprise-perf-regression-latency-650gb-with-nemesis","scylla-enterprise/perf-regression/scylla-enterprise-perf-regression-latency-650gb-with-nemesis"]""",
 )